From 26849296883df7517af7d474210c343c943b48e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 15 Mar 2013 11:59:45 +0100
Subject: erts: Use fast path for list_to_binary([Bin]) case

A common case is to wrap an argument to list_to_binary in a list
to ensure conversion can happen even though the argument may already
be a binary. Use fast path for this case.
---
 erts/emulator/beam/binary.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 33abac2f3d..c7926f18af 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -447,6 +447,7 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
 {
     Eterm bin;
+    Eterm h,t;
     ErlDrvSizeT size;
     byte* bytes;
 #ifdef DEBUG
@@ -459,6 +460,16 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
     if (is_not_list(arg)) {
 	goto error;
     }
+    /* check for [binary()] case */
+    h = CAR(list_val(arg));
+    t = CDR(list_val(arg));
+    if (is_binary(h) && is_nil(t) && !(
+		HEADER_SUB_BIN == *(binary_val(h)) && (
+		    ((ErlSubBin *)binary_val(h))->bitoffs != 0 ||
+		    ((ErlSubBin *)binary_val(h))->bitsize != 0
+		))) {
+	return h;
+    }
     switch (erts_iolist_size(arg, &size)) {
     case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT);
     case ERTS_IOLIST_TYPE: goto error;
-- 
cgit v1.2.3


From 75763bd57338b3efe8300ece094aacc8e1a7a467 Mon Sep 17 00:00:00 2001
From: Dan Gudmundsson <dgud@erlang.org>
Date: Mon, 25 Mar 2013 12:44:20 +0100
Subject: erts: Change erlang:open_port spawn to handle unicode

Previously only 'spawn_executable' handled unicode input.
Also change 'cd' option to always handle unicode.
Update open_port documentation and tests
---
 erts/emulator/beam/erl_bif_port.c | 69 ++++++++++-----------------------------
 erts/emulator/beam/erl_unicode.c  | 12 ++++++-
 erts/emulator/beam/global.h       |  6 ++++
 3 files changed, 34 insertions(+), 53 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 109c54fd7f..5b26e53c9f 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -784,43 +784,29 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 	    goto badarg;
 	}
     
-	if (*tp == am_spawn || *tp == am_spawn_driver) {	/* A process port */
+	if (*tp == am_spawn || *tp == am_spawn_driver || *tp == am_spawn_executable) {	/* A process port */
+	    int encoding;
 	    if (arity != make_arityval(2)) {
 		goto badarg;
 	    }
 	    name = tp[1];
-	    if (is_atom(name)) {
-		name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP,
-					       atom_tab(atom_val(name))->len+1);
-		sys_memcpy((void *) name_buf,
-			   (void *) atom_tab(atom_val(name))->name, 
-			   atom_tab(atom_val(name))->len);
-		name_buf[atom_tab(atom_val(name))->len] = '\0';
-	    } else if ((i = is_string(name))) {
-		name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
-		if (intlist_to_buf(name, name_buf, i) != i)
-		    erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-		name_buf[i] = '\0';
-	    } else {
+	    encoding = erts_get_native_filename_encoding();
+	    /* Do not convert the command to utf-16le yet, do that in win32 specific code */
+	    /* since the cmd is used for comparsion with drivers names and copied to port info */
+	    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+		encoding = ERL_FILENAME_UTF8;
+	    }
+	    if ((name_buf = erts_convert_filename_to_encoding(name, NULL, 0, ERTS_ALC_T_TMP,0,1, encoding, NULL))
+		== NULL) {
 		goto badarg;
 	    }
+
 	    if (*tp == am_spawn_driver) {
 		opts.spawn_type = ERTS_SPAWN_DRIVER;
+	    } else if (*tp == am_spawn_executable) {
+		opts.spawn_type = ERTS_SPAWN_EXECUTABLE;
 	    }
-	    driver = &spawn_driver;
-	} else if (*tp == am_spawn_executable) {	/* A program */
-	    /*
-	     * {spawn_executable,Progname}
-	     */
-	    
-	    if (arity != make_arityval(2)) {
-		goto badarg;
-	    }
-	    name = tp[1];
-	    if ((name_buf = erts_convert_filename_to_native(name, NULL, 0, ERTS_ALC_T_TMP,0,1, NULL)) == NULL) {
-		goto badarg;
-	    }
-	    opts.spawn_type = ERTS_SPAWN_EXECUTABLE;
+
 	    driver = &spawn_driver;
 	} else if (*tp == am_fd) { /* An fd port */
 	    int n;
@@ -861,29 +847,8 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
     }
 
     if (edir != NIL) {
-	/* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles 
-	   for spawn_executable... */
-	if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) {
-	    Eterm iolist;
-	    DeclareTmpHeap(heap,4,p);
-	    int r;
-	    
-	    UseTmpHeap(4,p);
-	    heap[0] = edir;
-	    heap[1] = make_list(heap+2);
-	    heap[2] = make_small(0);
-	    heap[3] = NIL;
-	    iolist = make_list(heap);
-	    r = erts_iolist_to_buf(iolist, (char*) dir, MAXPATHLEN);
-	    UnUseTmpHeap(4,p);
-	    if (ERTS_IOLIST_TO_BUF_FAILED(r)) {
-		goto badarg;
-	    }
-	    opts.wd = (char *) dir;
-	} else {
-	    if ((opts.wd = erts_convert_filename_to_native(edir, NULL, 0, ERTS_ALC_T_TMP,0,1,NULL)) == NULL) {
-		goto badarg;
-	    }
+	if ((opts.wd = erts_convert_filename_to_native(edir, NULL, 0, ERTS_ALC_T_TMP,0,1,NULL)) == NULL) {
+	    goto badarg;
 	}
     }
 
@@ -961,11 +926,11 @@ static char **convert_args(Eterm l)
     int n;
     int i = 0;
     Eterm str;
-    /* We require at least one element in list (argv[0]) */
     if (is_not_list(l) && is_not_nil(l)) {
 	return NULL;
     }
     n = list_length(l);
+    /* We require at least one element in argv[0] + NULL at end */
     pp = erts_alloc(ERTS_ALC_T_TMP, (n + 2) * sizeof(char **));
     pp[i++] = erts_default_arg0;
     while (is_list(l)) {
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index ad6f8b993a..71794e1101 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1981,9 +1981,19 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2)
  * string routines, that will certainly fail on some OS.
  */
 
-char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_size, ErtsAlcType_t alloc_type, int allow_empty, int allow_atom, Sint *used)
+char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_size,
+				      ErtsAlcType_t alloc_type, int allow_empty,
+				      int allow_atom, Sint *used)
 {
     int encoding = erts_get_native_filename_encoding();
+    return erts_convert_filename_to_encoding(name, statbuf, statbuf_size, alloc_type,
+					     allow_empty, allow_atom, encoding, used);
+}
+
+char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbuf_size,
+					ErtsAlcType_t alloc_type, int allow_empty,
+					int allow_atom, int encoding, Sint *used)
+{
     char* name_buf = NULL;
 
     if ((allow_atom && is_atom(name)) || 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 26ed5f82c1..f7e4c81181 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -788,6 +788,12 @@ char *erts_convert_filename_to_native(Eterm name, char *statbuf,
 				      ErtsAlcType_t alloc_type, 
 				      int allow_empty, int allow_atom,
 				      Sint *used /* out */);
+char *erts_convert_filename_to_encoding(Eterm name, char *statbuf,
+					size_t statbuf_size,
+					ErtsAlcType_t alloc_type,
+					int allow_empty, int allow_atom,
+					int encoding,
+					Sint *used /* out */);
 Eterm erts_convert_native_to_filename(Process *p, byte *bytes);
 Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left,
 			Uint *num_built, Uint *num_eaten, Eterm tail);
-- 
cgit v1.2.3


From e753b347a4c88d169992ceaa87e775aebf935bf3 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 4 Jun 2013 12:04:04 +0200
Subject: erts: Use "+Muacul de" as default

---
 erts/emulator/beam/erl_alloc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index a547191d6d..65f1f64c03 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -75,9 +75,9 @@
 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45
 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85
 
-#define ERTS_ALC_DEFAULT_ACUL 0
-#define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0
-#define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0
+#define ERTS_ALC_DEFAULT_ACUL ERTS_ALC_DEFAULT_ENABLED_ACUL
+#define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC
+#define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC
 
 #ifndef ERTS_SMP
 #  undef ERTS_ALC_DEFAULT_ACUL
-- 
cgit v1.2.3


From 55d6274f67cafe62e4923a6369c99a45822cb767 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Wei=C3=9Fl?= <jargon@molb.org>
Date: Fri, 14 Jun 2013 15:23:40 +0200
Subject: Fix external term format BIFs on floating point middle-endian
 machines

This complements 933e701 (OTP-10209).

Simple error example:
1> <<131,70,63,240,0,0,0,0,0,0>> = term_to_binary(1.0, [{minor_version,1}]).
** exception error: no match of right hand side value <<131,70,0,0,0,0,63,240,0,0>>
2> 1.0 = binary_to_term(<<131,70,63,240,0,0,0,0,0,0>>).
** exception error: no match of right hand side value 5.299808824e-315

But roundtrip always works:
3> 1.0 = binary_to_term(term_to_binary(1.0, [{minor_version,1}])).
1.0
---
 erts/emulator/beam/external.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 45025ad631..0d56de49c9 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2316,7 +2316,7 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla
 	    GET_DOUBLE(obj, f);
 	    if (dflags & DFLAG_NEW_FLOATS) {
 		*ep++ = NEW_FLOAT_EXT;
-#ifdef WORDS_BIGENDIAN
+#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
 		put_int32(f.fw[0], ep);
 		ep += 4;
 		put_int32(f.fw[1], ep);
@@ -2795,7 +2795,7 @@ dec_term_atom_common:
 		volatile unsigned long *fpexnp = erts_get_current_fp_exception();
 #endif
 
-#ifdef WORDS_BIGENDIAN
+#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
 		ff.fw[0] = get_int32(ep);
 		ep += 4;
 		ff.fw[1] = get_int32(ep);
-- 
cgit v1.2.3


From 29465408c90b2271b68e9559b5482fc6c4fcdde5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Wei=C3=9Fl?= <jargon@molb.org>
Date: Fri, 14 Jun 2013 15:23:54 +0200
Subject: Fix erlang:phash2() on floating point middle-endian machines

This complements 933e701 (OTP-10209). Without this patch the test case
"hash_SUITE:test_phash2/1" fails.

Simple error example:
1> 77147068 = erlang:phash2(1.0).
** exception error: no match of right hand side value 50524433
---
 erts/emulator/beam/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 62caa67ce1..ccc411462e 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1319,7 +1319,7 @@ make_hash2(Eterm term)
 	    {
 		FloatDef ff;
 		GET_DOUBLE(term, ff);
-#if defined(WORDS_BIGENDIAN)
+#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
 		UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
 #else
 		UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12);
-- 
cgit v1.2.3


From 5adbd7d22bccc57e17ed00cac09fe8a336bb39c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Wei=C3=9Fl?= <jargon@molb.org>
Date: Fri, 14 Jun 2013 15:23:59 +0200
Subject: Fix binary matching on floating point middle-endian machines

This complements 933e701 (OTP-10209). Without this patch the test case
"bs_match_misc_SUITE:t_float/1" fails.

Simple error example:
1> <<_,_,_,_,_,_,_,_>> = <<1.25/float>>.
<<63,244,0,0,0,0,0,0>>
2> <<1.25/float>> = <<63,244,0,0,0,0,0,0>>.
** exception error: no match of right hand side value <<63,244,0,0,0,0,0,0>>

The additional test case is added because in a former version of this
patch the ERTS_FP_ERROR_THOROUGH check for NaN/infinity was mistakenly
applied on the still word-switched double.
---
 erts/emulator/beam/erl_bits.c | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 3753b618e1..06d53efb5e 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -484,8 +484,16 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer
 	ERTS_FP_ERROR_THOROUGH(p, f32, return THE_NON_VALUE);
 	f.fd = f32;
     } else {
+#ifdef DOUBLE_MIDDLE_ENDIAN
+	FloatDef ftmp;
+	ftmp.fd = f64;
+	f.fw[0] = ftmp.fw[1];
+	f.fw[1] = ftmp.fw[0];
+	ERTS_FP_ERROR_THOROUGH(p, f.fd, return THE_NON_VALUE);
+#else
 	ERTS_FP_ERROR_THOROUGH(p, f64, return THE_NON_VALUE);
 	f.fd = f64;
+#endif
     }
     mb->offset += num_bits;
     hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT);
-- 
cgit v1.2.3


From 9e45648698862905561baa533831afd4dd23a02a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Wei=C3=9Fl?= <jargon@molb.org>
Date: Fri, 14 Jun 2013 15:24:04 +0200
Subject: Fix binary construction on floating point middle-endian machines

This complements 933e701 (OTP-10209). Without this patch the test cases
"in_guard/1" and "coerce_to_float/1" in bs_construct_SUITE fail.

The added lines in bs_construct_SUITE cover all branches that were not
covered before (small and big numbers if BIT_OFFSET(erts_bin_offset) != 0).
---
 erts/emulator/beam/erl_bits.c | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 06d53efb5e..43eb691338 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1022,8 +1022,13 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags)
 #endif
 	    } else if (is_small(arg)) {
 		u.f64 = (double) signed_val(arg);
+#ifdef DOUBLE_MIDDLE_ENDIAN
+		a = u.i32[1];
+		b = u.i32[0];
+#else
 		a = u.i32[0];
 		b = u.i32[1];
+#endif
 	    } else if (is_big(arg)) {
 		if (big_to_double(arg, &u.f64) < 0) {
 		    return 0;
@@ -1126,21 +1131,42 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags)
 	byte *bptr;
 	double f64;
 	float f32;
+#ifdef DOUBLE_MIDDLE_ENDIAN
+	FloatDef fbuf, ftmp;
+#endif
 	
 	if (num_bits == 64) {
 	    if (is_float(arg)) {
+#ifdef DOUBLE_MIDDLE_ENDIAN
+		FloatDef *fdp = (FloatDef*)(float_val(arg) + 1);
+		ftmp = *fdp;
+#else
 		bptr = (byte *) (float_val(arg) + 1);
+#endif
 	    } else if (is_small(arg)) {
 		f64 = (double) signed_val(arg);
+#ifdef DOUBLE_MIDDLE_ENDIAN
+		ftmp.fd = f64;
+#else
 		bptr = (byte *) &f64;
+#endif
 	    } else if (is_big(arg)) {
 		if (big_to_double(arg, &f64) < 0) {
 		    return 0;
 		}
+#ifdef DOUBLE_MIDDLE_ENDIAN
+		ftmp.fd = f64;
+#else
 		bptr = (byte *) &f64;
+#endif
 	    } else {
 		return 0;
 	    }
+#ifdef DOUBLE_MIDDLE_ENDIAN
+	    fbuf.fw[0] = ftmp.fw[1];
+	    fbuf.fw[1] = ftmp.fw[0];
+	    bptr = fbuf.fb;
+#endif
 	} else if (num_bits == 32) {
 	    if (is_float(arg)) {
 		FloatDef f;
-- 
cgit v1.2.3


From 5d67cc7f4c176ea6ab4d2538443d5fe264152861 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 19 Jun 2013 18:34:56 +0200
Subject: erts: Add new allocator strategy aoffcbf

with better performance than aoffcaobf as we don't have to rearrange
the search tree when there are blocks of equal size.
---
 erts/emulator/beam/erl_alloc.c             |  14 ++-
 erts/emulator/beam/erl_ao_firstfit_alloc.c | 190 +++++++++++++++++++++--------
 erts/emulator/beam/erl_ao_firstfit_alloc.h |  11 +-
 erts/emulator/beam/erl_bestfit_alloc.c     |   6 +-
 4 files changed, 160 insertions(+), 61 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index a547191d6d..7b77bcca58 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -245,7 +245,7 @@ set_default_acul(struct au_init *ip, int acul)
 {
     ip->thr_spec = 1;
     ip->atype = AOFIRSTFIT;
-    ip->init.aoff.bf_within_carrier = 1;
+    ip->init.aoff.flavor = AOFF_AOBF;
     ip->init.util.acul = acul;
 }
 
@@ -558,7 +558,7 @@ static ERTS_INLINE int
 strategy_support_carrier_migration(struct au_init *auip)
 {
     /*
-     * Currently only aoff and aoffcaobf support carrier
+     * Currently only aoff, aoffcbf and aoffcaobf support carrier
      * migration, i.e, type AOFIRSTFIT.
      */
     return auip->atype == AOFIRSTFIT;
@@ -583,7 +583,7 @@ ensure_carrier_migration_support(struct au_init *auip)
     if (!strategy_support_carrier_migration(auip)) {
 	/* Default to aoffcaobf */
 	auip->atype = AOFIRSTFIT;
-	auip->init.aoff.bf_within_carrier = 1;
+	auip->init.aoff.flavor = AOFF_AOBF;
     }
 }
 
@@ -1284,11 +1284,15 @@ handle_au_arg(struct au_init *auip,
 	    }
 	    else if (strcmp("aoff", alg) == 0) {
 		auip->atype = AOFIRSTFIT;
-		auip->init.aoff.bf_within_carrier = 0;
+		auip->init.aoff.flavor = AOFF_AOFF;
+	    }
+	    else if (strcmp("aoffcbf", alg) == 0) {
+		auip->atype = AOFIRSTFIT;
+		auip->init.aoff.flavor = AOFF_BF;
 	    }
 	    else if (strcmp("aoffcaobf", alg) == 0) {
 		auip->atype = AOFIRSTFIT;
-		auip->init.aoff.bf_within_carrier = 1;
+		auip->init.aoff.flavor = AOFF_AOBF;
 	    }
 	    else {
 		bad_value(param, sub_param + 1, alg);
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index f73ac2eb6e..4e6c8b317e 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -34,10 +34,11 @@
  *              sub-tree ('max_sz'). By that we can start from root and keep
  *              left (for low addresses) while dismissing entire sub-trees with
  *              too small blocks.
- * AOFFCBF:
+ * Bestfit within carrier:
  *              The only difference for "bestfit within carrier" is the tree
  *              sorting order. Blocks within the same carrier are sorted
- *              wrt size instead of address. 
+ *              wrt size instead of address. The 'max_sz' field is maintained
+ *              in order to dismiss entire carriers with too small blocks. 
  *
  * Authors: 	Rickard Green/Sverker Eriksson
  */
@@ -67,6 +68,15 @@
 #  define LEFT_VISITED_FLG	(((Uint) 1) << 2)
 #  define RIGHT_VISITED_FLG	(((Uint) 1) << 3)
 #endif
+#ifdef DEBUG
+#  define IS_BF_FLG	        (((Uint) 1) << 4)
+#endif
+
+#define IS_TREE_NODE(N)		(((AOFF_RBTree_t *) (N))->flags & TREE_NODE_FLG)
+#define IS_LIST_ELEM(N)		(!IS_TREE_NODE(((AOFF_RBTree_t *) (N))))
+
+#define SET_TREE_NODE(N)	(((AOFF_RBTree_t *) (N))->flags |= TREE_NODE_FLG)
+#define SET_LIST_ELEM(N)	(((AOFF_RBTree_t *) (N))->flags &= ~TREE_NODE_FLG)
 
 #define IS_RED(N)		(((AOFF_RBTree_t *) (N)) \
 				 && ((AOFF_RBTree_t *) (N))->flags & RED_FLG)
@@ -98,6 +108,16 @@ struct AOFF_RBTree_t_ {
 };
 #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr)
 
+/* BF block nodes keeps list of all with equal size
+ */
+typedef struct {
+    AOFF_RBTree_t t;
+    AOFF_RBTree_t *next;
+}AOFF_RBTreeList_t;
+
+#define LIST_NEXT(N) (((AOFF_RBTreeList_t*) (N))->next)
+#define LIST_PREV(N) (((AOFF_RBTreeList_t*) (N))->t.parent)
+
 typedef struct AOFF_Carrier_t_ AOFF_Carrier_t;
 
 struct AOFF_Carrier_t_ {
@@ -119,11 +139,11 @@ struct AOFF_Carrier_t_ {
 
 #ifdef HARD_DEBUG
 #  define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE)
-#  define HARD_CHECK_TREE(CRR,BF,ROOT,SZ) check_tree(CRR, BF, ROOT, SZ)
-static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint);
+#  define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ)
+static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint);
 #else
 #  define HARD_CHECK_IS_MEMBER(ROOT,NODE)
-#  define HARD_CHECK_TREE(CRR,BF,ROOT,SZ)
+#  define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ)
 #endif
 
 
@@ -161,25 +181,25 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node,
     else ASSERT(new_max == old_max);
 }
 
-static ERTS_INLINE SWord cmp_blocks(int bestfit,
+static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor,
 				    AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs)
 {
     ASSERT(lhs != rhs);
-    ASSERT(!bestfit || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr));
-    if (bestfit) {
+    ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr));
+    if (flavor != AOFF_AOFF) {
 	SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs);
-	if (diff) return diff;
+	if (diff || flavor == AOFF_BF) return diff;
     }
     return (char*)lhs - (char*)rhs;
 }
 
-static ERTS_INLINE SWord cmp_cand_blk(int bestfit,
+static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor,
 				      Block_t* cand_blk, AOFF_RBTree_t* rhs)
 {
-    if (bestfit) {
+    if (flavor != AOFF_AOFF) {
 	if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) {
 	    SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr);
-	    if (diff) return diff;
+	    if (diff || flavor == AOFF_BF) return diff;
 	}
     }
     return (char*)cand_blk - (char*)rhs;
@@ -198,7 +218,7 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*);
 
 /* Generic tree functions used by both carrier and block trees. */
 static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del);
-static void rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk);
+static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk);
 static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size);
 #ifdef HARD_DEBUG
 static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node);
@@ -234,21 +254,21 @@ erts_aoffalc_start(AOFFAllctr_t *alc,
 
     sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t));
 
-    alc->bf_within_carrier              = aoffinit->bf_within_carrier;
+    alc->flavor                         = aoffinit->flavor;
     allctr->mbc_header_size		= sizeof(AOFF_Carrier_t);
     allctr->min_mbc_size		= MIN_MBC_SZ;
     allctr->min_mbc_first_free_size	= MIN_MBC_FIRST_FREE_SZ;
-    allctr->min_block_size		= sizeof(AOFF_RBTree_t);
+    allctr->min_block_size = (aoffinit->flavor == AOFF_BF ?
+			      sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t));
 
-    allctr->vsn_str			= aoffinit->bf_within_carrier ?
-	ERTS_ALC_AOFF_CBF_ALLOC_VSN_STR : ERTS_ALC_AOFF_ALLOC_VSN_STR;
+    allctr->vsn_str			= ERTS_ALC_AOFF_ALLOC_VSN_STR;
 
 
     /* Callback functions */
 
     allctr->get_free_block		= aoff_get_free_block;
     allctr->link_free_block		= aoff_link_free_block;
-    allctr->unlink_free_block	        = aoff_unlink_free_block;
+    allctr->unlink_free_block           = aoff_unlink_free_block;
     allctr->info_options		= info_options;
 
     allctr->get_next_mbc_size		= NULL;
@@ -359,9 +379,7 @@ replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y)
     y->parent	= x->parent;
     y->right	= x->right;
     y->left	= x->left;
-
-    y->max_sz = x->max_sz;
-    lower_max_size(y, NULL);
+    y->max_sz   = x->max_sz;
 }
 
 static void
@@ -458,23 +476,47 @@ tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk)
 }
 
 static void
-aoff_unlink_free_block(Allctr_t *allctr, Block_t *del)
+aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk)
 {
-#ifdef HARD_DEBUG
     AOFFAllctr_t* alc = (AOFFAllctr_t*)allctr;
-#endif
-    AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(del);
+    AOFF_RBTree_t* del = (AOFF_RBTree_t*)blk;
+    AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr);
 
     ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz);
-    HARD_CHECK_IS_MEMBER(alc->mbc_root, &crr->rbt_node);
-    HARD_CHECK_TREE(&crr->crr, alc->bf_within_carrier, crr->root, 0);
+    HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0);
+
+    if (alc->flavor == AOFF_BF) {
+	ASSERT(del->flags & IS_BF_FLG);
+	if (IS_LIST_ELEM(del)) {
+	    /* Remove from list */
+	    ASSERT(LIST_PREV(del));
+	    ASSERT(LIST_PREV(del)->flags & IS_BF_FLG);
+	    LIST_NEXT(LIST_PREV(del)) = LIST_NEXT(del);
+	    if (LIST_NEXT(del)) {
+		ASSERT(LIST_NEXT(del)->flags & IS_BF_FLG);
+		LIST_PREV(LIST_NEXT(del)) = LIST_PREV(del);
+	    }
+	    return;
+	}
+	else if (LIST_NEXT(del)) {
+	    /* Replace tree node by next element in list... */
+	    
+	    ASSERT(AOFF_BLK_SZ(LIST_NEXT(del)) == AOFF_BLK_SZ(del));
+	    ASSERT(IS_LIST_ELEM(LIST_NEXT(del)));
+	    
+	    replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del));
+	    
+	    HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0);
+	    return;
+	}
+    }
 
     rbt_delete(&crr->root, (AOFF_RBTree_t*)del);
 
-    HARD_CHECK_TREE(&crr->crr, alc->bf_within_carrier, crr->root, 0);
+    HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0);
 
-   /* Update the carrier tree with a potentially new (lower) max_sz
-    */    
+    /* Update the carrier tree with a potentially new (lower) max_sz
+     */    
     if (crr->root) {
 	if (crr->rbt_node.hdr.bhdr == crr->root->max_sz) {
 	    return;
@@ -488,6 +530,7 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del)
     lower_max_size(&crr->rbt_node, NULL);
 }
 
+
 static void
 rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del)
 {
@@ -542,7 +585,9 @@ rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del)
     }
     if (y != z) {
 	/* We spliced out the successor of z; replace z by the successor */
+	ASSERT(z != &null_x);
 	replace(root, z, y);
+	lower_max_size(y, NULL);
     }
 
     if (spliced_is_black) {
@@ -666,12 +711,11 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block)
 
     ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr));
     ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0));
-    HARD_CHECK_IS_MEMBER(alc->mbc_root, &blk_crr->rbt_node);
-    HARD_CHECK_TREE(&blk_crr->crr, alc->bf_within_carrier, blk_crr->root, 0);
+    HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0);
 
-    rbt_insert(alc->bf_within_carrier, &blk_crr->root, blk);
+    rbt_insert(alc->flavor, &blk_crr->root, blk);
 
-    /* Update the carrier tree with a potential new (larger) max_sz
+    /* Update the carrier tree with a potentially new (larger) max_sz
     */
     crr_node = &blk_crr->rbt_node;
     if (blk_sz > crr_node->hdr.bhdr) {
@@ -683,15 +727,19 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block)
 	    if (!crr_node) break;
 	}
     }
-    HARD_CHECK_TREE(&blk_crr->crr, alc->bf_within_carrier, blk_crr->root, 0);
+    HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0);
 }
 
 static void
-rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
+rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 {
     Uint blk_sz = AOFF_BLK_SZ(blk);
 
-    blk->flags	= 0;
+#ifdef DEBUG
+    blk->flags  = (flavor == AOFF_BF) ? IS_BF_FLG : 0; 
+#else
+    blk->flags  = 0; 
+#endif
     blk->left	= NULL;
     blk->right	= NULL;
     blk->max_sz = blk_sz;
@@ -704,10 +752,12 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
     else {
 	AOFF_RBTree_t *x = *root;
 	while (1) {
+	    SWord diff; 
 	    if (x->max_sz < blk_sz) {
 		x->max_sz = blk_sz;
 	    }
-	    if (cmp_blocks(bestfit, blk, x) < 0) {
+	    diff = cmp_blocks(flavor, blk, x);
+	    if (diff < 0) {
 		if (!x->left) {
 		    blk->parent = x;
 		    x->left = blk;
@@ -715,7 +765,7 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 		}
 		x = x->left;
 	    }
-	    else {
+	    else if (diff > 0) {
 		if (!x->right) {
 		    blk->parent = x;
 		    x->right = blk;
@@ -723,6 +773,18 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 		}
 		x = x->right;
 	    }
+	    else {
+		ASSERT(flavor == AOFF_BF);
+		ASSERT(blk->flags & IS_BF_FLG);			    
+		ASSERT(x->flags & IS_BF_FLG);			    
+		SET_LIST_ELEM(blk);
+		LIST_NEXT(blk) = LIST_NEXT(x);
+		LIST_PREV(blk) = x;
+		if (LIST_NEXT(x))
+		    LIST_PREV(LIST_NEXT(x)) = blk;
+		LIST_NEXT(x) = blk;
+		return;
+	    }
 	}
 
 	/* Insert block into size tree */
@@ -732,6 +794,10 @@ rbt_insert(int bestfit, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 	if (IS_RED(blk->parent))
 	    tree_insert_fixup(root, blk);
     }
+    if (flavor == AOFF_BF) {
+	SET_TREE_NODE(blk);
+	LIST_NEXT(blk) = NULL;
+    }
 }
 
 static AOFF_RBTree_t*
@@ -780,7 +846,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size,
     /* Get block within carrier tree
      */
 #ifdef HARD_DEBUG
-    dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->bf_within_carrier, crr->root, size);
+    dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size);
 #endif
 
     blk = rbt_search(crr->root, size);
@@ -793,7 +859,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size,
     if (!blk)
 	return NULL;
 
-    if (cand_blk && cmp_cand_blk(alc->bf_within_carrier, cand_blk, blk) < 0) {
+    if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) {
 	return NULL; /* cand_blk was better */
     }
 
@@ -813,7 +879,7 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier)
     /* Link carrier in address order tree
      */
     crr->rbt_node.hdr.bhdr = 0;
-    rbt_insert(0, root, &crr->rbt_node);
+    rbt_insert(AOFF_AOFF, root, &crr->rbt_node);
 
     /* aoff_link_free_block will add free block later */
     crr->root = NULL;
@@ -843,7 +909,7 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier)
 
     /* Link carrier in address order tree
      */
-    rbt_insert(0, root, &crr->rbt_node);
+    rbt_insert(AOFF_AOFF, root, &crr->rbt_node);
 
     HARD_CHECK_TREE(NULL, 0, *root, 0);
 }
@@ -883,6 +949,7 @@ static struct {
     Eterm as;
     Eterm aoff;
     Eterm aoffcaobf;
+    Eterm aoffcbf;
 #ifdef DEBUG
     Eterm end_of_atoms;
 #endif
@@ -912,6 +979,7 @@ init_atoms(void)
     AM_INIT(as);
     AM_INIT(aoff);
     AM_INIT(aoffcaobf);
+    AM_INIT(aoffcbf);
 
 #ifdef DEBUG
     for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
@@ -943,13 +1011,15 @@ info_options(Allctr_t *allctr,
 {
     AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr;
     Eterm res = THE_NON_VALUE;
+    const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"};
+    Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf};
 
     if (print_to_p) {
 	erts_print(*print_to_p,
 		   print_to_arg,
 		   "%sas: %s\n",
 		   prefix,
-		   alc->bf_within_carrier ? "aoffcaobf" : "aoff");
+		   flavor_str[alc->flavor]);
     }
 
     if (hpp || szp) {
@@ -959,8 +1029,7 @@ info_options(Allctr_t *allctr,
 		     __FILE__, __LINE__);;
 
 	res = NIL;
-	add_2tup(hpp, szp, &res, am.as,
-		 alc->bf_within_carrier ? am.aoffcaobf : am.aoff);
+	add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]);
     }
 
     return res;
@@ -978,6 +1047,7 @@ UWord
 erts_aoffalc_test(UWord op, UWord a1, UWord a2)
 {
     switch (op) {
+    case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF;
     case 0x501: {
 	AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; 
 	Uint size = (Uint) a2;
@@ -987,10 +1057,13 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2)
     case 0x502:	return (UWord) ((AOFF_RBTree_t *) a1)->parent;
     case 0x503:	return (UWord) ((AOFF_RBTree_t *) a1)->left;
     case 0x504:	return (UWord) ((AOFF_RBTree_t *) a1)->right;
+    case 0x505:	return (UWord) LIST_NEXT(a1);
     case 0x506:	return (UWord) IS_BLACK((AOFF_RBTree_t *) a1);
+    case 0x507:	return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1);
     case 0x508: return (UWord) 0; /* IS_BF_ALGO */
     case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz;
-    case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->bf_within_carrier;
+    case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF;
+    case 0x50b:	return (UWord) LIST_PREV(a1);
     default:	ASSERT(0); return ~((UWord) 0);
     }
 }
@@ -1049,7 +1122,7 @@ static void print_tree(AOFF_RBTree_t*);
  */
 
 static AOFF_RBTree_t *
-check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size)
+check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size)
 {
     AOFF_RBTree_t *res = NULL;
     Sint blacks;
@@ -1061,6 +1134,7 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size)
 #ifdef PRINT_TREE
     print_tree(root);
 #endif
+    ASSERT(within_crr || flavor == AOFF_AOFF);
 
     if (!root)
 	return res;
@@ -1116,6 +1190,20 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size)
 	    ASSERT(crr == within_crr);
 	    ASSERT((char*)x > (char*)crr);
 	    ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr)));
+
+	}
+	if (flavor == AOFF_BF) {
+	    AOFF_RBTree_t* y = x;
+	    AOFF_RBTree_t* nxt = LIST_NEXT(y);
+	    ASSERT(IS_TREE_NODE(x));
+	    while (nxt) {
+		ASSERT(IS_LIST_ELEM(nxt));
+		ASSERT(AOFF_BLK_SZ(nxt) == AOFF_BLK_SZ(x));
+		ASSERT(FBLK_TO_MBC(&nxt->hdr) == within_crr);
+		ASSERT(LIST_PREV(nxt) == y);
+		y = nxt;
+		nxt = LIST_NEXT(nxt);
+	    }
 	}
 
 	if (IS_RED(x)) {
@@ -1127,13 +1215,13 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size)
 
 	if (x->left) {
 	    ASSERT(x->left->parent == x);
-	    ASSERT(cmp_blocks(bestfit, x->left, x) < 0);
+	    ASSERT(cmp_blocks(flavor, x->left, x) < 0);
 	    ASSERT(x->left->max_sz <= x->max_sz);	    
 	}
 
 	if (x->right) {
 	    ASSERT(x->right->parent == x);
-	    ASSERT(cmp_blocks(bestfit, x->right, x) > 0);
+	    ASSERT(cmp_blocks(flavor, x->right, x) > 0);
 	    ASSERT(x->right->max_sz <= x->max_sz);	    
 	}
 	ASSERT(x->max_sz >= AOFF_BLK_SZ(x));
@@ -1142,7 +1230,7 @@ check_tree(Carrier_t* within_crr, int bestfit, AOFF_RBTree_t* root, Uint size)
 	       || x->max_sz == (x->right ? x->right->max_sz : 0));
 
 	if (size && AOFF_BLK_SZ(x) >= size) {
-	    if (!res || cmp_blocks(bestfit, x, res) < 0) {
+	    if (!res || cmp_blocks(flavor, x, res) < 0) {
 		res = x;
 	    }
 	}
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index 87427e8e62..6fa2c0e5bf 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -24,12 +24,17 @@
 #include "erl_alloc_util.h"
 
 #define ERTS_ALC_AOFF_ALLOC_VSN_STR "0.9"
-#define ERTS_ALC_AOFF_CBF_ALLOC_VSN_STR "0.9"
 
 typedef struct AOFFAllctr_t_ AOFFAllctr_t;
 
+enum AOFF_Flavor {
+    AOFF_AOFF = 0,
+    AOFF_AOBF = 1,
+    AOFF_BF   = 2
+};
+
 typedef struct {
-    int bf_within_carrier;
+    enum AOFF_Flavor flavor;
 } AOFFAllctrInit_t;
 
 #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/}
@@ -52,7 +57,7 @@ struct AOFFAllctr_t_ {
     Allctr_t		allctr; /* Has to be first! */
 
     struct AOFF_RBTree_t_* mbc_root;
-    int bf_within_carrier;
+    enum AOFF_Flavor flavor;
 };
 
 UWord erts_aoffalc_test(UWord, UWord, UWord);
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 58e53c3d00..41f449bb28 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -966,15 +966,17 @@ UWord
 erts_bfalc_test(UWord op, UWord a1, UWord a2)
 {
     switch (op) {
-    case 0x200:	return (UWord) ((BFAllctr_t *) a1)->address_order;
+    case 0x200:	return (UWord) ((BFAllctr_t *) a1)->address_order; /* IS_AOBF */
     case 0x201:	return (UWord) ((BFAllctr_t *) a1)->mbc_root;
     case 0x202:	return (UWord) ((RBTree_t *) a1)->parent;
     case 0x203:	return (UWord) ((RBTree_t *) a1)->left;
     case 0x204:	return (UWord) ((RBTree_t *) a1)->right;
-    case 0x205:	return (UWord) ((RBTreeList_t *) a1)->next;
+    case 0x205:	return (UWord) LIST_NEXT(a1);
     case 0x206:	return (UWord) IS_BLACK((RBTree_t *) a1);
     case 0x207:	return (UWord) IS_TREE_NODE((RBTree_t *) a1);
     case 0x208:	return (UWord) 1; /* IS_BF_ALGO */
+    case 0x20a: return (UWord) !((BFAllctr_t *) a1)->address_order; /* IS_BF */
+    case 0x20b:	return (UWord) LIST_PREV(a1);
     default:	ASSERT(0); return ~((UWord) 0);
     }
 }
-- 
cgit v1.2.3


From 8212fdf19997e08a52ff9374283ff5e148561a0c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 19 Jun 2013 18:37:36 +0200
Subject: erts: Make aoffcbf default when migration is enabled

---
 erts/emulator/beam/erl_alloc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 7b77bcca58..306a4f24de 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -245,7 +245,7 @@ set_default_acul(struct au_init *ip, int acul)
 {
     ip->thr_spec = 1;
     ip->atype = AOFIRSTFIT;
-    ip->init.aoff.flavor = AOFF_AOBF;
+    ip->init.aoff.flavor = AOFF_BF;
     ip->init.util.acul = acul;
 }
 
@@ -581,9 +581,9 @@ ensure_carrier_migration_support(struct au_init *auip)
      * default to a strategy that can...
      */
     if (!strategy_support_carrier_migration(auip)) {
-	/* Default to aoffcaobf */
+	/* Default to aoffcbf */
 	auip->atype = AOFIRSTFIT;
-	auip->init.aoff.flavor = AOFF_AOBF;
+	auip->init.aoff.flavor = AOFF_BF;
     }
 }
 
-- 
cgit v1.2.3


From 604d320f23c42c349f3f44710bcbc05fde7db661 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 4 Jul 2013 17:44:23 +0200
Subject: Make emulator arguments available via the system_info BIF

erlang:system_info(emu_args)
---
 erts/emulator/beam/erl_bif_info.c |  3 ++
 erts/emulator/beam/erl_init.c     |  2 +
 erts/emulator/beam/erl_utils.h    |  4 ++
 erts/emulator/beam/utils.c        | 91 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 100 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 74a37f374d..447c1fdf18 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2646,6 +2646,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 #endif
 	BIF_RET(am_true);
     }
+    else if (ERTS_IS_ATOM_STR("emu_args", BIF_ARG_1)) {
+	BIF_RET(erts_get_emu_args(BIF_P));
+    }
     else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) {
 #if defined(USE_DTRACE)
 	DECL_AM(dtrace);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 4bae3dfeb4..8d137df7ae 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -637,6 +637,8 @@ early_init(int *argc, char **argv) /*
     char envbuf[21]; /* enough for any 64-bit integer */
     size_t envbufsz;
 
+    erts_save_emu_args(*argc, argv);
+
     erts_sched_compact_load = 1;
     erts_printf_eterm_func = erts_printf_term;
     erts_disable_tolerant_timeofday = 0;
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index a2064bd8a3..79c5105d92 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -24,6 +24,8 @@
 #include "erl_smp.h"
 #include "erl_printf.h"
 
+struct process;
+
 typedef struct {
 #ifdef DEBUG
     int smp_api;
@@ -155,6 +157,8 @@ Uint32 block_hash(byte *, unsigned, Uint32);
 Uint32 make_hash2(Eterm);
 Uint32 make_hash(Eterm);
 
+void erts_save_emu_args(int argc, char **argv);
+Eterm erts_get_emu_args(struct process *c_p);
 
 Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str);
 Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 62caa67ce1..eb1c8f6173 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3466,6 +3466,97 @@ erts_free_read_env(void *value)
 	erts_free(ERTS_ALC_T_TMP, value);
 }
 
+
+typedef struct {
+    size_t sz;
+    char *ptr;
+} ErtsEmuArg;
+
+typedef struct {
+    int argc;
+    ErtsEmuArg *arg;
+    size_t no_bytes;
+} ErtsEmuArgs;
+
+ErtsEmuArgs saved_emu_args = {0};
+
+void
+erts_save_emu_args(int argc, char **argv)
+{
+#ifdef DEBUG
+    char *end_ptr;
+#endif
+    char *ptr;
+    int i;
+    size_t arg_sz[100];
+    size_t size;
+
+    ASSERT(!saved_emu_args.argc);
+
+    size = sizeof(ErtsEmuArg)*argc;
+    for (i = 0; i < argc; i++) {
+	size_t sz = sys_strlen(argv[i]);
+	if (i < sizeof(arg_sz)/sizeof(arg_sz[0]))
+	    arg_sz[i] = sz;
+	size += sz+1;
+    } 
+    ptr = (char *) malloc(size);
+#ifdef DEBUG
+    end_ptr = ptr + size;
+#endif
+    saved_emu_args.arg = (ErtsEmuArg *) ptr;
+    ptr += sizeof(ErtsEmuArg)*argc;
+    saved_emu_args.argc = argc;
+    saved_emu_args.no_bytes = 0;
+    for (i = 0; i < argc; i++) {
+	size_t sz;
+	if (i < sizeof(arg_sz)/sizeof(arg_sz[0]))
+	    sz = arg_sz[i];
+	else
+	    sz = sys_strlen(argv[i]);
+	saved_emu_args.arg[i].ptr = ptr;
+	saved_emu_args.arg[i].sz = sz;
+	saved_emu_args.no_bytes += sz;
+	ptr += sz+1;
+	sys_strcpy(saved_emu_args.arg[i].ptr, argv[i]);
+    }
+    ASSERT(ptr == end_ptr);
+}
+
+Eterm
+erts_get_emu_args(Process *c_p)
+{
+#ifdef DEBUG
+    Eterm *end_hp;
+#endif
+    int i;
+    Uint hsz;
+    Eterm *hp, res;
+
+    hsz = saved_emu_args.no_bytes*2;
+    hsz += saved_emu_args.argc*2;
+
+    hp = HAlloc(c_p, hsz);
+#ifdef DEBUG
+    end_hp = hp + hsz;
+#endif
+    res = NIL;
+
+    for (i = saved_emu_args.argc-1; i >= 0; i--) {
+	Eterm arg = buf_to_intlist(&hp,
+				   saved_emu_args.arg[i].ptr,
+				   saved_emu_args.arg[i].sz,
+				   NIL);
+	res = CONS(hp, arg, res);
+	hp += 2;
+    }
+
+    ASSERT(hp == end_hp);
+
+    return res;
+}
+
+
 /*
  * To be used to silence unused result warnings, but do not abuse it.
  */
-- 
cgit v1.2.3


From dbd97f2569af24ef395fa3d60bb2b00543296ce8 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 5 Jul 2013 16:40:51 +0200
Subject: Make ethread library information available via system_info BIF

erlang:system_info(ethread_info)
---
 erts/emulator/beam/erl_bif_info.c |  70 +-------------
 erts/emulator/beam/erl_utils.h    |   1 +
 erts/emulator/beam/utils.c        | 196 +++++++++++++++++++++++++++++++++++++-
 3 files changed, 198 insertions(+), 69 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 447c1fdf18..b1dc62b4fd 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2577,74 +2577,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
 	res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit);
 	BIF_RET(res);
-    } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) {
-#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \
-    || defined(ETHR_NATIVE_ATOMIC64_IMPL) \
-    || defined(ETHR_NATIVE_DW_ATOMIC_IMPL)
-	int i;
-	char **str;
-#endif
-#ifdef ETHR_NATIVE_ATOMIC32_IMPL
-	erts_printf("32-bit native atomics: %s\n",
-		    ETHR_NATIVE_ATOMIC32_IMPL);
-	str = ethr_native_atomic32_ops();
-	for (i = 0; str[i]; i++)
-	    erts_printf("ethr_native_atomic32_%s()\n", str[i]);
-#endif
-#ifdef ETHR_NATIVE_ATOMIC64_IMPL
-	erts_printf("64-bit native atomics: %s\n",
-		    ETHR_NATIVE_ATOMIC64_IMPL);
-	str = ethr_native_atomic64_ops();
-	for (i = 0; str[i]; i++)
-	    erts_printf("ethr_native_atomic64_%s()\n", str[i]);
-#endif
-#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL
-	if (ethr_have_native_dw_atomic()) {
-	    erts_printf("Double word native atomics: %s\n",
-			ETHR_NATIVE_DW_ATOMIC_IMPL);
-	    str = ethr_native_dw_atomic_ops();
-	    for (i = 0; str[i]; i++)
-		erts_printf("ethr_native_dw_atomic_%s()\n", str[i]);
-	    str = ethr_native_su_dw_atomic_ops();
-	    for (i = 0; str[i]; i++)
-		erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]);
-	}
-#endif
-#ifdef ETHR_NATIVE_SPINLOCK_IMPL
-	erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL);
-#endif
-#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL
-	erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL);
-#endif
-#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
-	erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
-					   ? "yes" : "no"));
-#endif
-#ifdef ETHR_X86_OUT_OF_ORDER
-	erts_printf("x86"
-#ifdef ARCH_64
-		    "_64"
-#endif
-		    " out of order\n");
-#endif
-#ifdef ETHR_SPARC_TSO
-	erts_printf("Sparc TSO\n");
-#endif
-#ifdef ETHR_SPARC_PSO
-	erts_printf("Sparc PSO\n");
-#endif
-#ifdef ETHR_SPARC_RMO
-	erts_printf("Sparc RMO\n");
-#endif
-#if defined(ETHR_PPC_HAVE_LWSYNC)
-	erts_printf("Have lwsync instruction: yes\n");
-#elif defined(ETHR_PPC_HAVE_NO_LWSYNC)
-	erts_printf("Have lwsync instruction: no\n");
-#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__)
-	erts_printf("Have lwsync instruction: %s (runtime test)\n",
-		    ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no");
-#endif
-	BIF_RET(am_true);
+    } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) {
+	BIF_RET(erts_get_ethread_info(BIF_P));
     }
     else if (ERTS_IS_ATOM_STR("emu_args", BIF_ARG_1)) {
 	BIF_RET(erts_get_emu_args(BIF_P));
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 79c5105d92..80d29d554a 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -159,6 +159,7 @@ Uint32 make_hash(Eterm);
 
 void erts_save_emu_args(int argc, char **argv);
 Eterm erts_get_emu_args(struct process *c_p);
+Eterm erts_get_ethread_info(struct process * c_p);
 
 Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str);
 Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index eb1c8f6173..a285c1d62b 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3543,7 +3543,7 @@ erts_get_emu_args(Process *c_p)
     res = NIL;
 
     for (i = saved_emu_args.argc-1; i >= 0; i--) {
-	Eterm arg = buf_to_intlist(&hp,
+    Eterm arg = buf_to_intlist(&hp,
 				   saved_emu_args.arg[i].ptr,
 				   saved_emu_args.arg[i].sz,
 				   NIL);
@@ -3557,6 +3557,200 @@ erts_get_emu_args(Process *c_p)
 }
 
 
+Eterm
+erts_get_ethread_info(Process *c_p)
+{
+    Uint sz, *szp;
+    Eterm res, *hp, **hpp, *end_hp = NULL;
+
+    sz = 0;
+    szp = &sz;
+    hpp = NULL;
+
+    while (1) {
+	Eterm tup, list, name;
+#if defined(ETHR_NATIVE_ATOMIC32_IMPL)	  \
+    || defined(ETHR_NATIVE_ATOMIC64_IMPL)	\
+    || defined(ETHR_NATIVE_DW_ATOMIC_IMPL)
+	char buf[1024];
+	int i;
+	char **str;
+#endif
+
+	res = NIL;
+
+#ifdef ETHR_X86_MEMBAR_H__
+
+	tup = erts_bld_tuple(hpp, szp, 2,
+			     erts_bld_string(hpp, szp, "sse2"),
+#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+			     erts_bld_string(hpp, szp,
+					     (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+					      ? "yes" : "no"))
+#else
+			     erts_bld_string(hpp, szp, "yes")
+#endif
+	    );
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+	tup = erts_bld_tuple(hpp, szp, 2,
+			     erts_bld_string(hpp, szp,
+					     "x86"
+#ifdef ARCH_64
+					     "_64"
+#endif
+					     " OOO"),
+			     erts_bld_string(hpp, szp,
+#ifdef ETHR_X86_OUT_OF_ORDER
+					     "yes"
+#else
+					     "no"
+#endif
+				 ));
+
+	res = erts_bld_cons(hpp, szp, tup, res);
+#endif
+
+#ifdef ETHR_SPARC_V9_MEMBAR_H__
+
+	tup = erts_bld_tuple(hpp, szp, 2,
+			     erts_bld_string(hpp, szp, "Sparc V9"),
+			     erts_bld_string(hpp, szp,
+#if defined(ETHR_SPARC_TSO)
+					     "TSO"
+#elif defined(ETHR_SPARC_PSO)
+					     "PSO"
+#elif defined(ETHR_SPARC_RMO)
+					     "RMO"
+#else
+					     "undefined"
+#endif
+				 ));
+
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+#endif
+
+#ifdef ETHR_PPC_MEMBAR_H__
+
+	tup = erts_bld_tuple(hpp, szp, 2,
+			     erts_bld_string(hpp, szp, "lwsync"),
+			     erts_bld_string(hpp, szp,
+#if defined(ETHR_PPC_HAVE_LWSYNC)
+					     "yes"
+#elif defined(ETHR_PPC_HAVE_NO_LWSYNC)
+					     "no"
+#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__)
+					     ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"
+#else
+					     "undefined"
+#endif
+				 ));
+
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+#endif
+
+	tup = erts_bld_tuple(hpp, szp, 2,
+			     erts_bld_string(hpp, szp, "Native rw-spinlocks"),
+#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL
+			     erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL)
+#else
+			     erts_bld_string(hpp, szp, "no")
+#endif
+	    );
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+	tup = erts_bld_tuple(hpp, szp, 2,
+			     erts_bld_string(hpp, szp, "Native spinlocks"),
+#ifdef ETHR_NATIVE_SPINLOCK_IMPL
+			     erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL)
+#else
+			     erts_bld_string(hpp, szp, "no")
+#endif
+	    );
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+
+	list = NIL;
+#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL
+	if (ethr_have_native_dw_atomic()) {
+	    name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL);
+	    str = ethr_native_dw_atomic_ops();
+	    for (i = 0; str[i]; i++) {
+		erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]);
+		list = erts_bld_cons(hpp, szp,
+				     erts_bld_string(hpp, szp, buf),
+				     list);
+	    }
+	    str = ethr_native_su_dw_atomic_ops();
+	    for (i = 0; str[i]; i++) {
+		erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]);
+		list = erts_bld_cons(hpp, szp,
+				     erts_bld_string(hpp, szp, buf),
+				     list);
+	    }
+	}
+	else 
+#endif
+	    name = erts_bld_string(hpp, szp, "no");
+
+	tup = erts_bld_tuple(hpp, szp, 3,
+			     erts_bld_string(hpp, szp, "Double word native atomics"),
+			     name,
+			     list);
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+	list = NIL;
+#ifdef ETHR_NATIVE_ATOMIC64_IMPL
+	name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL);
+	str = ethr_native_atomic64_ops();
+	for (i = 0; str[i]; i++) {
+	    erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]);
+	    list = erts_bld_cons(hpp, szp,
+				 erts_bld_string(hpp, szp, buf),
+				 list);
+	}
+#else
+	name = erts_bld_string(hpp, szp, "no");
+#endif
+	tup = erts_bld_tuple(hpp, szp, 3,
+			     erts_bld_string(hpp, szp, "64-bit native atomics"),
+			     name,
+			     list);
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+	list = NIL;
+#ifdef ETHR_NATIVE_ATOMIC32_IMPL
+	name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL);
+	str = ethr_native_atomic32_ops();
+	for (i = 0; str[i]; i++) {
+	    erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]);
+	    list = erts_bld_cons(hpp, szp,
+				erts_bld_string(hpp, szp, buf),
+				list);
+	}
+#else
+	name = erts_bld_string(hpp, szp, "no");
+#endif
+	tup = erts_bld_tuple(hpp, szp, 3,
+			     erts_bld_string(hpp, szp, "32-bit native atomics"),
+			     name,
+			     list);
+	res = erts_bld_cons(hpp, szp, tup, res);
+
+	if (hpp) {
+	    HRelease(c_p, end_hp, *hpp)
+	    return res;
+	}
+
+	hp = HAlloc(c_p, sz);
+	end_hp = hp + sz;
+	hpp = &hp;
+	szp = NULL;
+    }
+}
+
 /*
  * To be used to silence unused result warnings, but do not abuse it.
  */
-- 
cgit v1.2.3


From b3b0c44ca3f1b87c719f782f09dd0331dbb1a351 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 8 Jul 2013 18:04:14 +0200
Subject: Make information about use of jump table available via system_info
 BIF

erlang:system_info(beam_jump_table)
---
 erts/emulator/beam/beam_emu.c     | 9 +++++++++
 erts/emulator/beam/erl_bif_info.c | 3 +++
 erts/emulator/beam/global.h       | 2 ++
 3 files changed, 14 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 5781009f58..da36c4437e 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6256,3 +6256,12 @@ erts_current_reductions(Process *current, Process *p)
     }
 }
 
+int
+erts_beam_jump_table(void)
+{
+#if defined(NO_JUMP_TABLE)
+    return 0;
+#else
+    return 1;
+#endif
+}
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b1dc62b4fd..e4fbd21c0e 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2583,6 +2583,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("emu_args", BIF_ARG_1)) {
 	BIF_RET(erts_get_emu_args(BIF_P));
     }
+    else if (ERTS_IS_ATOM_STR("beam_jump_table", BIF_ARG_1)) {
+	BIF_RET(erts_beam_jump_table() ? am_true : am_false);
+    }
     else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) {
 #if defined(USE_DTRACE)
 	DECL_AM(dtrace);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 25aedc91c6..e5807b5ce6 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1042,6 +1042,8 @@ extern erts_driver_t vanilla_driver;
 extern erts_driver_t spawn_driver;
 extern erts_driver_t fd_driver;
 
+int erts_beam_jump_table(void);
+
 /* Should maybe be placed in erl_message.h, but then we get an include mess. */
 ERTS_GLB_INLINE Eterm *
 erts_alloc_message_heap_state(Uint size,
-- 
cgit v1.2.3


From 6b16d650f99044eb2e1975f2d954aa07ed8aae35 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 11 Jul 2013 16:03:07 +0200
Subject: Fix 'no previous prototype' warning for dtrace functions

---
 erts/emulator/beam/global.h | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 25aedc91c6..c0d66d6cc1 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1217,6 +1217,13 @@ erts_alloc_message_heap(Uint size,
 #  define UnUseTmpHeapNoproc(Size) /* Nothing */
 #endif /* HEAP_ON_C_STACK */
 
+ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf);
+ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf);
+ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf);
+ERTS_GLB_INLINE void dtrace_fun_decode(Process *process,
+				       Eterm module, Eterm function, int arity,
+				       char *process_buf, char *mfa_buf);
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 #include "dtrace-wrapper.h"
-- 
cgit v1.2.3


From 54c658c89dc8edae1e420861601eec068bc74676 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 11 Jul 2013 16:33:52 +0200
Subject: =?UTF-8?q?Fix=20variable=20=E2=80=98rp=5Fhad=5Flocks=E2=80=99=20s?=
 =?UTF-8?q?et=20but=20not=20used=20warning?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 erts/emulator/beam/erl_nif.c | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d4c2b5bdcc..48f8be8dd3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -310,9 +310,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     Process* rp;
     Process* c_p;
     ErlHeapFragment* frags;
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-    ErtsProcLocks rp_had_locks;
-#endif
     Eterm receiver = to_pid->pid;
     int flush_me = 0;
     int scheduler = erts_get_scheduler_id() != 0;
@@ -332,10 +329,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 #endif
     }    
 
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-    rp_had_locks = rp_locks;
-#endif
-
     rp = (scheduler
 	  ? erts_proc_lookup(receiver)
 	  : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
-- 
cgit v1.2.3


From 14c0a18a9970f2287bbb09c46bee0c33358e3fc9 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 4 Jul 2013 15:23:06 +0200
Subject: erts: Add cflags, ldflags and config.h into executable

---
 erts/emulator/beam/atom.names     |  3 +++
 erts/emulator/beam/erl_bif_info.c | 21 +++++++++++++++++++++
 erts/emulator/beam/global.h       |  2 +-
 erts/emulator/beam/utils.c        |  2 +-
 4 files changed, 26 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index eba1d0fa23..c2f32ba089 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -139,6 +139,7 @@ atom caseless
 atom catchlevel
 atom cd
 atom cdr
+atom cflags
 atom characters_to_binary_int
 atom characters_to_list_int
 atom clear
@@ -150,6 +151,7 @@ atom compact
 atom compat_rel
 atom compile
 atom compressed
+atom config_h
 atom connect
 atom connected
 atom connection_closed
@@ -301,6 +303,7 @@ atom label
 atom large_heap
 atom last_calls
 atom latin1
+atom ldflags
 atom Le='=<'
 atom lf
 atom line
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index e4fbd21c0e..a43eee6420 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -30,6 +30,7 @@
 #include "bif.h"
 #include "big.h"
 #include "erl_version.h"
+#include "erl_compile_flags.h"
 #include "erl_db_util.h"
 #include "erl_message.h"
 #include "erl_binary.h"
@@ -2610,6 +2611,26 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(am_true);
     }
 #endif
+    else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) {
+      Eterm *hp = HAlloc(BIF_P, 3*3+3*2); /* three two tuples +
+					     three list elems */
+      Eterm res = NIL;
+      Eterm *lhp = HAlloc(BIF_P, 2*(strlen(erts_build_flags_CONFIG_H)+
+				    strlen(erts_build_flags_CFLAGS)+
+				    strlen(erts_build_flags_LDFLAGS)));
+
+      res = CONS(hp+9,TUPLE2(hp, am_config_h,
+			     buf_to_intlist(&lhp, erts_build_flags_CONFIG_H,
+					    strlen(erts_build_flags_CONFIG_H), NIL)),res);
+      res = CONS(hp+11,TUPLE2(hp+3, am_cflags,
+			      buf_to_intlist(&lhp, erts_build_flags_CFLAGS,
+					     strlen(erts_build_flags_CFLAGS), NIL)),res);
+      res = CONS(hp+13,TUPLE2(hp+6, am_ldflags,
+			      buf_to_intlist(&lhp, erts_build_flags_LDFLAGS,
+					     strlen(erts_build_flags_LDFLAGS), NIL)),res);
+
+      BIF_RET(res);
+    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index e5807b5ce6..56f8f2b3ea 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -937,7 +937,7 @@ char* Sint_to_buf(Sint, struct Sint_buf*);
 #define ERTS_IOLIST_OVERFLOW 1
 #define ERTS_IOLIST_TYPE 2
 
-Eterm buf_to_intlist(Eterm**, char*, size_t, Eterm); /* most callers pass plain char*'s */
+Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */
 
 #define ERTS_IOLIST_TO_BUF_OVERFLOW	(~((ErlDrvSizeT) 0))
 #define ERTS_IOLIST_TO_BUF_TYPE_ERROR	(~((ErlDrvSizeT) 1))
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 84deaecb87..bd2be7afca 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2983,7 +2983,7 @@ char* Sint_to_buf(Sint n, struct Sint_buf *buf)
 */
 
 Eterm
-buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail)
+buf_to_intlist(Eterm** hpp, const char *buf, size_t len, Eterm tail)
 {
     Eterm* hp = *hpp;
     size_t i = len;
-- 
cgit v1.2.3


From 5e768c8765bc70d70599dd9a6e5d27875de490e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 11 Jul 2013 18:39:04 +0200
Subject: Fix erlang:system_info(compile_info)

Allocation needs to be in correct order.
---
 erts/emulator/beam/erl_bif_info.c | 41 ++++++++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a43eee6420..673dfc658c 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2612,24 +2612,29 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     }
 #endif
     else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) {
-      Eterm *hp = HAlloc(BIF_P, 3*3+3*2); /* three two tuples +
-					     three list elems */
-      Eterm res = NIL;
-      Eterm *lhp = HAlloc(BIF_P, 2*(strlen(erts_build_flags_CONFIG_H)+
-				    strlen(erts_build_flags_CFLAGS)+
-				    strlen(erts_build_flags_LDFLAGS)));
-
-      res = CONS(hp+9,TUPLE2(hp, am_config_h,
-			     buf_to_intlist(&lhp, erts_build_flags_CONFIG_H,
-					    strlen(erts_build_flags_CONFIG_H), NIL)),res);
-      res = CONS(hp+11,TUPLE2(hp+3, am_cflags,
-			      buf_to_intlist(&lhp, erts_build_flags_CFLAGS,
-					     strlen(erts_build_flags_CFLAGS), NIL)),res);
-      res = CONS(hp+13,TUPLE2(hp+6, am_ldflags,
-			      buf_to_intlist(&lhp, erts_build_flags_LDFLAGS,
-					     strlen(erts_build_flags_LDFLAGS), NIL)),res);
-
-      BIF_RET(res);
+	Uint  sz;
+	Eterm res = NIL, tup, text;
+	Eterm *hp = HAlloc(BIF_P, 3*(2 + 3) + /* three 2-tuples and three cons */
+		2*(strlen(erts_build_flags_CONFIG_H) +
+		   strlen(erts_build_flags_CFLAGS) +
+		   strlen(erts_build_flags_LDFLAGS)));
+
+	sz   = strlen(erts_build_flags_CONFIG_H);
+	text = buf_to_intlist(&hp, erts_build_flags_CONFIG_H, sz, NIL);
+	tup  = TUPLE2(hp, am_config_h, text); hp += 3;
+	res  = CONS(hp, tup, res); hp += 2;
+
+	sz   = strlen(erts_build_flags_CFLAGS);
+	text = buf_to_intlist(&hp, erts_build_flags_CFLAGS, sz, NIL);
+	tup  = TUPLE2(hp, am_cflags, text); hp += 3;
+	res  = CONS(hp, tup, res); hp += 2;
+
+	sz   = strlen(erts_build_flags_LDFLAGS);
+	text = buf_to_intlist(&hp, erts_build_flags_LDFLAGS, sz, NIL);
+	tup  = TUPLE2(hp, am_ldflags, text); hp += 3;
+	res  = CONS(hp, tup, res); hp += 2;
+
+	BIF_RET(res);
     }
 
     BIF_ERROR(BIF_P, BADARG);
-- 
cgit v1.2.3


From 2bf244968fb368b204e7af4b33b72d6a71a937c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 24 Jul 2013 19:55:47 +0200
Subject: erts: Extend erl_driver interface with lock names

Lock and thread names are already a feature in the driver interface.
This extension will let developers read these names. Eases debugging.
---
 erts/emulator/beam/erl_driver.h     |  8 ++++---
 erts/emulator/beam/erl_drv_thread.c | 42 +++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index e280563de1..42b14cd7bc 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -546,6 +546,11 @@ EXTERN int erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2);
 EXTERN void erl_drv_thread_exit(void *resp);
 EXTERN int erl_drv_thread_join(ErlDrvTid, void **respp);
 
+EXTERN char* erl_drv_mutex_name(ErlDrvMutex *mtx);
+EXTERN char* erl_drv_cond_name(ErlDrvCond *cnd);
+EXTERN char* erl_drv_rwlock_name(ErlDrvRWLock *rwlck);
+EXTERN char* erl_drv_thread_name(ErlDrvTid tid);
+
 /*
  * Misc.
  */
@@ -681,6 +686,3 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 
 /* also in global.h, but driver's can't include global.h */
 void dtrace_drvport_str(ErlDrvPort port, char *port_buf);
-
-
-
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index a49a155701..4f1bba8657 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -188,6 +188,17 @@ erl_drv_mutex_destroy(ErlDrvMutex *dmtx)
 #endif
 }
 
+
+char *
+erl_drv_mutex_name(ErlDrvMutex *dmtx)
+{
+#ifdef USE_THREADS
+    return dmtx ? dmtx->name : NULL;
+#else
+    return NULL;
+#endif
+}
+
 int
 erl_drv_mutex_trylock(ErlDrvMutex *dmtx)
 {
@@ -258,6 +269,15 @@ erl_drv_cond_destroy(ErlDrvCond *dcnd)
 #endif
 }
 
+char *
+erl_drv_cond_name(ErlDrvCond *dcnd)
+{
+#ifdef USE_THREADS
+    return dcnd ? dcnd->name : NULL;
+#else
+    return NULL;
+#endif
+}
 
 void
 erl_drv_cond_signal(ErlDrvCond *dcnd)
@@ -331,6 +351,16 @@ erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck)
 #endif
 }
 
+char *
+erl_drv_rwlock_name(ErlDrvRWLock *drwlck)
+{
+#ifdef USE_THREADS
+    return drwlck ? drwlck->name : NULL;
+#else
+    return NULL;
+#endif
+}
+
 int
 erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 {
@@ -617,6 +647,18 @@ erl_drv_thread_create(char *name,
 #endif
 }
 
+char *
+erl_drv_thread_name(ErlDrvTid tid)
+{
+#ifdef USE_THREADS
+    struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid;
+    return dtid ? dtid->name : NULL;
+#else
+    return NULL;
+#endif
+}
+
+
 ErlDrvTid
 erl_drv_thread_self(void)
 {
-- 
cgit v1.2.3


From 397caf87b39380f5517954320f25fdf9b97d0c90 Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Tue, 30 Jul 2013 11:39:35 +0200
Subject: Change default of erlang:halt/2 to the documented

Bug reported by Jose Valim on the erlang-bugs mailing list:
erlang:halt(0,[]) does not flush as advertised in the documentation.
Should be the same as erlang:halt(0,[{flush,true}]), but is in fact
the same as  erlang:halt(0,[{flush,false}]).
---
 erts/emulator/beam/bif.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ff237b6a78..755c5e6882 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3934,7 +3934,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 {
     Sint code;
     Eterm optlist = BIF_ARG_2;
-    int flush = 0;
+    int flush = 1;
 
     for (optlist = BIF_ARG_2;
 	 is_list(optlist);
-- 
cgit v1.2.3


From 7a0cc794dddd65ef825571056b71e3fca7f4f315 Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Mon, 15 Jul 2013 11:23:38 +0200
Subject: Add erts_prefix to pcre_library and update erl_bif_re

---
 erts/emulator/beam/erl_bif_re.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 3d34c2a77f..4289c79ba2 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -492,7 +492,7 @@ typedef struct _return_info {
 } ReturnInfo;
 
 typedef struct _restart_context {
-    pcre_extra extra;
+    erts_pcre_extra extra;
     void *restart_data;
     Uint32 flags;
     char *subject; /* to be able to free it when done */
-- 
cgit v1.2.3


From 390ee65ba78d571fdc0a8fccb6c456d7346eb9bc Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Fri, 19 Jul 2013 17:51:11 +0200
Subject: Handle CRLF correctly in global regexp

---
 erts/emulator/beam/erl_bif_re.c | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 4289c79ba2..12fc834685 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -379,6 +379,8 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
     Eterm ret;
     size_t pattern_size;
     int capture_count;
+    int use_crlf;
+    unsigned long options;
     if (!result) {
 	/* Return {error_tag, {Code, String, Offset}} */
 	int elen = sys_strlen(errstr);
@@ -393,14 +395,20 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
     } else {
 	erts_pcre_fullinfo(result, NULL, PCRE_INFO_SIZE, &pattern_size);
 	erts_pcre_fullinfo(result, NULL, PCRE_INFO_CAPTURECOUNT, &capture_count);
+	erts_pcre_fullinfo(result, NULL, PCRE_INFO_OPTIONS, &options);
+	options &= PCRE_NEWLINE_CR|PCRE_NEWLINE_LF | PCRE_NEWLINE_CRLF |
+               PCRE_NEWLINE_ANY | PCRE_NEWLINE_ANYCRLF;
+	use_crlf = (options == PCRE_NEWLINE_ANY ||
+		    options == PCRE_NEWLINE_CRLF ||
+		    options == PCRE_NEWLINE_ANYCRLF);
 	/* XXX: Optimize - keep in offheap binary to allow this to 
 	   be kept across traps w/o need of copying */
 	ret = new_binary(p, (byte *) result, pattern_size);
 	erts_pcre_free(result);
-	hp = HAlloc(p, (with_ok) ? (3+5) : 5);
-	ret = TUPLE4(hp,am_re_pattern, make_small(capture_count), make_small(unicode),ret);
+	hp = HAlloc(p, (with_ok) ? (3+6) : 6);
+	ret = TUPLE5(hp,am_re_pattern, make_small(capture_count), make_small(unicode),make_small(use_crlf),ret);
 	if (with_ok) {
-	    hp += 5;
+	    hp += 6;
 	    ret = TUPLE2(hp,am_ok,ret);
 	}	    
     }
@@ -875,7 +883,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
     is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && 
 		   (capture[CAPSPEC_TYPE] == am_list));
 
-    if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 4)) {
+    if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 5)) {
 	if (is_binary(arg2) || is_list(arg2) || is_nil(arg2)) {
 	    /* Compile from textual RE */
 	    ErlDrvSizeT slen;
@@ -947,7 +955,8 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 
 	tp = tuple_val(arg2);
 	if (tp[1] != am_re_pattern || is_not_small(tp[2]) || 
-	    is_not_small(tp[3]) || is_not_binary(tp[4])) {
+	    is_not_small(tp[3]) || is_not_small(tp[4]) || 
+	    is_not_binary(tp[5])) {
 	    BIF_ERROR(p,BADARG);
 	}
 
@@ -967,9 +976,9 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	}
 
 	ovsize = 3*(unsigned_val(tp[2])+1);
-	code_size = binary_size(tp[4]);
+	code_size = binary_size(tp[5]);
 	if ((code_tmp = (const pcre *) 
-	     erts_get_aligned_binary_bytes(tp[4], &temp_alloc)) == NULL) {
+	     erts_get_aligned_binary_bytes(tp[5], &temp_alloc)) == NULL) {
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    BIF_ERROR(p, BADARG);
 	}
@@ -1055,9 +1064,10 @@ handle_iolist:
 #ifdef DEBUG
     loop_count = 0xFFFFFFFF;
 #endif
-    
-    rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, slength, startoffset, 
-		   options, restart.ovector, ovsize);
+
+    rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, 
+			slength, startoffset, 
+			options, restart.ovector, ovsize);
     ASSERT(loop_count != 0xFFFFFFFF);
     BUMP_REDS(p, loop_count / LOOP_FACTOR);
     if (rc == PCRE_ERROR_LOOP_LIMIT) {
-- 
cgit v1.2.3


From f499b39ddc5025074c5d22f272f1dd55df79f009 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 2 Aug 2013 15:04:14 +0200
Subject: erts: Fix race in ptab that can cause PID mix-ups

Since: R16B01

Symptom: A spawned process may get the same PID as an existing
process. The new process will "steal" the PID and make the old process
unreachable through the PID.
The problem also applies to port identities but has only been seen
for processes.

Conditions: SMP emulator with at least two scheduler threads. Rapid
spawning and termination of a large number of processes. A small
number of free slots in the process table will also increase the risk
for this bug.

Workaround: Use command line options "+P legacy" and "+Q legacy"

Cause: The race happens if a process terminates and gets stalled while
releasing its process table slot. The stall has to be so long (due to OS
preemptive scheduling most probably) for other schedluer threads to
consume all other free slots for newly spawn processes.

Fix: Write invalid-markers in the free-pid-table and do atomic exhange
operations in retry-loops to make sure each thread gets a unique PID.
---
 erts/emulator/beam/erl_ptab.c | 139 ++++++++++++++++++++++++++----------------
 erts/emulator/beam/erl_ptab.h |   2 +-
 2 files changed, 89 insertions(+), 52 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index d69619dd44..fa5482b841 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -433,7 +433,7 @@ erts_ptab_mem_size(ErtsPTab *ptab)
 {
     UWord size = ptab->r.o.max*sizeof(erts_smp_atomic_t);
     if (ptab->r.o.free_id_data)
-	size += ptab->r.o.max*sizeof(Uint32);
+	size += ptab->r.o.max*sizeof(erts_smp_atomic32_t);
     return size;
 }
 
@@ -474,7 +474,7 @@ erts_ptab_init_table(ErtsPTab *ptab,
     tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic_t));
     alloc_sz = tab_sz;
     if (!legacy)
-	alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(Uint32));
+	alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t));
     ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, alloc_sz);
     tab_end = ((char *) ptab->r.o.tab) + tab_sz;
     tab_entry = ptab->r.o.tab;
@@ -497,6 +497,10 @@ erts_ptab_init_table(ErtsPTab *ptab,
 
     ASSERT(ptab->r.o.pix_cl_shift + ptab->r.o.pix_cli_shift == bits);
 
+    ptab->r.o.invalid_element = invalid_element;
+    ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id);
+    ptab->r.o.release_element = release_element;
+
     if (legacy) {
 	ptab->r.o.free_id_data = NULL;
 	ptab->r.o.dix_cl_mask = 0;
@@ -506,11 +510,11 @@ erts_ptab_init_table(ErtsPTab *ptab,
     }
     else {
 
-	tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(Uint32));
-	ptab->r.o.free_id_data = (Uint32 *) tab_end;
+	tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t));
+	ptab->r.o.free_id_data = (erts_smp_atomic32_t *) tab_end;
 
 	tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE;
-	ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(Uint32));
+	ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic32_t));
 
 	ptab->r.o.dix_cl_mask = tab_cache_lines-1;
 	ptab->r.o.dix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1);
@@ -525,7 +529,9 @@ erts_ptab_init_table(ErtsPTab *ptab,
 	ix = 0;
 	for (cl = 0; cl < tab_cache_lines; cl++) {
 	    for (cli = 0; cli < ix_per_cache_line; cli++) {
-		ptab->r.o.free_id_data[ix] = cli*tab_cache_lines+cl;
+		erts_smp_atomic32_init_nob(&ptab->r.o.free_id_data[ix],
+					   cli*tab_cache_lines+cl);
+		ASSERT(erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data);
 		ix++;
 	    }
 	}
@@ -534,9 +540,6 @@ erts_ptab_init_table(ErtsPTab *ptab,
 	erts_smp_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1);
 
     }
-    ptab->r.o.invalid_element = invalid_element;
-    ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id);
-    ptab->r.o.release_element = release_element;
 
     erts_smp_interval_init(&ptab->list.data.interval);
     ptab->list.data.deleted.start = NULL;
@@ -606,11 +609,13 @@ erts_ptab_new_element(ErtsPTab *ptab,
 	= erts_smp_current_interval_nob(erts_ptab_interval(ptab));
 
     if (ptab->r.o.free_id_data) {
+	do {
+	    ix = (Uint32) erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix);
+	    ix = ix_to_free_id_data_ix(ptab, ix);
 
-	ix = (Uint32) erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix);
-	ix = ix_to_free_id_data_ix(ptab, ix);
-
-	data = ptab->r.o.free_id_data[ix];
+	    data = erts_smp_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix],
+					      (erts_aint32_t)ptab->r.o.invalid_data);
+	}while ((Eterm)data == ptab->r.o.invalid_data);
 
 	init_ptab_el(init_arg, (Eterm) data);
 
@@ -763,7 +768,7 @@ erts_ptab_delete_element(ErtsPTab *ptab,
     erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL);
 
     if (ptab->r.o.free_id_data) {
-
+	Uint32 prev_data;
 	/* Next data for this slot... */
 	data = (Uint32) erts_ptab_id2data(ptab, ptab_el->id);
 	data += ptab->r.o.max;
@@ -772,14 +777,17 @@ erts_ptab_delete_element(ErtsPTab *ptab,
 	    data += ptab->r.o.max;
 	    data &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE);
 	}
-
 	ASSERT(data != ptab->r.o.invalid_data);
 	ASSERT(pix == erts_ptab_data2pix(ptab, data));
 
-	ix = (Uint32) erts_smp_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix);
-	ix = ix_to_free_id_data_ix(ptab, ix);
-
-	ptab->r.o.free_id_data[ix] = data;
+	do { 
+	    ix = (Uint32) erts_smp_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix);
+	    ix = ix_to_free_id_data_ix(ptab, ix);
+    
+	    prev_data = erts_smp_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix],
+						      data,
+						      ptab->r.o.invalid_data);
+	}while ((Eterm)prev_data != ptab->r.o.invalid_data);
     }
 
     ASSERT(erts_smp_atomic32_read_nob(&ptab->vola.tile.count) > 0);
@@ -1392,6 +1400,31 @@ erts_ptab_init(void)
  * Debug stuff
  */
 
+static void assert_ptab_consistency(ErtsPTab *ptab)
+{
+#ifdef DEBUG
+    if (ptab->r.o.free_id_data) {
+	Uint32 ix, pix, data;
+	int free_pids = 0;
+	int null_slots = 0;
+	
+	for (ix=0; ix < ptab->r.o.max; ix++) {
+	    if (erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) {
+		++free_pids;
+		data = erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]);
+		pix = erts_ptab_data2pix(ptab, (Eterm) data);
+		ASSERT(erts_ptab_pix2intptr_nob(ptab, pix) == ERTS_AINT_NULL);
+	    }
+	    if (erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) {
+		++null_slots;
+	    }
+	}	
+	ASSERT(free_pids == null_slots);
+	ASSERT(free_pids == ptab->r.o.max - erts_smp_atomic32_read_nob(&ptab->vola.tile.count));
+    }
+#endif
+}
+
 Sint
 erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next)
 {
@@ -1402,46 +1435,49 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next)
 
     erts_ptab_rwlock(ptab);
 
+    assert_ptab_consistency(ptab);
+
     if (ptab->r.o.free_id_data) {
-	Uint32 aid_ix, dix;
+	Uint32 id_ix, dix;
 
 	if (set) {
-	    Uint32 max_ix, ser, num, start;
+	    Uint32 i, max_ix, num, stop_id_ix;
 	    max_ix = ptab->r.o.max - 1;
-	    ser = next & ~max_ix;
-	    start = num = next & max_ix;
-	
-	    aid_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1;
-
-	    do {
-		Uint32 pix = erts_ptab_data2pix(ptab, num);
+	    num = next;
+	    id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix);
+
+	    for (i=0; i <= max_ix; ++i) {
+		Uint32 pix;
+		++num;
+		num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE);
+		if (num == ptab->r.o.invalid_data) {
+		    num += ptab->r.o.max;
+		    num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE);
+		}
+		pix = erts_ptab_data2pix(ptab, num);
 		if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) {
-		    dix = ix_to_free_id_data_ix(ptab, aid_ix);
-		    ptab->r.o.free_id_data[dix] = ser + num;
-		    ASSERT(pix == erts_ptab_data2pix(ptab, ser+num));
-		    if (aid_ix == max_ix)
-			aid_ix = 0;
-		    else
-			aid_ix++;
+		    ++id_ix;
+		    dix = ix_to_free_id_data_ix(ptab, id_ix);
+		    erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num);
+		    ASSERT(pix == erts_ptab_data2pix(ptab, num));
 		}
-		if (num == max_ix)
-		    num = 0;
-		else
-		    num++;
-	    } while (num != start);
+	    }
+	    erts_smp_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix);
 
-#ifdef DEBUG
-	    if (aid_ix == 0)
-		aid_ix = max_ix;
-	    else
-		aid_ix--;
-	    ASSERT((aid_ix & max_ix) == (((Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.fid_ix)) & max_ix));
-#endif
+	    /* Write invalid_data in rest of free_id_data[]: */
+	    stop_id_ix = (1 + erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix;
+	    while (1) {
+		id_ix = (id_ix+1) & max_ix;
+		if (id_ix == stop_id_ix)
+		    break;
+		dix = ix_to_free_id_data_ix(ptab, id_ix);
+		erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix],
+					  ptab->r.o.invalid_data);
+	    }
 	}
-
-	aid_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1;
-	dix = ix_to_free_id_data_ix(ptab, aid_ix);
-	res = (Sint) ptab->r.o.free_id_data[dix];
+	id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1;
+	dix = ix_to_free_id_data_ix(ptab, id_ix);
+	res = (Sint) erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[dix]);
     }
     else {
 	/* Deprecated legacy algorithm... */
@@ -1485,6 +1521,7 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next)
 	}
     }
 
+    assert_ptab_consistency(ptab);
     erts_ptab_rwunlock(ptab);
 
     return res;
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index c2d8bd9cad..e3e05f14af 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -100,7 +100,7 @@ typedef struct {
 
 typedef struct {
     erts_smp_atomic_t *tab;
-    Uint32 *free_id_data;
+    erts_smp_atomic32_t *free_id_data;
     Uint32 max;
     Uint32 pix_mask;
     Uint32 pix_cl_mask;
-- 
cgit v1.2.3


From a6a08b084f5abe2ab190387918e8a0f4366d454c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Tue, 6 Aug 2013 10:54:57 +0200
Subject: erts: Do not enable TRACE_SILENT when testing a ms

---
 erts/emulator/beam/erl_db_util.c | 5 ++++-
 erts/emulator/beam/global.h      | 7 ++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 713ac0ba18..ef3749a2c4 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2319,6 +2319,8 @@ restart:
 	    break;
 	case matchSilent:
 	    --esp;
+	    if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT)
+	      break;
 	    if (*esp == am_true) {
 		erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		ERTS_TRACE_FLAGS(c_p) |= F_TRACE_SILENT;
@@ -4971,7 +4973,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 	    save_cp = p->cp;
 	    p->cp = NULL;
 	    res = erts_match_set_run(p, mps, arr, n,
-				     ERTS_PAM_COPY_RESULT, &ret_flags);
+		      ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT,
+		      &ret_flags);
 	    p->cp = save_cp;
 	} else {
 	    n = 0;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index cecfa8a0fd..bacd5a5752 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1018,9 +1018,10 @@ Eterm erts_match_set_lint(Process *p, Eterm matchexpr);
 extern void erts_match_set_release_result(Process* p);
 
 enum erts_pam_run_flags {
-    ERTS_PAM_TMP_RESULT=0,
-    ERTS_PAM_COPY_RESULT=1,
-    ERTS_PAM_CONTIGUOUS_TUPLE=2
+    ERTS_PAM_TMP_RESULT=1,
+    ERTS_PAM_COPY_RESULT=2,
+    ERTS_PAM_CONTIGUOUS_TUPLE=4,
+    ERTS_PAM_IGNORE_TRACE_SILENT=8
 };
 extern Eterm erts_match_set_run(Process *p, Binary *mpsp, 
 				Eterm *args, int num_args,
-- 
cgit v1.2.3


From 6146e7642d4bb9f7c9bb5f8cbca548c1d9667e5c Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Thu, 18 Jul 2013 10:18:58 +0200
Subject: Add new options to Erlang re interface and mend dupnames

Add notempty_atstart, no_start_optimize, ucp and
never_utf options from new PCRE version.

Use the new notempty_atstart in global matching.

Add inspect/2 function

Correctly handle dupnames when capturing a name, as
in Perl, get the leftmost matching occurence.
Also added all_names, to get all the names in the pattern
in alphabetical (name) order.
To be able to use this in global matching, an inspect
function that can dig out a namelist was added.
---
 erts/emulator/beam/atom.names   |   6 ++
 erts/emulator/beam/bif.tab      |   6 ++
 erts/emulator/beam/erl_bif_re.c | 233 +++++++++++++++++++++++++++++++++++++++-
 3 files changed, 240 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index eba1d0fa23..cf8f511b85 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -71,6 +71,7 @@ atom ac
 atom active
 atom all
 atom all_but_first
+atom all_names
 atom alloc_info
 atom alloc_sizes
 atom allocated
@@ -348,11 +349,13 @@ atom multi_scheduling
 atom multiline
 atom name
 atom named_table
+atom namelist
 atom native_addresses
 atom Neq='=/='
 atom Neqeq='/='
 atom net_kernel
 atom net_kernel_terminated
+atom never_utf
 atom new
 atom new_index
 atom new_uniq
@@ -378,6 +381,7 @@ atom nosuspend
 atom no_float
 atom no_integer
 atom no_network
+atom no_start_optimize
 atom not
 atom not_a_list
 atom not_loaded
@@ -388,6 +392,7 @@ atom notalive
 atom notbol
 atom noteol
 atom notempty
+atom notempty_atstart
 atom notify
 atom notsup
 atom nouse_stdio
@@ -554,6 +559,7 @@ atom true
 atom tuple
 atom type
 atom ucompile
+atom ucp
 atom undef
 atom ungreedy
 atom unicode
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index dc8e9101de..7c8e4b31cf 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -573,6 +573,12 @@ bif erlang:binary_to_float/1
 
 bif io:printable_range/0
 
+#
+# New in R17A
+#
+
+bif re:inspect/2
+
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 12fc834685..c74125ae41 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -288,6 +288,10 @@ parse_options(Eterm listp, /* in */
 		    eopt |= PCRE_NOTEMPTY; 
 		    fl |= PARSE_FLAG_UNIQUE_EXEC_OPT;
 		    break;
+		case am_notempty_atstart:
+		    eopt |= PCRE_NOTEMPTY_ATSTART; 
+		    fl |= PARSE_FLAG_UNIQUE_EXEC_OPT;
+		    break;
 		case am_notbol:
 		    eopt |= PCRE_NOTBOL; 
 		    fl |= PARSE_FLAG_UNIQUE_EXEC_OPT;
@@ -296,6 +300,10 @@ parse_options(Eterm listp, /* in */
 		    eopt |= PCRE_NOTEOL; 
 		    fl |= PARSE_FLAG_UNIQUE_EXEC_OPT;
 		    break;
+		case am_no_start_optimize:
+		    copt |= PCRE_NO_START_OPTIMIZE; 
+		    fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT;
+		    break;
 		case am_caseless:
 		    copt |= PCRE_CASELESS; 
 		    fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT;
@@ -332,6 +340,14 @@ parse_options(Eterm listp, /* in */
 		    copt |= PCRE_UNGREEDY; 
 		    fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT;
 		    break;
+		case am_ucp:
+		    copt |= PCRE_UCP; 
+		    fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT;
+		    break;
+		case am_never_utf:
+		    copt |= PCRE_NEVER_UTF; 
+		    fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT;
+		    break;
 		case am_unicode:
 		    copt |= PCRE_UTF8; 
 		    fl |= (PARSE_FLAG_UNIQUE_COMPILE_OPT | PARSE_FLAG_UNICODE);
@@ -359,7 +375,7 @@ parse_options(Eterm listp, /* in */
     if (compile_options != NULL) {
 	*compile_options = copt;
     }
-    if (exec_options != NULL) {
+   if (exec_options != NULL) {
 	*exec_options = eopt;
     }
     if (flags != NULL) {
@@ -585,6 +601,17 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete
 				      ri->num_spec * 2 * sizeof(Eterm));
 		for (i = 0; i < ri->num_spec; ++i) {
 		    x = ri->v[i];
+		    if (x < -1) {
+			int n = i-x+1;
+			int j;
+			for (j = i+1; j < ri->num_spec && j < n; ++j) {
+			    if (restartp->ovector[(ri->v[j])*2] >= 0) {
+				x = ri->v[j];
+				break;
+			    }
+			}
+			i = n-1;
+		    }
 		    if (x < rc && x >= 0) {
 			tmp_vect[n*2] = make_signed_integer(restartp->ovector[x*2],p);
 			tmp_vect[n*2+1] = make_signed_integer(restartp->ovector[x*2+1]-restartp->ovector[x*2],p);
@@ -666,6 +693,17 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete
 				      ri->num_spec * sizeof(Eterm));
 		for (i = 0; i < ri->num_spec; ++i) {
 		    x = ri->v[i];
+		    if (x < -1) {
+			int n = i-x+1;
+			int j;
+			for (j = i+1; j < ri->num_spec && j < n; ++j) {
+			    if (restartp->ovector[(ri->v[j])*2] >= 0) {
+				x = ri->v[j];
+				break;
+			    }
+			}
+			i = n-1;
+		    }
 		    if (x < rc && x >= 0) {
 			char *cp;
 			int len;
@@ -730,6 +768,49 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete
  */
 
 #define RINFO_SIZ(Num) (sizeof(ReturnInfo) + (sizeof(int) * (Num - 1)))
+#define PICK_INDEX(NameEntry)					        \
+    ((int) ((((unsigned) ((unsigned char *) (NameEntry))[0]) << 8) +	\
+	    ((unsigned) ((unsigned char *) (NameEntry))[1])))
+
+
+static void build_one_capture(const pcre *code, ReturnInfo **ri, int *sallocated, int has_dupnames, char *name) 
+{
+    ReturnInfo *r = (*ri);
+    if (has_dupnames) {
+	/* Build a sequence of positions, starting with -size if
+	   more than one, otherwise just put the index there... */
+	char *first,*last;
+	int esize = erts_pcre_get_stringtable_entries(code,name,&first,&last);
+	if (esize == PCRE_ERROR_NOSUBSTRING) {
+	    r->v[r->num_spec - 1] = -1;
+	} else if(last == first) {
+	    r->v[r->num_spec - 1] = PICK_INDEX(first);
+	} else {
+	    int num = ((last - first) / esize) + 1;
+	    int i;
+	    ASSERT(num > 1);
+	    r->v[r->num_spec - 1] = -num; /* A value less than -1 means
+					       multiple indexes for same name */
+	    for (i = 0; i < num; ++i) {
+		++(r->num_spec);
+		if(r->num_spec > (*sallocated)) {
+		    (*sallocated) += 10;
+		    r = erts_realloc(ERTS_ALC_T_RE_SUBJECT, r, 
+				      RINFO_SIZ((*sallocated)));
+		}
+		r->v[r->num_spec - 1] = PICK_INDEX(first);
+		first += esize;
+	    }
+	}
+    } else {
+	/* Use the faster binary search if no duplicate names are present */  
+	if ((r->v[r->num_spec - 1] = erts_pcre_get_stringnumber(code,name)) ==
+	    PCRE_ERROR_NOSUBSTRING) {
+	    r->v[r->num_spec - 1] = -1;
+	}
+    }
+    *ri = r;
+}    
 
 static ReturnInfo *
 build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
@@ -778,6 +859,53 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 	}
 	ri->v[ri->num_spec - 1] = 0;
 	break;
+    case am_all_names:
+	{
+	    int rc,i,top;
+	    int entrysize;
+	    char *nametable, *last = NULL;
+	    int has_dupnames;
+	    unsigned long options;
+
+	    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0)
+		goto error;
+	    if ((rc = erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0)
+		goto error;
+	    if (top <= 0) {
+		ri->num_spec = 0;
+		ri->type = RetNone;
+		break;
+	    }
+	    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize) != 0)
+		goto error;
+	    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, (unsigned char **) &nametable) != 0)
+		goto error;
+	    
+	    has_dupnames = ((options & PCRE_DUPNAMES) != 0);
+
+	    for(i=0;i<top;++i) {
+		if (last == NULL || !has_dupnames || strcmp(last+2,nametable+2)) {
+		    if (ri->num_spec < 0)
+			ri->num_spec = 0;
+		    ++(ri->num_spec);
+		    if(ri->num_spec > sallocated) {
+			sallocated += 10;
+			ri = erts_realloc(ERTS_ALC_T_RE_SUBJECT, ri, RINFO_SIZ(sallocated));
+		    }
+		    if (has_dupnames) {
+			/* This could be more effective, we actually have 
+			   the names and could fill in the vector
+			   immediately. Now we lookup the name again. */
+			build_one_capture(code,&ri,&sallocated,has_dupnames,nametable+2);
+		    } else {
+			ri->v[ri->num_spec - 1] = PICK_INDEX(nametable);	
+		    }
+		}
+		last = nametable;
+		nametable += entrysize;
+	    }
+	    break;
+	}
     default:
 	if (is_list(capture_spec[CAPSPEC_VALUES])) {
 	    for(l=capture_spec[CAPSPEC_VALUES];is_list(l);l = CDR(list_val(l))) {
@@ -793,6 +921,11 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 		if (term_to_int(val,&x)) {
 		    ri->v[ri->num_spec - 1] = x;
 		} else if (is_atom(val) || is_binary(val) || is_list(val)) {
+		    int has_dupnames;
+		    unsigned long options;
+		    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0)
+			goto error;
+		    has_dupnames = ((options & PCRE_DUPNAMES) != 0);
 		    if (is_atom(val)) {
 			Atom *ap = atom_tab(atom_val(val));
 			if ((ap->len + 1) > tmpbsiz) {
@@ -823,10 +956,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 			}
 			tmpb[slen] = '\0';
 		    }
-		    if ((ri->v[ri->num_spec - 1] = erts_pcre_get_stringnumber(code,tmpb)) ==
-			PCRE_ERROR_NOSUBSTRING) {
-			ri->v[ri->num_spec - 1] = -1;
-		    }
+		    build_one_capture(code,&ri,&sallocated,has_dupnames,tmpb);
 		} else {
 		    goto error;
 		}
@@ -1159,6 +1289,99 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3)
     BIF_RET(res);
 }
     
+BIF_RETTYPE
+re_inspect_2(BIF_ALIST_2) 
+{
+    Eterm *tp,*tmp_vec,*hp;
+    int rc,i,top,j;
+    int entrysize;
+    char *nametable, *last,*name;
+    int has_dupnames;
+    unsigned long options;
+    int num_names;
+    Eterm res;
+    const pcre *code;
+    byte *temp_alloc = NULL;
+
+    if (is_not_tuple(BIF_ARG_1) || (arityval(*tuple_val(BIF_ARG_1)) != 5)) {
+	goto error;
+    }
+    tp = tuple_val(BIF_ARG_1);
+    if (tp[1] != am_re_pattern || is_not_small(tp[2]) || 
+	is_not_small(tp[3]) || is_not_small(tp[4]) || 
+	is_not_binary(tp[5])) {
+	goto error;
+    }
+    if (BIF_ARG_2 != am_namelist) {
+	goto error;
+    }
+    if ((code = (const pcre *) 
+	 erts_get_aligned_binary_bytes(tp[5], &temp_alloc)) == NULL) {
+	goto error;
+    }
+
+    /* OK, so let's try to get some info */
+    
+    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0)
+	goto error;
+    if ((rc = erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0)
+	goto error;
+    if (top <= 0) {
+	hp = HAlloc(BIF_P, 3);
+	res = TUPLE2(hp,am_namelist,NIL);
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_RET(res);
+    }
+    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize) != 0)
+	goto error;
+    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, (unsigned char **) &nametable) != 0)
+	goto error;
+    
+    has_dupnames = ((options & PCRE_DUPNAMES) != 0);
+    /* First, count the names */
+    num_names = 0;
+    last = NULL;
+    name = nametable;
+    for(i=0;i<top;++i) {
+	if (last == NULL || !has_dupnames || strcmp(last+2,name+2)) {
+	    ++num_names;
+	}
+	last = name;
+	name += entrysize;
+    }
+    tmp_vec =  erts_alloc(ERTS_ALC_T_RE_TMP_BUF, 
+			  num_names * sizeof(Eterm));
+    /* Re-iterate and fill tmp_vec */
+    last = NULL;
+    name = nametable;
+    j = 0;
+    for(i=0;i<top;++i) {
+	if (last == NULL || !has_dupnames || strcmp(last+2,name+2)) {
+	    tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, strlen(name+2));
+	}
+	last = name;
+	name += entrysize;
+    }
+    ASSERT(j == num_names);
+    hp = HAlloc(BIF_P, 3+2*j);
+    res = NIL;
+    for(i = j-1 ;i >= 0; --i) {
+	res = CONS(hp,tmp_vec[i],res);
+	hp += 2;
+    }
+    res = TUPLE2(hp,am_namelist,res);
+    erts_free_aligned_binary_bytes(temp_alloc);
+    erts_free(ERTS_ALC_T_RE_TMP_BUF, tmp_vec);
+    BIF_RET(res);
+
+ error:
+    /* tmp_vec never allocated when we reach here */
+    erts_free_aligned_binary_bytes(temp_alloc);
+    BIF_ERROR(BIF_P,BADARG);
+}
+    
+
+	
     
 
 	
-- 
cgit v1.2.3


From 1f4c016785a924b2e42fbb7858640be3d46e9625 Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Tue, 30 Jul 2013 17:41:14 +0200
Subject: Add return_errors option to re:run/3

---
 erts/emulator/beam/atom.names   |  3 +++
 erts/emulator/beam/erl_bif_re.c | 56 ++++++++++++++++++++++++++++++++++-------
 2 files changed, 50 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index cf8f511b85..8e5a079181 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -320,6 +320,8 @@ atom low
 atom Lt='<'
 atom machine
 atom match
+atom match_limit
+atom match_limit_recursion
 atom match_spec
 atom max
 atom maximum
@@ -477,6 +479,7 @@ atom register
 atom registered_name
 atom reload
 atom rem
+atom report_errors
 atom reset
 atom restart
 atom return_from
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index c74125ae41..46b5b848cf 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -180,6 +180,7 @@ static Eterm make_signed_integer(int x, Process *p)
 #define PARSE_FLAG_STARTOFFSET 8
 #define PARSE_FLAG_CAPTURE_OPT 16
 #define PARSE_FLAG_GLOBAL 32
+#define PARSE_FLAG_REPORT_ERRORS 64
 
 #define CAPSPEC_VALUES 0
 #define CAPSPEC_TYPE 1
@@ -276,7 +277,7 @@ parse_options(Eterm listp, /* in */
 		default:
 		    return -1; 
 		}
-	    }else if (is_not_atom(item)) {
+	    } else if (is_not_atom(item)) {
 		return -1;
 	    } else {
 		switch(item) {
@@ -348,6 +349,10 @@ parse_options(Eterm listp, /* in */
 		    copt |= PCRE_NEVER_UTF; 
 		    fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT;
 		    break;
+		case am_report_errors:
+		    fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT | 
+			   PARSE_FLAG_REPORT_ERRORS);
+		    break;
 		case am_unicode:
 		    copt |= PCRE_UTF8; 
 		    fl |= (PARSE_FLAG_UNIQUE_COMPILE_OPT | PARSE_FLAG_UNICODE);
@@ -389,7 +394,7 @@ parse_options(Eterm listp, /* in */
  */
 
 static Eterm 
-build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, const char *errstr, int errofset, int unicode, int with_ok) 
+build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, const char *errstr, int errofset, int unicode, int with_ok, Eterm extra_err_tag) 
 {
     Eterm *hp;
     Eterm ret;
@@ -402,11 +407,18 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
 	int elen = sys_strlen(errstr);
 	int need = 3 /* tuple of 2 */ + 
 	    3 /* tuple of 2 */ + 
-	    (2 * elen) /* The error string list */;
+	    (2 * elen) /* The error string list */ +
+	    ((extra_err_tag != NIL) ? 3 : 0);
 	hp = HAlloc(p, need);
 	ret = buf_to_intlist(&hp, (char *) errstr, elen, NIL);
 	ret = TUPLE2(hp, ret, make_small(errofset));
 	hp += 3;
+	if (extra_err_tag != NIL) {
+	    /* Return {error_tag, {extra_tag, 
+	       {Code, String, Offset}}} instead */
+	    ret =  TUPLE2(hp, extra_err_tag, ret);
+	    hp += 3;
+	}
 	ret = TUPLE2(hp, error_tag, ret);
     } else {
 	erts_pcre_fullinfo(result, NULL, PCRE_INFO_SIZE, &pattern_size);
@@ -478,7 +490,7 @@ re_compile(Process* p, Eterm arg1, Eterm arg2)
 			   &errstr, &errofset, default_table);
 
     ret = build_compile_result(p, am_error, result, errcode,
-			       errstr, errofset, unicode, 1);
+			       errstr, errofset, unicode, 1, NIL);
     erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
     BIF_RET(ret);
 }
@@ -526,6 +538,7 @@ typedef struct _restart_context {
 } RestartContext;
 
 #define RESTART_FLAG_SUBJECT_IN_BINARY 0x1
+#define RESTART_FLAG_REPORT_MATCH_LIMIT 0x2
 
 static void cleanup_restart_context(RestartContext *rc) 
 {
@@ -566,7 +579,19 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete
     Eterm res;
     Eterm *hp;
     if (rc <= 0) {
-	res = am_nomatch;
+	if (restartp->flags & RESTART_FLAG_REPORT_MATCH_LIMIT) {
+	    if (rc == PCRE_ERROR_MATCHLIMIT) {
+		hp = HAlloc(p,3);
+		res = TUPLE2(hp,am_error,am_match_limit);
+	    } else if (rc == PCRE_ERROR_RECURSIONLIMIT) {
+		hp = HAlloc(p,3);
+		res = TUPLE2(hp,am_error,am_match_limit_recursion);
+	    } else {
+		res = am_nomatch;
+	    }
+	} else {
+	    res = am_nomatch;
+	}
     } else {
 	ReturnInfo *ri = restartp->ret_info; 
 	ReturnInfo defri = {RetIndex,0,{0}};
@@ -1043,10 +1068,20 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    result = erts_pcre_compile2(expr, comp_options, &errcode, 
 				   &errstr, &errofset, default_table);
 	    if (!result) {
-		erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
 		/* Compilation error gives badarg except in the compile 
-		   function */
-		BIF_ERROR(p,BADARG);
+		   function or if we have PARSE_FLAG_REPORT_ERRORS */
+		if (pflags &  PARSE_FLAG_REPORT_ERRORS) {
+		    res = build_compile_result(p, am_error, result, errcode,
+					       errstr, errofset, 
+					       (pflags & 
+						PARSE_FLAG_UNICODE) ? 1 : 0, 
+					       1, am_compile);
+		    erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
+		    BIF_RET(res);
+		} else {
+		    erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
+		    BIF_ERROR(p,BADARG);
+		}
 	    }
 	    if (pflags & PARSE_FLAG_GLOBAL) {
 		Eterm precompiled = 
@@ -1055,7 +1090,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 					 errstr, errofset, 
 					 (pflags & 
 					  PARSE_FLAG_UNICODE) ? 1 : 0,
-					 0);
+					 0, NIL);
 		Eterm *hp,r;
 		erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
 		hp = HAlloc(p,4);
@@ -1190,6 +1225,9 @@ handle_iolist:
 	}
     }
 
+    if (pflags & PARSE_FLAG_REPORT_ERRORS) {
+	restart.flags |= RESTART_FLAG_REPORT_MATCH_LIMIT;
+    }
 
 #ifdef DEBUG
     loop_count = 0xFFFFFFFF;
-- 
cgit v1.2.3


From 8cbc9296944b5d1397d15e5615890b61549d5064 Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Wed, 31 Jul 2013 15:17:27 +0200
Subject: Add match_limit and match_limit_recursion options

Added to re:run and sets the corresponding fields in 'extra' struct
for the PCRE match engine. The result can be viewed by also
setting 'report_errors' when matching.

Some housekeeping was also done...

The offset option also did not properly check for offset's >= 0.

Change nomatch to BADARG when pre-compiled mp() is faked:
By constructing a 5-tuple with faked content but the right data types,
you could do a re:run which returned nomatch when in fact the mp() was
bad. The cheapest solution is to check the return from pcre_exec
better.

Remove unreachable code in erts_bif_re.c:
Replaced tests for things that logically simply
cannot happen with ASSERT.
---
 erts/emulator/beam/erl_bif_re.c | 156 ++++++++++++++++++++++++++++++----------
 1 file changed, 120 insertions(+), 36 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 46b5b848cf..796fd301ad 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -181,6 +181,8 @@ static Eterm make_signed_integer(int x, Process *p)
 #define PARSE_FLAG_CAPTURE_OPT 16
 #define PARSE_FLAG_GLOBAL 32
 #define PARSE_FLAG_REPORT_ERRORS 64
+#define PARSE_FLAG_MATCH_LIMIT 128
+#define PARSE_FLAG_MATCH_LIMIT_RECURSION 256
 
 #define CAPSPEC_VALUES 0
 #define CAPSPEC_TYPE 1
@@ -193,7 +195,9 @@ parse_options(Eterm listp, /* in */
 	      int *exec_options, /* out */
 	      int *flags,/* out */
 	      int *startoffset, /* out */
-	      Eterm *capture_spec) /* capture_spec[CAPSPEC_SIZE] */ /* out */
+	      Eterm *capture_spec, /* capture_spec[CAPSPEC_SIZE] */ /* out */
+	      int *match_limit, /* out */
+	      int *match_limit_recursion)  /* out */
 {
     int copt,eopt,fl;
     Eterm item;
@@ -235,7 +239,7 @@ parse_options(Eterm listp, /* in */
 		case am_offset:
 		    { 
 			int tmp;
-			if (!term_to_int(tp[2],&tmp)) {
+			if (!term_to_int(tp[2],&tmp) || tmp < 0) {
 			    return -1; 
 			}
 			if (startoffset != NULL) {
@@ -244,6 +248,31 @@ parse_options(Eterm listp, /* in */
 		    }
 		    fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT|PARSE_FLAG_STARTOFFSET);
 		    break;
+		case am_match_limit:
+		    { 
+			int tmp;
+			if (!term_to_int(tp[2],&tmp) || tmp < 0) {
+			    return -1; 
+			}
+			if (match_limit != NULL) {
+			    *match_limit = tmp;
+			}
+		    }
+		    fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT|PARSE_FLAG_MATCH_LIMIT);
+		    break;
+		case am_match_limit_recursion:
+		    { 
+			int tmp;
+			if (!term_to_int(tp[2],&tmp) || tmp < 0) {
+			    return -1; 
+			}
+			if (match_limit_recursion != NULL) {
+			    *match_limit_recursion = tmp;
+			}
+		    }
+		    fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT|
+			   PARSE_FLAG_MATCH_LIMIT_RECURSION);
+		    break;
 		case am_newline:
 		    if (!is_atom(tp[2])) {
 			return -1; 
@@ -460,9 +489,12 @@ re_compile(Process* p, Eterm arg1, Eterm arg2)
     int options = 0;
     int pflags = 0;
     int unicode = 0;
+#ifdef DEBUG
+    int buffres;
+#endif
 
 
-    if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL)
+    if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL,NULL,NULL)
 	< 0) {
 	BIF_ERROR(p,BADARG);
     }
@@ -481,10 +513,13 @@ re_compile(Process* p, Eterm arg1, Eterm arg2)
         BIF_ERROR(p,BADARG);
     }
     expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
-    if (erts_iolist_to_buf(arg1, expr, slen) != 0) {
-	erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
-	BIF_ERROR(p,BADARG);
-    }
+#ifdef DEBUG
+    buffres =
+#endif
+    erts_iolist_to_buf(arg1, expr, slen);
+
+    ASSERT(buffres >= 0);
+
     expr[slen]='\0';
     result = erts_pcre_compile2(expr, options, &errcode, 
 			   &errstr, &errofset, default_table);
@@ -910,8 +945,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 
 	    for(i=0;i<top;++i) {
 		if (last == NULL || !has_dupnames || strcmp(last+2,nametable+2)) {
-		    if (ri->num_spec < 0)
-			ri->num_spec = 0;
+		    ASSERT(ri->num_spec >= 0);
 		    ++(ri->num_spec);
 		    if(ri->num_spec > sallocated) {
 			sallocated += 10;
@@ -936,8 +970,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 	    for(l=capture_spec[CAPSPEC_VALUES];is_list(l);l = CDR(list_val(l))) {
 		int x;
 		Eterm val = CAR(list_val(l));
-		if (ri->num_spec < 0)
-		    ri->num_spec = 0;
+		ASSERT(ri->num_spec >= 0);
 		++(ri->num_spec);
 		if(ri->num_spec > sallocated) {
 		    sallocated += 10;
@@ -965,6 +998,10 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 			tmpb[ap->len] = '\0';
 		    } else {
 			ErlDrvSizeT slen;
+#ifdef DEBUG
+			int buffres;
+#endif
+
 			if (erts_iolist_size(val, &slen)) {
 			    goto error;
 			}
@@ -976,9 +1013,12 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 						    (tmpbsiz = slen + 1));
 			    }
 			}
-			if (erts_iolist_to_buf(val, tmpb, slen) != 0) {
-			    goto error;
-			}
+
+#ifdef DEBUG
+			buffres =
+#endif
+			erts_iolist_to_buf(val, tmpb, slen);
+			ASSERT(buffres >= 0);
 			tmpb[slen] = '\0';
 		    }
 		    build_one_capture(code,&ri,&sallocated,has_dupnames,tmpb);
@@ -1030,8 +1070,11 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
     unsigned long loop_count;
     Eterm capture[CAPSPEC_SIZE] = CAPSPEC_INIT;
     int is_list_cap;
+    int match_limit = 0;
+    int match_limit_recursion = 0;
 
-    if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture)
+    if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture,
+		      &match_limit,&match_limit_recursion)
 	< 0) {
 	BIF_ERROR(p,BADARG);
     }
@@ -1048,6 +1091,9 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    const char *errstr = "";
 	    int errofset = 0;
 	    int capture_count;
+#ifdef DEBUG
+	    int buffres;
+#endif
 
 	    if (pflags & PARSE_FLAG_UNICODE && 
 		(!is_binary(arg2) || !is_binary(arg1) ||
@@ -1060,10 +1106,14 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    }
 	    
 	    expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
-	    if (erts_iolist_to_buf(arg2, expr, slen) != 0) {
-		erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
-		BIF_ERROR(p,BADARG);
-	    }
+	    
+#ifdef DEBUG
+	    buffres =
+#endif
+	    erts_iolist_to_buf(arg2, expr, slen);
+
+	    ASSERT(buffres >= 0);
+
 	    expr[slen]='\0';
 	    result = erts_pcre_compile2(expr, comp_options, &errcode, 
 				   &errstr, &errofset, default_table);
@@ -1168,6 +1218,16 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
     restart.extra.restart_flags = 0;
     restart.extra.loop_counter_return = &loop_count;
     restart.ret_info = NULL;
+
+    if (pflags & PARSE_FLAG_MATCH_LIMIT) {
+	restart.extra.flags |= PCRE_EXTRA_MATCH_LIMIT;
+	restart.extra.match_limit = match_limit;
+    }
+
+    if (pflags & PARSE_FLAG_MATCH_LIMIT_RECURSION) {
+	restart.extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+	restart.extra.match_limit_recursion = match_limit_recursion;
+    }
     
     if (pflags & PARSE_FLAG_CAPTURE_OPT) {
 	if ((restart.ret_info = build_capture(capture,restart.code)) == NULL) {
@@ -1203,6 +1263,9 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	restart.subject = (char *) (pb->bytes+offset);
 	restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY;
     } else {
+#ifdef DEBUG
+	int buffres;
+#endif
 handle_iolist:
 	if (erts_iolist_size(arg1, &slength)) {
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
@@ -1214,15 +1277,11 @@ handle_iolist:
 	}
 	restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength);
 
-	if (erts_iolist_to_buf(arg1, restart.subject, slength) != 0) {
-	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
-	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
-	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.subject);
-	    if (restart.ret_info != NULL) {
-		erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info);
-	    }
-	    BIF_ERROR(p,BADARG);
-	}
+#ifdef DEBUG
+	buffres =
+#endif
+	erts_iolist_to_buf(arg1, restart.subject, slength);
+	ASSERT(buffres >= 0);
     }
 
     if (pflags & PARSE_FLAG_REPORT_ERRORS) {
@@ -1236,6 +1295,12 @@ handle_iolist:
     rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, 
 			slength, startoffset, 
 			options, restart.ovector, ovsize);
+
+    if (rc == PCRE_ERROR_BADENDIANNESS || rc == PCRE_ERROR_BADMAGIC) {
+	cleanup_restart_context(&restart);
+	BIF_ERROR(p,BADARG);
+    }
+    
     ASSERT(loop_count != 0xFFFFFFFF);
     BUMP_REDS(p, loop_count / LOOP_FACTOR);
     if (rc == PCRE_ERROR_LOOP_LIMIT) {
@@ -1255,7 +1320,7 @@ handle_iolist:
 		  arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */,
 		  magic_bin);
     }
-    
+
     res = build_exec_return(p, rc, &restart, arg1);
  
     cleanup_restart_context(&restart);
@@ -1331,7 +1396,7 @@ BIF_RETTYPE
 re_inspect_2(BIF_ALIST_2) 
 {
     Eterm *tp,*tmp_vec,*hp;
-    int rc,i,top,j;
+    int i,top,j;
     int entrysize;
     char *nametable, *last,*name;
     int has_dupnames;
@@ -1340,6 +1405,10 @@ re_inspect_2(BIF_ALIST_2)
     Eterm res;
     const pcre *code;
     byte *temp_alloc = NULL;
+#ifdef DEBUG
+    int infores;
+#endif
+    
 
     if (is_not_tuple(BIF_ARG_1) || (arityval(*tuple_val(BIF_ARG_1)) != 5)) {
 	goto error;
@@ -1362,18 +1431,33 @@ re_inspect_2(BIF_ALIST_2)
     
     if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0)
 	goto error;
-    if ((rc = erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0)
-	goto error;
+
+#ifdef DEBUG
+    infores =
+#endif
+    erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top);
+
+    ASSERT(infores == 0);
+
     if (top <= 0) {
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp,am_namelist,NIL);
 	erts_free_aligned_binary_bytes(temp_alloc);
 	BIF_RET(res);
     }
-    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize) != 0)
-	goto error;
-    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, (unsigned char **) &nametable) != 0)
-	goto error;
+#ifdef DEBUG
+    infores =
+#endif
+    erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize);
+
+    ASSERT(infores == 0);
+
+#ifdef DEBUG
+    infores =
+#endif
+    erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, (unsigned char **) &nametable);
+
+    ASSERT(infores == 0);
     
     has_dupnames = ((options & PCRE_DUPNAMES) != 0);
     /* First, count the names */
-- 
cgit v1.2.3


From 925d69c56d376d1e2371cd2236c981bc97ccb706 Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Tue, 6 Aug 2013 15:07:47 +0200
Subject: Clarify relation between erts_iolist_{size|to_buf}

Just some clarifying comments to future progremmers to keep the relation
between error returns from the two functions.
---
 erts/emulator/beam/utils.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 84deaecb87..c8695e61d5 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3021,6 +3021,14 @@ buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail)
 ** Return remaining bytes in buffer on success
 **        ERTS_IOLIST_TO_BUF_OVERFLOW on overflow
 **        ERTS_IOLIST_TO_BUF_TYPE_ERROR on type error (including that result would not be a whole number of bytes)
+**
+** Note! 
+** Do not detect indata errors in this fiunction that are not detected by erts_iolist_size!
+**
+** A caller should be able to rely on a successful return from erts_iolist_to_buf
+** if erts_iolist_size is previously successfully called and erts_iolist_to_buf 
+** is called with a buffer at least as large as the value given by erts_iolist_size.
+** 
 */
 
 ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
@@ -3127,6 +3135,11 @@ ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
 
 /*
  * Return 0 if successful, and non-zero if unsuccessful.
+ *
+ * It is vital that if erts_iolist_to_buf would return an error for
+ * any type of term data, this function should do so as well.
+ * Any input term error detected in erts_iolist_to_buf should also
+ * be detected in this function!
  */
 int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
 {
-- 
cgit v1.2.3


From 52cb62b7930d9c7b9e04a210ff6b02946f27ae79 Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Wed, 14 Aug 2013 17:09:13 +0200
Subject: Workaround TR gnu/181328, GCC 4.2.1 20070831 on FreeBSD 9.1

The bug manifests so that initialization of an automatic variable
happens after a pointer to the variable is dereferenced, causing
build_exec_return to take the wrong branch by default.

The bug is verified to happen even outside the VM and
is reported to FreeBSD as trouble ID gnu/181328.

The workaround is a slight code rewrite which is optimized differently.

Also removed two warnings about dereferencing type-punned
pointers, which did not affect the bug.
---
 erts/emulator/beam/erl_bif_re.c | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 796fd301ad..99c31738a5 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -628,11 +628,15 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete
 	    res = am_nomatch;
 	}
     } else {
-	ReturnInfo *ri = restartp->ret_info; 
+	ReturnInfo *ri;
 	ReturnInfo defri = {RetIndex,0,{0}};
-	if (ri == NULL) {
+
+	if (restartp->ret_info == NULL) {
 	    ri = &defri;
+	} else {
+	    ri = restartp->ret_info;
 	}
+
 	if (ri->type == RetNone) {
 	    res = am_match;
 	} else if (ri->type == RetIndex){
@@ -923,7 +927,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 	{
 	    int rc,i,top;
 	    int entrysize;
-	    char *nametable, *last = NULL;
+	    unsigned char *nametable, *last = NULL;
 	    int has_dupnames;
 	    unsigned long options;
 
@@ -938,13 +942,13 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 	    }
 	    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize) != 0)
 		goto error;
-	    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, (unsigned char **) &nametable) != 0)
+	    if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable) != 0)
 		goto error;
 	    
 	    has_dupnames = ((options & PCRE_DUPNAMES) != 0);
 
 	    for(i=0;i<top;++i) {
-		if (last == NULL || !has_dupnames || strcmp(last+2,nametable+2)) {
+		if (last == NULL || !has_dupnames || strcmp((char *) last+2,(char *) nametable+2)) {
 		    ASSERT(ri->num_spec >= 0);
 		    ++(ri->num_spec);
 		    if(ri->num_spec > sallocated) {
@@ -955,7 +959,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 			/* This could be more effective, we actually have 
 			   the names and could fill in the vector
 			   immediately. Now we lookup the name again. */
-			build_one_capture(code,&ri,&sallocated,has_dupnames,nametable+2);
+			build_one_capture(code,&ri,&sallocated,has_dupnames,(char *) nametable+2);
 		    } else {
 			ri->v[ri->num_spec - 1] = PICK_INDEX(nametable);	
 		    }
@@ -1236,7 +1240,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    BIF_ERROR(p,BADARG);
 	}
     }
-	    
+
     /*  Optimized - if already in binary off heap, keep that and avoid
        copying, also binary returns can be sub binaries in that case */
 
@@ -1398,7 +1402,7 @@ re_inspect_2(BIF_ALIST_2)
     Eterm *tp,*tmp_vec,*hp;
     int i,top,j;
     int entrysize;
-    char *nametable, *last,*name;
+    unsigned char *nametable, *last,*name;
     int has_dupnames;
     unsigned long options;
     int num_names;
@@ -1455,7 +1459,7 @@ re_inspect_2(BIF_ALIST_2)
 #ifdef DEBUG
     infores =
 #endif
-    erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, (unsigned char **) &nametable);
+    erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable);
 
     ASSERT(infores == 0);
     
@@ -1465,7 +1469,8 @@ re_inspect_2(BIF_ALIST_2)
     last = NULL;
     name = nametable;
     for(i=0;i<top;++i) {
-	if (last == NULL || !has_dupnames || strcmp(last+2,name+2)) {
+	if (last == NULL || !has_dupnames || strcmp((char *) last+2,
+						    (char *) name+2)) {
 	    ++num_names;
 	}
 	last = name;
@@ -1478,8 +1483,9 @@ re_inspect_2(BIF_ALIST_2)
     name = nametable;
     j = 0;
     for(i=0;i<top;++i) {
-	if (last == NULL || !has_dupnames || strcmp(last+2,name+2)) {
-	    tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, strlen(name+2));
+	if (last == NULL || !has_dupnames || strcmp((char *) last+2,
+						    (char *) name+2)) {
+	    tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, strlen((char *) name+2));
 	}
 	last = name;
 	name += entrysize;
-- 
cgit v1.2.3


From 20e0509d4e04fada3019639bc82d78b89f06b0fc Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Tue, 13 Aug 2013 17:14:11 +0200
Subject: erts: Add option to include nifs statically

Both crypto and asn1 are supported.
---
 erts/emulator/beam/erl_nif.c | 18 +++++++++++++-----
 erts/emulator/beam/erl_nif.h | 15 +++++++++++----
 erts/emulator/beam/global.h  |  6 ++++++
 erts/emulator/beam/utils.c   | 21 +++++++++++++++++++++
 4 files changed, 51 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 48f8be8dd3..673012a9af 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1213,7 +1213,8 @@ static void close_lib(struct erl_module_nif* lib)
 	lib->entry->unload(&env, lib->priv_data);
 	post_nif_noproc(&env);
     }
-    erts_sys_ddll_close(lib->handle);
+    if (!erts_is_static_nif(lib->handle))
+      erts_sys_ddll_close(lib->handle);
     lib->handle = NULL;
 }
 
@@ -1564,7 +1565,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     static const char upgrade[] = "upgrade";
     char* lib_name = NULL;
     void* handle = NULL;
-    void* init_func;
+    void* init_func = NULL;
     ErlNifEntry* entry = NULL;
     ErlNifEnv env;
     int len, i, err;
@@ -1577,6 +1578,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     int veto;
     struct erl_module_nif* lib = NULL;
     int reload_warning = 0;
+    char tmp_buf[255];
 
     len = list_length(BIF_ARG_1);
     if (len < 0) {
@@ -1613,13 +1615,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     mod=erts_get_module(mod_atom, erts_active_code_ix());
     ASSERT(mod != NULL);
 
+    init_func = erts_static_nif_get_nif_init(erts_basename(lib_name,tmp_buf));
+    if (init_func != NULL)
+      handle = init_func;
+
     if (!in_area(caller, mod->curr.code, mod->curr.code_length)) {
 	ASSERT(in_area(caller, mod->old.code, mod->old.code_length));
 
 	ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old "
 			     "module '%T' not allowed", mod_atom);
     }    
-    else if ((err=erts_sys_ddll_open2(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
+    else if (init_func == NULL &&
+	     (err=erts_sys_ddll_open2(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
 	const char slogan[] = "Failed to load NIF library";
 	if (strstr(errdesc.str, lib_name) != NULL) {
 	    ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str);
@@ -1628,7 +1635,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    ret = load_nif_error(BIF_P, "load_failed", "%s %s: '%s'", slogan, lib_name, errdesc.str);
 	}
     }
-    else if (erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) != ERL_DE_NO_ERROR) {
+    else if (init_func == NULL &&
+	     erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) != ERL_DE_NO_ERROR) {
 	ret  = load_nif_error(BIF_P, bad_lib, "Failed to find library init"
 			      " function: '%s'", errdesc.str);
 	
@@ -1784,7 +1792,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
         if (lib != NULL) {
 	    erts_free(ERTS_ALC_T_NIF, lib);
 	}
-	if (handle != NULL) {
+	if (handle != NULL && !erts_is_static_nif(handle)) {
 	    erts_sys_ddll_close(handle);
 	}
 	erts_sys_ddll_free_error(&errdesc);
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 8006741a63..5f4dc21d5c 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -168,7 +168,7 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  undef ERL_NIF_API_FUNC_DECL
 #endif
 
-#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER)
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER) && !defined(STATIC_ERLANG_NIF)
 #  define ERL_NIF_API_FUNC_MACRO(NAME) (WinDynNifCallbacks.NAME)
 #  include "erl_nif_api_funcs.h"
 /* note that we have to keep ERL_NIF_API_FUNC_MACRO defined */
@@ -180,15 +180,22 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  undef ERL_NIF_API_FUNC_DECL
 #endif
 
-
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks;
-#  define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
+#  ifdef STATIC_ERLANG_NIF
+#    define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* MODNAME ## _nif_init(TWinDynNifCallbacks* callbacks)
+#  else
+#    define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
+#  endif
 #  define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
 #else 
 #  define ERL_NIF_INIT_GLOB
 #  define ERL_NIF_INIT_BODY
-#  define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void)
+#  ifdef STATIC_ERLANG_NIF
+#    define ERL_NIF_INIT_DECL(MODNAME)  ErlNifEntry* MODNAME ## _nif_init(void)
+#  else
+#    define ERL_NIF_INIT_DECL(MODNAME)  ErlNifEntry* nif_init(void)
+#  endif
 #endif
 
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 7f7b6f5d1e..2acc86cf42 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -852,11 +852,17 @@ Port *erts_get_heart_port(void);
 void erts_lcnt_enable_io_lock_count(int enable);
 #endif
 
+/* driver_tab.c */
+typedef void *(*ErtsStaticNifInitFPtr)(void);
+ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name);
+int erts_is_static_nif(void *handle);
+
 /* erl_drv_thread.c */
 void erl_drv_thr_init(void);
 
 /* utils.c */
 void erts_cleanup_offheap(ErlOffHeap *offheap);
+const char *erts_basename(const char* path, char* buff);
 
 Uint64 erts_timestamp_millis(void);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index c71f8f0712..6293286c75 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -4019,6 +4019,27 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 #endif
 }
 
+const char *erts_basename(const char* path, char* buff) {
+  /* This function is not compliant with bash basename. Edge cases like "//"
+     and "/path//" do not work properly.
+   */
+  int i;
+  int len = strlen(path);
+  const char *basename = path;
+  for (i = 0; path[i] != '\0'; i++) {
+    if (path[i] == '/') {
+      if (path[i+1] == '\0') {
+	    memcpy(buff,basename,len - (basename-path));
+	    buff[len - (basename-path)-1] = '\0';
+	    basename = buff;
+	    break;
+      } else { basename = path+i;}
+    }
+  }
+  if (basename == path)
+    return path;
+  return basename+1;
+}
 
 /*
  * A millisecond timestamp without time correction where there's no hrtime
-- 
cgit v1.2.3


From 94d174949635cd4e641daaac0a7df98ea35f2c55 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 15 Aug 2013 17:04:52 +0200
Subject: erts: Add support for static linked-in drivers

None of the OTP linked-in driver are supported
---
 erts/emulator/beam/erl_driver.h | 14 ++++++++++----
 erts/emulator/beam/global.h     |  1 +
 erts/emulator/beam/io.c         |  1 +
 3 files changed, 12 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index e280563de1..f43f5a766e 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -365,17 +365,23 @@ typedef struct erl_drv_entry {
  * It must initialize a ErlDrvEntry structure and return a pointer to it.
  */
 
+#ifdef STATIC_ERLANG_DRIVER
+#  define ERLANG_DRIVER_NAME(NAME) NAME ## _driver_init
+#else
+#  define ERLANG_DRIVER_NAME(NAME) driver_init
+#endif
+
 /* For windows dynamic drivers */
 #ifndef ERL_DRIVER_TYPES_ONLY
 
 #if defined(__WIN32__)
 #  define DRIVER_INIT(DRIVER_NAME) \
-    __declspec(dllexport) ErlDrvEntry* driver_init(void); \
-    __declspec(dllexport) ErlDrvEntry* driver_init(void)
+  __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void);	\
+    __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
 #else 
 #  define DRIVER_INIT(DRIVER_NAME) \
-    ErlDrvEntry* driver_init(void); \
-    ErlDrvEntry* driver_init(void)
+    ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \
+    ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
 #endif
 
 #define ERL_DRV_BUSY_MSGQ_DISABLED	(~((ErlDrvSizeT) 0))
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2acc86cf42..ef34f5b221 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -856,6 +856,7 @@ void erts_lcnt_enable_io_lock_count(int enable);
 typedef void *(*ErtsStaticNifInitFPtr)(void);
 ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name);
 int erts_is_static_nif(void *handle);
+void erts_init_static_drivers(void);
 
 /* erl_drv_thread.c */
 void erl_drv_thr_init(void);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c1e66b59af..fb39c18d9d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2765,6 +2765,7 @@ void erts_init_io(int port_tab_size,
     init_driver(&fd_driver, &fd_driver_entry, NULL);
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
     init_driver(&spawn_driver, &spawn_driver_entry, NULL);
+    erts_init_static_drivers();
     for (dp = driver_tab; *dp != NULL; dp++)
 	erts_add_driver_entry(*dp, NULL, 1);
 
-- 
cgit v1.2.3


From e897846b560002f58edaae543cf28f36e3edfaf4 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Fri, 23 Aug 2013 12:06:27 +0200
Subject: erts: Fix print out of acul option in crash dump

---
 erts/emulator/beam/erl_alloc_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index bf8a37c71b..e6d9f83aed 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4369,7 +4369,7 @@ info_options(Allctr_t *allctr,
 #endif
 		   "option lmbcs: %beu\n"
 		   "option smbcs: %beu\n"
-		   "option mbcgs: %beu\n",
+		   "option mbcgs: %beu\n"
 		   "option acul: %d\n",
 		   topt,
 		   allctr->ramv ? "true" : "false",
-- 
cgit v1.2.3


From 6d1f1d43aa0a9e0a2cb275ef760339c49518fc69 Mon Sep 17 00:00:00 2001
From: Patrik Nyblom <pan@erlang.org>
Date: Fri, 16 Aug 2013 17:14:25 +0200
Subject: Create better distribution of files over async threads

The actual port id is used to create a key from the
pointer value which is the ErlDrvPort. To do this
a new driver api function driver_async_port_key is
added and the driver API minor version is updated.

The documentation is updated and the faulty description of
how to spread ports over async threads is updated to
use the new API.

Testcase also added.
---
 erts/emulator/beam/erl_async.c  | 14 ++++++++++++++
 erts/emulator/beam/erl_driver.h |  4 +++-
 2 files changed, 17 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index 054d1a48f6..e6d72f569b 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -582,6 +582,20 @@ int erts_async_ready_clean(void *varq, void *val)
 
 #endif
 
+/*
+** Generate a fair async key prom an ErlDrvPort
+** The port data gives a fair distribution grom port pointer
+** to unsigned integer - to be used in key for driver_async below.
+*/
+unsigned int driver_async_port_key(ErlDrvPort port)
+{
+    ErlDrvTermData td = driver_mk_port(port);
+    if (td == (ErlDrvTermData) NIL) {
+	return 0;
+    }
+    return (unsigned int) (UWord) internal_port_data(td);
+}
+
 /*
 ** Schedule async_invoke on a worker thread
 ** NOTE will be syncrounous when threads are unsupported
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index e280563de1..1ab6e17f56 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -133,7 +133,7 @@ typedef struct {
 
 #define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
 #define ERL_DRV_EXTENDED_MAJOR_VERSION	2
-#define ERL_DRV_EXTENDED_MINOR_VERSION	1
+#define ERL_DRV_EXTENDED_MINOR_VERSION	2
 
 /*
  * The emulator will refuse to load a driver with different major
@@ -638,6 +638,8 @@ EXTERN int erl_drv_send_term(ErlDrvTermData port,
 			     int len);
 
 /* Async IO functions */
+EXTERN unsigned int driver_async_port_key(ErlDrvPort port);
+
 EXTERN long driver_async(ErlDrvPort ix,
 			 unsigned int* key,
 			 void (*async_invoke)(void*), 
-- 
cgit v1.2.3


From b368fb879676fa6dd2d251ba6583d0a86d04aeb0 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Tue, 27 Aug 2013 12:24:30 +0200
Subject: erts: Fix segfault when scheduling release_port later_op

begin_port_cleanup can be called from a non-scheduler thread, this
means that there is no esdp available and thus erts_schedule_thr_prgr_later_op
segfaults.
---
 erts/emulator/beam/erl_port_task.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 7d53ce7152..547a42beb2 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1838,6 +1838,16 @@ release_port(void *vport)
 {
     erts_port_dec_refc((Port *) vport);
 }
+
+static void
+schedule_release_port(void *vport) {
+  Port *pp = (Port*)vport;
+  /* This is only used when a port release was ordered from a non-scheduler */
+  erts_schedule_thr_prgr_later_op(release_port,
+				  (void *) pp,
+				  &pp->common.u.release);
+}
+
 #endif
 
 static void
@@ -2033,10 +2043,15 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
      * Schedule cleanup of port structure...
      */
 #ifdef ERTS_SMP
-    /* Has to be more or less immediate to release any driver */
-    erts_schedule_thr_prgr_later_op(release_port,
-				    (void *) pp,
-				    &pp->common.u.release);
+    /* We might not be a scheduler, eg. traceing to port we are sys_msg_dispatcher */
+    if (!erts_get_scheduler_data()) {
+      erts_schedule_misc_aux_work(1, schedule_release_port, (void*)pp);
+    } else {
+      /* Has to be more or less immediate to release any driver */
+      erts_schedule_thr_prgr_later_op(release_port,
+				      (void *) pp,
+				      &pp->common.u.release);
+    }
 #else
     pp->cleanup = 1;
 #endif
-- 
cgit v1.2.3


From 43b1f1755b1ca9003d88655bfebe5ca9e46cfbf6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 28 Aug 2013 20:55:16 +0200
Subject: erts: Remove unused constant DRIVER_TAB_SIZE

---
 erts/emulator/beam/global.h | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ef34f5b221..e3a0487aeb 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -186,11 +186,6 @@ extern void erts_ddll_remove_monitor(Process *p,
 extern Eterm erts_ddll_monitor_driver(Process *p,
 				      Eterm description,
 				      ErtsProcLocks plocks);
-/*
- * Max no. of drivers (linked in and dynamically loaded). Each table
- * entry uses 4 bytes.
- */
-#define DRIVER_TAB_SIZE 32
 
 /*
 ** Just like the driver binary but with initial flags
-- 
cgit v1.2.3


From 69e01c0f5d332d34b95575203ae9d7829b9a45fa Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Tue, 20 Aug 2013 21:09:47 -0400
Subject: add erl option to set schedulers by percentages

For applications where measurements show enhanced performance from the use
of a non-default number of emulator scheduler threads, having to accurately
set the right number of scheduler threads across multiple hosts each with
different numbers of logical processors is difficult because the erl +S
option requires absolute numbers of scheduler threads and scheduler threads
online to be specified.

To address this issue, add a +SP option to erl, similar to the existing +S
option but allowing the number of scheduler threads and scheduler threads
online to be set as percentages of logical processors configured and
logical processors available, respectively. For example, "+SP 50:25" sets
the number of scheduler threads to 50% of the logical processors
configured, and the number of scheduler threads online to 25% of the
logical processors available. The +SP option also interacts with any
settings specified with the +S option, such that the combination of options
"+S 4:4 +SP 50:50" (in either order) results in 2 scheduler threads and 2
scheduler threads online.

Add documentation for the +SP option.

Add tests for the +SP option to scheduler_SUITE.

Add tests and documentation for two existing features of the +S option: +S
0:0 resets the scheduler thread count and scheduler threads online count to
their defaults, and specifying negative numbers for +S results in those
values being subtracted from the default values for the host.
---
 erts/emulator/beam/erl_init.c | 169 ++++++++++++++++++++++++++++++------------
 1 file changed, 121 insertions(+), 48 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 8d137df7ae..f7bfa91b54 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -549,9 +549,12 @@ void erts_usage(void)
 		 ERTS_SCHED_THREAD_MAX_STACK_SIZE);
     erts_fprintf(stderr, "-spp Bool   set port parallelism scheduling hint\n");
     erts_fprintf(stderr, "-S n1:n2    set number of schedulers (n1), and number of\n");
-    erts_fprintf(stderr, "            schedulers online (n2), valid range for both\n");
-    erts_fprintf(stderr, "            numbers are [1-%d]\n",
+    erts_fprintf(stderr, "            schedulers online (n2), maximum for both\n");
+    erts_fprintf(stderr, "            numbers is %d\n",
 		 ERTS_MAX_NO_OF_SCHEDULERS);
+    erts_fprintf(stderr, "-SP p1:p2   specify schedulers (p1) and schedulers online (p2)\n");
+    erts_fprintf(stderr, "	      as percentages of logical processors configured and logical\n");
+    erts_fprintf(stderr, "	      processors available, respectively\n");
     erts_fprintf(stderr, "-t size     set the maximum number of atoms the "
 			 "emulator can handle\n");
     erts_fprintf(stderr, "            valid range is [%d-%d]\n",
@@ -631,6 +634,8 @@ early_init(int *argc, char **argv) /*
     int ncpuavail;
     int schdlrs;
     int schdlrs_onln;
+    int schdlrs_percentage = 100;
+    int schdlrs_onln_percentage = 100;
     int max_main_threads;
     int max_reader_groups;
     int reader_groups;
@@ -758,63 +763,128 @@ early_init(int *argc, char **argv) /*
 		    }
 		    break;
 		}
-		case 'S' : {
-		    int tot, onln;
-		    char *arg = get_arg(argv[i]+2, argv[i+1], &i);
-		    switch (sscanf(arg, "%d:%d", &tot, &onln)) {
-		    case 0:
-			switch (sscanf(arg, ":%d", &onln)) {
+		case 'S' :
+		    if (argv[i][2] == 'P') {
+			int ptot, ponln;
+			char *arg = get_arg(argv[i]+3, argv[i+1], &i);
+			switch (sscanf(arg, "%d:%d", &ptot, &ponln)) {
+			case 0:
+			    switch (sscanf(arg, ":%d", &ponln)) {
+			    case 1:
+				if (ponln < 0)
+				    goto bad_SP;
+				ptot = 100;
+				goto chk_SP;
+			    default:
+				goto bad_SP;
+			    }
 			case 1:
-			    tot = no_schedulers;
-			    goto chk_S;
+			    if (ptot < 0)
+				goto bad_SP;
+			    ponln = ptot < 100 ? ptot : 100;
+			    goto chk_SP;
+			case 2:
+			    if (ptot < 0 || ponln < 0)
+				goto bad_SP;
+			chk_SP:
+			    schdlrs_percentage = ptot;
+			    schdlrs_onln_percentage = ponln;
+			    break;
 			default:
-			    goto bad_S;
-			}
-		    case 1:
-			onln = tot < schdlrs_onln ? tot : schdlrs_onln;
-		    case 2:
-		    chk_S:
-			if (tot > 0)
-			    schdlrs = tot;
-			else
-			    schdlrs = no_schedulers + tot;
-			if (onln > 0)
-			    schdlrs_onln = onln;
-			else
-			    schdlrs_onln = no_schedulers_online + onln;
-			if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) {
-			    erts_fprintf(stderr,
-					 "bad amount of schedulers %d\n",
-					 tot);
-			    erts_usage();
-			}
-			if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) {
+                        bad_SP:
+                            erts_fprintf(stderr,
+                                         "bad schedulers percentage specifier %s\n",
+                                         arg);
+                            erts_usage();
+                            break;
+                        }
+
+                        VERBOSE(DEBUG_SYSTEM,
+                                ("using %d:%d scheduler percentages\n",
+                                 schdlrs_percentage, schdlrs_onln_percentage));
+                    } else {
+			int tot, onln;
+			char *arg = get_arg(argv[i]+2, argv[i+1], &i);
+			switch (sscanf(arg, "%d:%d", &tot, &onln)) {
+			case 0:
+			    switch (sscanf(arg, ":%d", &onln)) {
+			    case 1:
+				tot = no_schedulers;
+				goto chk_S;
+			    default:
+				goto bad_S;
+			    }
+			case 1:
+			    onln = tot < schdlrs_onln ? tot : schdlrs_onln;
+			case 2:
+			chk_S:
+			    if (tot > 0)
+				schdlrs = tot;
+			    else
+				schdlrs = no_schedulers + tot;
+			    if (onln > 0)
+				schdlrs_onln = onln;
+			    else
+				schdlrs_onln = no_schedulers_online + onln;
+			    if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) {
+				erts_fprintf(stderr,
+					     "bad amount of schedulers %d\n",
+					     tot);
+				erts_usage();
+			    }
+			    if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) {
+				erts_fprintf(stderr,
+					     "bad amount of schedulers online %d "
+					     "(total amount of schedulers %d)\n",
+					     schdlrs_onln, schdlrs);
+				erts_usage();
+			    }
+			    break;
+			default:
+			bad_S:
 			    erts_fprintf(stderr,
-					 "bad amount of schedulers online %d "
-					 "(total amount of schedulers %d)\n",
-					 schdlrs_onln, schdlrs);
+					 "bad amount of schedulers %s\n",
+					 arg);
 			    erts_usage();
+			    break;
 			}
-			break;
-		    default:
-		    bad_S:
-			erts_fprintf(stderr,
-				     "bad amount of schedulers %s\n",
-				     arg);
-			erts_usage();
-			break;
-		    }
 
-		    VERBOSE(DEBUG_SYSTEM,
-			    ("using %d:%d scheduler(s)\n", tot, onln));
-		    break;
-		}
+			VERBOSE(DEBUG_SYSTEM,
+				("using %d:%d scheduler(s)\n", tot, onln));
+		    }
+                    break;
 		default:
 		    break;
 		}
 	    }
 	    i++;
 	}
+
+#ifdef ERTS_SMP
+	/* apply any scheduler percentages */
+	if (schdlrs_percentage != 100 || schdlrs_onln_percentage != 100) {
+	    schdlrs = schdlrs * schdlrs_percentage / 100;
+	    schdlrs_onln = schdlrs_onln * schdlrs_onln_percentage / 100;
+	    if (schdlrs < 1)
+                schdlrs = 1;
+            if (ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) {
+		erts_fprintf(stderr,
+			     "bad schedulers percentage %d "
+			     "(total amount of schedulers %d)\n",
+			     schdlrs_percentage, schdlrs);
+		erts_usage();
+	    }
+	    if (schdlrs_onln < 1)
+                schdlrs_onln = 1;
+            if (schdlrs < schdlrs_onln) {
+		erts_fprintf(stderr,
+			     "bad schedulers online percentage %d "
+			     "(total amount of schedulers %d, online %d)\n",
+			     schdlrs_onln_percentage, schdlrs, schdlrs_onln);
+		erts_usage();
+	    }
+	}
+#endif
     }
 
 #ifndef USE_THREADS
@@ -1312,7 +1382,10 @@ erl_start(int argc, char **argv)
 	    break;
 
 	case 'S' : /* Was handled in early_init() just read past it */
-	    (void) get_arg(argv[i]+2, argv[i+1], &i);
+	    if (argv[i][2] == 'P')
+		(void) get_arg(argv[i]+3, argv[i+1], &i);
+	    else
+		(void) get_arg(argv[i]+2, argv[i+1], &i);
 	    break;
 
 	case 's' : {
-- 
cgit v1.2.3


From 06ab9f03fef5883a39591edf9406274a63606724 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 2 Sep 2013 16:27:26 +0200
Subject: Silence gcc warnings in non-smp build

---
 erts/emulator/beam/erl_init.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f7bfa91b54..8c4fffa75b 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -884,6 +884,10 @@ early_init(int *argc, char **argv) /*
 		erts_usage();
 	    }
 	}
+#else
+	/* Silence gcc warnings */
+	(void)schdlrs_percentage;
+	(void)schdlrs_onln_percentage;
 #endif
     }
 
-- 
cgit v1.2.3


From 3e71527e639101c9482b98ebde1c3d98a0419ec5 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Tue, 10 Sep 2013 09:06:13 -0400
Subject: fix system_flag(scheduling_statistics,disable)

Clear the "enabled" flag for scheduling statistics when disable is
specified.
---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 2439a46ab6..434d5ca147 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7237,7 +7237,7 @@ erts_sched_stat_modify(int what)
 	break;
     case ERTS_SCHED_STAT_MODIFY_DISABLE:
 	erts_smp_thr_progress_block();
-	erts_sched_stat.enabled = 1;
+	erts_sched_stat.enabled = 0;
 	erts_smp_thr_progress_unblock();
 	break;
     case ERTS_SCHED_STAT_MODIFY_CLEAR:
-- 
cgit v1.2.3


From 21095e6830f37676dd29c33a590851ba2c76499b Mon Sep 17 00:00:00 2001
From: Pierre Fenoll <pierrefenoll@gmail.com>
Date: Tue, 10 Sep 2013 15:48:29 +0100
Subject: Remove ^L characters hidden randomly in the code. Not those used in
 text files as delimiters.

While working on a tool that processes Erlang code and testing it against this repo,
I found out about those little sneaky 0xff. I thought it may be of help to other
people build such tools to remove non-conforming-to-standard characters.
---
 erts/emulator/beam/beam_emu.c  |  3 ---
 erts/emulator/beam/beam_load.c | 16 ----------------
 2 files changed, 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index da36c4437e..78ab6fa30f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5654,7 +5654,6 @@ build_stacktrace(Process* c_p, Eterm exc) {
     return res;
 }
 
-
 static BeamInstr*
 call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
 {
@@ -5702,7 +5701,6 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
     return ep->addressv[erts_active_code_ix()];
 }
 
-
 static Export*
 apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg)
 {
@@ -6208,7 +6206,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     return make_fun(funp);
 }
 
-
 
 int catchlevel(Process *p)
 {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 4193eb4f3f..938fd8f2c9 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -535,7 +535,6 @@ static int must_swap_floats;
 Uint erts_total_code_size;
 /**********************************************************************/
 
-
 void init_load(void)
 {
     FloatDef f;
@@ -1209,7 +1208,6 @@ verify_chunks(LoaderState* stp)
     return 0;
 }
 
-
 static int
 load_atom_table(LoaderState* stp)
 {
@@ -1255,7 +1253,6 @@ load_atom_table(LoaderState* stp)
     return 0;
 }
 
-
 static int
 load_import_table(LoaderState* stp)
 {
@@ -1308,7 +1305,6 @@ load_import_table(LoaderState* stp)
     return 0;
 }
 
-
 static int
 read_export_table(LoaderState* stp)
 {
@@ -1641,7 +1637,6 @@ read_line_table(LoaderState* stp)
     return 0;
 }
 
-
 static int
 read_code_header(LoaderState* stp)
 {
@@ -1711,7 +1706,6 @@ read_code_header(LoaderState* stp)
     return 0;
 }
 
-
 #define VerifyTag(Stp, Actual, Expected) \
     if (Actual != Expected) { \
        LoadError2(Stp, "bad tag %d; expected %d", Actual, Expected); \
@@ -1730,7 +1724,6 @@ read_code_header(LoaderState* stp)
     
 #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
-
 static int
 load_code(LoaderState* stp)
 {
@@ -2512,7 +2505,6 @@ load_code(LoaderState* stp)
     return retval;
 }
 
-
 #define succ(St, X, Y) ((X).type == (Y).type && (X).val + 1 == (Y).val)
 #define succ2(St, X, Y) ((X).type == (Y).type && (X).val + 2 == (Y).val)
 #define succ3(St, X, Y) ((X).type == (Y).type && (X).val + 3 == (Y).val)
@@ -3958,7 +3950,6 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
 }
 
 
-
 /*
  * Freeze the code in memory, move the string table into place,
  * resolve all labels.
@@ -4276,7 +4267,6 @@ freeze_code(LoaderState* stp)
     return 0;
 }
 
-
 static void
 final_touch(LoaderState* stp)
 {
@@ -4378,7 +4368,6 @@ final_touch(LoaderState* stp)
     }
 }
 
-
 static int
 transform_engine(LoaderState* st)
 {
@@ -4716,7 +4705,6 @@ transform_engine(LoaderState* st)
     return rval;
 }
 
-
 static void
 short_file(int line, LoaderState* stp, unsigned needed)
 {
@@ -4724,7 +4712,6 @@ short_file(int line, LoaderState* stp, unsigned needed)
 		stp->file_name, needed);
 }
 
-
 static void
 load_printf(int line, LoaderState* context, char *fmt,...)
 {
@@ -5190,7 +5177,6 @@ native_addresses(Process* p, Eterm mod)
     return result;
 }
 
-
 /*
  * Builds a list of all exported functions in the given module:
  *     [{Name, Arity},...]
@@ -5240,7 +5226,6 @@ exported_from_module(Process* p, /* Process whose heap to use. */
     return result;
 }
 
-
 /*
  * Returns a list of all attributes for the module.
  *
@@ -5281,7 +5266,6 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
     return result;
 }
 
-
 /*
  * Returns a list containing compilation information.
  *
-- 
cgit v1.2.3


From 439d7e1b81b77e3930d81a6ea9bbadb3cff30fa4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Sep 2013 18:48:18 +0200
Subject: erts: Fix loading of NIF library with unicode in path

---
 erts/emulator/beam/erl_nif.c | 20 +++++++-------------
 erts/emulator/beam/global.h  |  3 +--
 erts/emulator/beam/utils.c   | 22 ----------------------
 3 files changed, 8 insertions(+), 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 673012a9af..ee480fb661 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1568,9 +1568,10 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     void* init_func = NULL;
     ErlNifEntry* entry = NULL;
     ErlNifEnv env;
-    int len, i, err;
+    int i, err;
     Module* mod;
     Eterm mod_atom;
+    const Atom* mod_atomp;
     Eterm f_atom;
     BeamInstr* caller;
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
@@ -1578,21 +1579,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     int veto;
     struct erl_module_nif* lib = NULL;
     int reload_warning = 0;
-    char tmp_buf[255];
 
-    len = list_length(BIF_ARG_1);
-    if (len < 0) {
+    lib_name = erts_convert_filename_to_native(BIF_ARG_1, NULL, 0,
+                                               ERTS_ALC_T_TMP, 1, 0, NULL);
+    if (!lib_name) {
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1);
-
-    if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) {
-	erts_free(ERTS_ALC_T_TMP, lib_name);
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    lib_name[len] = '\0';
-
     if (!erts_try_seize_code_write_permission(BIF_P)) {
 	erts_free(ERTS_ALC_T_TMP, lib_name);
 	ERTS_BIF_YIELD2(bif_export[BIF_load_nif_2],
@@ -1615,7 +1608,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     mod=erts_get_module(mod_atom, erts_active_code_ix());
     ASSERT(mod != NULL);
 
-    init_func = erts_static_nif_get_nif_init(erts_basename(lib_name,tmp_buf));
+    mod_atomp = atom_tab(atom_val(mod_atom));
+    init_func = erts_static_nif_get_nif_init(mod_atomp->name, mod_atomp->len);
     if (init_func != NULL)
       handle = init_func;
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index e3a0487aeb..11e464b639 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -849,7 +849,7 @@ void erts_lcnt_enable_io_lock_count(int enable);
 
 /* driver_tab.c */
 typedef void *(*ErtsStaticNifInitFPtr)(void);
-ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name);
+ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len);
 int erts_is_static_nif(void *handle);
 void erts_init_static_drivers(void);
 
@@ -858,7 +858,6 @@ void erl_drv_thr_init(void);
 
 /* utils.c */
 void erts_cleanup_offheap(ErlOffHeap *offheap);
-const char *erts_basename(const char* path, char* buff);
 
 Uint64 erts_timestamp_millis(void);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 6293286c75..43720084d1 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -4019,28 +4019,6 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 #endif
 }
 
-const char *erts_basename(const char* path, char* buff) {
-  /* This function is not compliant with bash basename. Edge cases like "//"
-     and "/path//" do not work properly.
-   */
-  int i;
-  int len = strlen(path);
-  const char *basename = path;
-  for (i = 0; path[i] != '\0'; i++) {
-    if (path[i] == '/') {
-      if (path[i+1] == '\0') {
-	    memcpy(buff,basename,len - (basename-path));
-	    buff[len - (basename-path)-1] = '\0';
-	    basename = buff;
-	    break;
-      } else { basename = path+i;}
-    }
-  }
-  if (basename == path)
-    return path;
-  return basename+1;
-}
-
 /*
  * A millisecond timestamp without time correction where there's no hrtime
  * - for tracing on "long" things...
-- 
cgit v1.2.3


From 51a15e3a712c2a28234be3a267d80af9e839c31c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 16 Sep 2013 13:39:05 +0200
Subject: erts: Remove unit in 'Time left' printout

The unit 'ms' is removed due to ease of parsing.
---
 erts/emulator/beam/erl_bif_timer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index d67695e533..03ac97283c 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -616,7 +616,7 @@ erts_print_bif_timer_info(int to, void *to_arg)
 			      : btm->receiver.proc.ess->common.id);
 	    erts_print(to, to_arg, "=timer:%T\n", receiver);
 	    erts_print(to, to_arg, "Message: %T\n", btm->message);
-	    erts_print(to, to_arg, "Time left: %u ms\n",
+	    erts_print(to, to_arg, "Time left: %u\n",
 		       erts_time_left(&btm->tm));
 	}
     }
-- 
cgit v1.2.3


From 683281e2362ab950810141647fc7ce3654ffc4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 16 Sep 2013 13:42:03 +0200
Subject: erts: Remove space in 'Buckets' printout

Fix to ease parsing of information.
---
 erts/emulator/beam/erl_db_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 2fea4671e1..06dac8f161 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2106,7 +2106,7 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl)
     DbTableHash *tb = &tbl->hash;
     int i;
 
-    erts_print(to, to_arg, "Buckets: %d \n", NACTIVE(tb));
+    erts_print(to, to_arg, "Buckets: %d\n", NACTIVE(tb));
     
     if (show) {
 	for (i = 0; i < NACTIVE(tb); i++) {
-- 
cgit v1.2.3


From 138737feb3acd4dd5be310bc8f39712adcafc175 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 19 Sep 2013 17:50:39 +0200
Subject: erts: Fix compiler warning

---
 erts/emulator/beam/erl_nif.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ee480fb661..9e2e588161 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1609,7 +1609,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     ASSERT(mod != NULL);
 
     mod_atomp = atom_tab(atom_val(mod_atom));
-    init_func = erts_static_nif_get_nif_init(mod_atomp->name, mod_atomp->len);
+    init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len);
     if (init_func != NULL)
       handle = init_func;
 
-- 
cgit v1.2.3


From f7c9b020f21d57bceddc8596faa275be20625557 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 19 Sep 2013 18:00:01 +0200
Subject: erts: Factor out erts_convert_filename_to_wchar()

from erts_convert_filename_to_encoding()
---
 erts/emulator/beam/erl_unicode.c | 83 ++++++++++++++++++++++++----------------
 erts/emulator/beam/global.h      |  4 ++
 2 files changed, 55 insertions(+), 32 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 8f1b22ff92..a448b600ca 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2023,11 +2023,11 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
     } else if (is_binary(name)) {
 	byte *temp_alloc = NULL;
 	byte *bytes;
-	byte *err_pos;
-	Uint size,num_chars;
+	Uint size;
 	
 	size = binary_size(name);
 	bytes = erts_get_aligned_binary_bytes(name, &temp_alloc);
+
 	if (encoding != ERL_FILENAME_WIN_WCHAR) {
 	    /*Add 0 termination only*/
 	    if (used) 
@@ -2039,36 +2039,11 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	    }
 	    memcpy(name_buf,bytes,size);
 	    name_buf[size]=0;
-	} else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || 
-		   erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
-	    byte *p;
-	    /* What to do now? Maybe latin1, so just take byte for byte instead */
-	    if (used) 
-		*used = (Sint) (size+1)*2;
-	    if ((size+1)*2 > statbuf_size) {
-		name_buf = (char *) erts_alloc(alloc_type, (size+1)*2);
-	    } else {
-		name_buf = statbuf;
-	    }
-	    p = (byte *) name_buf;
-	    while (size--) {
-		*p++ = *bytes++;
-		*p++ = 0;
-	    }
-	    *p++ = 0;
-	    *p++ = 0;
-	} else { /* WIN_WCHAR and valid UTF8 */
-	    if (used) 
-		*used = (Sint) (num_chars+1)*2;
-	    if ((num_chars+1)*2 > statbuf_size) {
-		name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2);
-	    } else {
-		name_buf = statbuf;
-	    }
-	    erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
-	    name_buf[num_chars*2] = 0;
-	    name_buf[num_chars*2+1] = 0;
-	}
+	} else {
+            name_buf = erts_convert_filename_to_wchar(bytes, size,
+                                                      statbuf, statbuf_size,
+                                                      alloc_type, used, 0);
+        }
 	erts_free_aligned_binary_bytes(temp_alloc);
     } else {
 	return NULL;
@@ -2076,6 +2051,50 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
     return name_buf;
 }
 
+char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
+                                     char *statbuf, size_t statbuf_size,
+                                     ErtsAlcType_t alloc_type, Sint* used,
+                                     Uint extra_wchars)
+{
+    byte *err_pos;
+    Uint num_chars;
+    char* name_buf = NULL;
+    Sint need;
+    char *p;
+
+    if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK ||
+        erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+
+        /* What to do now? Maybe latin1, so just take byte for byte instead */
+        need = (Sint) (size + extra_wchars + 1) * 2;
+        if (need > statbuf_size) {
+            name_buf = (char *) erts_alloc(alloc_type, need);
+        } else {
+            name_buf = statbuf;
+        }
+        p = name_buf;
+        while (size--) {
+            *p++ = *bytes++;
+            *p++ = 0;
+        }
+    } else { /* WIN_WCHAR and valid UTF8 */
+        need = (Sint) (num_chars + extra_wchars + 1) * 2;
+        if (need > statbuf_size) {
+            name_buf = (char *) erts_alloc(alloc_type, need);
+        } else {
+            name_buf = statbuf;
+        }
+        erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
+        p = name_buf + num_chars*2;
+    }
+    *p++ = 0;
+    *p++ = 0;
+    if (used)
+        *used = p - name_buf;
+    return name_buf;
+}
+
+
 static int filename_len_16bit(byte *str) 
 {
     byte *p = str;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 11e464b639..c040b259a0 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -918,6 +918,10 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf,
 					int allow_empty, int allow_atom,
 					int encoding,
 					Sint *used /* out */);
+char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
+                                     char *statbuf, size_t statbuf_size,
+                                     ErtsAlcType_t alloc_type, Sint* used,
+                                     Uint extra_wchars);
 Eterm erts_convert_native_to_filename(Process *p, byte *bytes);
 Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left,
 			Uint *num_built, Uint *num_eaten, Eterm tail);
-- 
cgit v1.2.3


From 6fea2c436a6f4a501be632e9bb7453570c09fb8e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 19 Sep 2013 19:13:13 +0200
Subject: erts, crypto: Support NIF library with unicode filename on windows

---
 erts/emulator/beam/erl_nif.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 9e2e588161..e87959f0ab 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1568,7 +1568,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     void* init_func = NULL;
     ErlNifEntry* entry = NULL;
     ErlNifEnv env;
-    int i, err;
+    int i, err, encoding;
     Module* mod;
     Eterm mod_atom;
     const Atom* mod_atomp;
@@ -1580,8 +1580,14 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     struct erl_module_nif* lib = NULL;
     int reload_warning = 0;
 
-    lib_name = erts_convert_filename_to_native(BIF_ARG_1, NULL, 0,
-                                               ERTS_ALC_T_TMP, 1, 0, NULL);
+    encoding = erts_get_native_filename_encoding();
+    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+        /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */
+        /* since lib_name is used in error messages */
+        encoding = ERL_FILENAME_UTF8;
+    }
+    lib_name = erts_convert_filename_to_encoding(BIF_ARG_1, NULL, 0,
+                                                 ERTS_ALC_T_TMP, 1, 0, encoding, NULL);
     if (!lib_name) {
 	BIF_ERROR(BIF_P, BADARG);
     }
-- 
cgit v1.2.3


From 17f6fd613e8b7039526fbb063c6751fc0c2008c3 Mon Sep 17 00:00:00 2001
From: Mike Sassak <mike.sassak@rackspace.com>
Date: Fri, 13 Sep 2013 16:37:50 -0700
Subject: Check all pattern arguments passed to binary:matches/2

Including an empty binary as one of multiple patterns passed to
binary:matches/2 would crash BEAM with: "Cannot reallocate
1342177280 bytes of memory (of type "tmp")". This ensures each pattern
is valid before trying to match.
---
 erts/emulator/beam/erl_bif_binary.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 0db19a1ee6..ff775691b3 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -927,6 +927,9 @@ static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp)
 	    if (binary_bitsize(b) != 0) {
 		goto badarg;
 	    }
+	    if (binary_size(b) == 0) {
+		goto badarg;
+	    }
 	    ++words;
 	    characters += binary_size(b);
 	}
-- 
cgit v1.2.3


From 75a79e6547fd66c2194d6f488c30ad888a715f4b Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Wed, 4 Sep 2013 10:21:47 -0400
Subject: add system_info(ets_limit)

Add system_info(ets_limit) to provide a way to retrieve the runtime's
maximum number of ETS tables. Add tests and documentation for it too.

Also repair the alphabetical order of system_info/1 argument descriptions
in the documentation and in the erlang.erl clauses. Add new preloaded
erlang.erl due to that change. Also ensure all system_info/1 clauses are
represented in the erlang.xml source documentation -- a couple had been
inadvertently dropped in previous commits when other clauses were added.
---
 erts/emulator/beam/erl_bif_info.c | 3 +++
 erts/emulator/beam/erl_db.c       | 7 +++++++
 erts/emulator/beam/erl_db.h       | 2 ++
 3 files changed, 12 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 673dfc658c..3b25efd9af 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2636,6 +2636,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 
 	BIF_RET(res);
     }
+    else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) {
+        BIF_RET(make_small(erts_db_get_max_tabs()));
+    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 98c2988323..40b8eaf8fb 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3811,6 +3811,13 @@ erts_db_foreach_offheap(DbTable *tb,
     tb->common.meth->db_foreach_offheap(tb, func, arg);
 }
 
+/* retrieve max number of ets tables */
+Uint
+erts_db_get_max_tabs()
+{
+    return db_max_tabs;
+}
+
 /*
  * For testing of meta tables only.
  *
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 6b62e10eb7..5b4681fc90 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -79,6 +79,8 @@ extern erts_smp_atomic_t erts_ets_misc_mem_size;
 
 Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
 
+Uint erts_db_get_max_tabs(void);
+
 #endif
 
 #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__)
-- 
cgit v1.2.3


From b6b0b73ecec7facefb3b9c5a7ef663599cfee4aa Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 20 Sep 2013 20:13:40 +0200
Subject: erts: Fix bug in atom to filename conversions

Buggy old code assumed latin1 atoms.
---
 erts/emulator/beam/erl_unicode.c | 59 ++++++++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index a448b600ca..ec8ea5f044 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2174,16 +2174,31 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding)
 	ap = atom_tab(atom_val(ioterm));
 	switch (encoding) {
 	case ERL_FILENAME_LATIN1:
-	    need = ap->len;
+	    need = ap->latin1_chars;  /* May be -1 */
 	    break;
 	case ERL_FILENAME_UTF8_MAC:
 	case ERL_FILENAME_UTF8:
-	    for (i = 0; i < ap->len; i++) {
-		need += (ap->name[i] >= 0x80) ? 2 : 1;
-	    }
+	    need = ap->len;
 	    break;
 	case ERL_FILENAME_WIN_WCHAR:
-	    need = 2*(ap->len);
+            if (ap->latin1_chars >= 0) {
+		need = 2* ap->latin1_chars;
+            }
+	    else {
+		for (i = 0; i < ap->len; ) {
+                    if (ap->name[i] < 0x80) {
+			i++;
+                    } else if (ap->name[i] < 0xE0) {
+			i += 2;
+                    } else if (ap->name[i] < 0xF0) {
+			i += 3;
+                    } else {
+			need = -1;
+			break;
+		    }
+		    need += 2;
+		}
+	    }
 	    break;
 	default:
 	    need = -1;
@@ -2313,26 +2328,36 @@ void erts_native_filename_put(Eterm ioterm, int encoding, byte *p)
 	switch (encoding) {
 	case ERL_FILENAME_LATIN1:
 	    for (i = 0; i < ap->len; i++) {
-		*p++ = ap->name[i];
-	    }
-	    break;
-	case ERL_FILENAME_UTF8_MAC:
-	case ERL_FILENAME_UTF8:
-	    for (i = 0; i < ap->len; i++) {
-		if(ap->name[i] < 0x80) {
+		if (ap->name[i] < 0x80) {
 		    *p++ = ap->name[i];
 		} else {
-		    *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0));
-		    *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80));
+		    ASSERT(ap->name[i] < 0xC4);
+		    *p++ = ((ap->name[i] & 3) << 6) | (ap->name[i+1] & 0x3F);
+		    i++;
 		}
 	    }
 	    break;
+	case ERL_FILENAME_UTF8_MAC:
+	case ERL_FILENAME_UTF8:
+	    sys_memcpy(p, ap->name, ap->len);
+	    break;
 	case ERL_FILENAME_WIN_WCHAR:
 	    for (i = 0; i < ap->len; i++) {
 		/* Little endian */
-		*p++ = ap->name[i];
-		*p++ = 0;
-	    }
+                if (ap->name[i] < 0x80) {
+		    *p++ = ap->name[i];
+		    *p++ = 0;
+                } else if (ap->name[i] < 0xE0) {
+		    *p++ = ((ap->name[i] & 3) << 6) | (ap->name[i+1] & 0x3F);
+		    *p++ = ((ap->name[i] & 0x1C) >> 2);
+		    i++;
+                } else {
+		    ASSERT(ap->name[i] < 0xF0);
+		    *p++ = ((ap->name[i+1] & 3) << 6) | (ap->name[i+2] & 0x3C);
+		    *p++ = ((ap->name[i] & 0xF) << 4) | ((ap->name[i+1] & 0x3C) >> 2);
+		    i += 2;
+		}
+            }
 	    break;
 	default:
 	    ASSERT(0);
-- 
cgit v1.2.3


From cabfe2cdc0b8d879431f81233addd2a43ff1f742 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 11 Sep 2013 21:49:32 +0200
Subject: Implement platform specific aligned sys_alloc and use when supported

erts_sys_aligned_alloc() is currently implemented using posix_memalign if
it exist, or using _aligned_malloc on Windows.

If erts_sys_aligned_alloc() exist allocators will create sys_alloc
carriers similar to how this was done pre-R16.
---
 erts/emulator/beam/erl_alloc.c      |  21 ++------
 erts/emulator/beam/erl_alloc.h      |  10 ++++
 erts/emulator/beam/erl_alloc_util.c | 103 ++++++++++++++++++++++--------------
 erts/emulator/beam/erl_alloc_util.h |  27 +++++++---
 erts/emulator/beam/sys.h            |   3 ++
 5 files changed, 99 insertions(+), 65 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d748f86d75..13692edbfc 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -235,7 +235,6 @@ set_default_sbmbc_alloc_opts(struct au_init *ip)
     ip->init.bf.ao		= 1;
     ip->init.util.ramv		= 0;
     ip->init.util.mmsbc		= 0;
-    ip->init.util.mmmbc		= 500;
     ip->init.util.sbct		= ~((UWord) 0);
     ip->init.util.name_prefix	= "sbmbc_";
     ip->init.util.alloc_no	= ERTS_ALC_A_SBMBC;
@@ -261,7 +260,6 @@ set_default_sl_alloc_opts(struct au_init *ip)
     ip->thr_spec		= 1;
     ip->atype			= GOODFIT;
     ip->init.util.name_prefix	= "sl_";
-    ip->init.util.mmmbc		= 5;
     ip->init.util.alloc_no	= ERTS_ALC_A_SHORT_LIVED;
 #ifndef SMALL_MEMORY
     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
@@ -285,7 +283,6 @@ set_default_std_alloc_opts(struct au_init *ip)
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
     ip->init.util.name_prefix	= "std_";
-    ip->init.util.mmmbc		= 5;
     ip->init.util.alloc_no	= ERTS_ALC_A_STANDARD;
 #ifndef SMALL_MEMORY
     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
@@ -305,7 +302,6 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.bf.ao		= 1;
     ip->init.util.ramv		= 0;
     ip->init.util.mmsbc		= 0;
-    ip->init.util.mmmbc		= 0;
     ip->init.util.sbct		= ~((UWord) 0);
     ip->init.util.name_prefix	= "ll_";
     ip->init.util.alloc_no	= ERTS_ALC_A_LONG_LIVED;
@@ -353,7 +349,6 @@ set_default_eheap_alloc_opts(struct au_init *ip)
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
     ip->thr_spec		= 1;
     ip->atype			= GOODFIT;
-    ip->init.util.mmmbc		= 100;
     ip->init.util.name_prefix	= "eheap_";
     ip->init.util.alloc_no	= ERTS_ALC_A_EHEAP;
 #ifndef SMALL_MEMORY
@@ -376,7 +371,6 @@ set_default_binary_alloc_opts(struct au_init *ip)
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
-    ip->init.util.mmmbc		= 50;
     ip->init.util.name_prefix	= "binary_";
     ip->init.util.alloc_no	= ERTS_ALC_A_BINARY;
 #ifndef SMALL_MEMORY
@@ -394,7 +388,6 @@ set_default_ets_alloc_opts(struct au_init *ip)
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
-    ip->init.util.mmmbc		= 100;
     ip->init.util.name_prefix	= "ets_";
     ip->init.util.alloc_no	= ERTS_ALC_A_ETS;
 #ifndef SMALL_MEMORY
@@ -462,13 +455,6 @@ adjust_tpref(struct au_init *ip, int no_sched)
 	/* ... shrink smallest multi-block carrier size */
 	if (ip->default_.smbcs)
 	    ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
-	/* ... and more than three allocators shrink
-	   max mseg multi-block carriers */
-	if (ip->default_.mmmbc && no_sched > 2) {
-	    ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1);
-	    if (ip->init.util.mmmbc < 3)
-		ip->init.util.mmmbc = 3;
-	}
     }
 }
 
@@ -2609,8 +2595,8 @@ erts_allocator_options(void *proc)
 #endif
     Uint sz, *szp, *hp, **hpp;
     Eterm res, features, settings;
-    Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
-    Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
+    Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6];
+    Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6];
     int a, length;
     SysAllocStat sas;
     Uint *endp = NULL;
@@ -2723,6 +2709,9 @@ erts_allocator_options(void *proc)
     if (use_mseg)
 	terms[length++] = am_atom_put("mseg_alloc", 10);
 #endif
+#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    terms[length++] = am_atom_put("sys_aligned_alloc", 17);
+#endif
 
     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
 
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index ba5ec9c367..7168241f76 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -54,6 +54,16 @@ void erts_sys_alloc_init(void);
 void *erts_sys_alloc(ErtsAlcType_t, void *, Uint);
 void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint);
 void erts_sys_free(ErtsAlcType_t, void *, void *);
+#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+/*
+ * Note 'alignment' must remain the same in calls to
+ * 'erts_sys_aligned_realloc()' and 'erts_sys_aligned_free()'
+ * as in the initial call to 'erts_sys_aligned_alloc()'.
+ */
+void *erts_sys_aligned_alloc(UWord alignment, UWord size);
+void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size);
+void erts_sys_aligned_free(UWord alignment, void *ptr);
+#endif
 
 Eterm erts_memory(int *, void *, void *, Eterm);
 Eterm erts_allocated_areas(int *, void *, void *);
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index ac7f420708..24f297dc58 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -76,14 +76,14 @@ static int initialized = 0;
 
 int erts_have_sbmbc_alloc;
 
-#if HAVE_ERTS_MSEG
+#if ERTS_SA_MB_CARRIERS
 
-#define MSEG_UNIT_SHIFT         MSEG_ALIGN_BITS
-#define MSEG_UNIT_SZ            (1 << MSEG_UNIT_SHIFT)
-#define MSEG_UNIT_MASK		((~(UWord)0) << MSEG_UNIT_SHIFT)
+#define ERTS_SACRR_UNIT_SHIFT		ERTS_SUPER_ALIGN_BITS
+#define ERTS_SACRR_UNIT_SZ		(1 << ERTS_SACRR_UNIT_SHIFT)
+#define ERTS_SACRR_UNIT_MASK		((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT)
 
-#define MSEG_UNIT_FLOOR(X)	((X) & MSEG_UNIT_MASK)
-#define MSEG_UNIT_CEILING(X)	MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK)
+#define ERTS_SACRR_UNIT_FLOOR(X)	((X) & ERTS_SACRR_UNIT_MASK)
+#define ERTS_SACRR_UNIT_CEILING(X)	ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK)
 
 #endif
 
@@ -208,9 +208,9 @@ MBC after deallocating first block:
 
 #if MBC_ABLK_OFFSET_BITS
 
-#  define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS)
+#  define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << ERTS_SUPER_ALIGN_BITS)
 
-#  define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT)
+#  define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> ERTS_SACRR_UNIT_SHIFT)
 
 #  define SET_MBC_ABLK_HDR(B, Sz, F, C) \
     (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \
@@ -225,8 +225,8 @@ MBC after deallocating first block:
 
 #  define ABLK_TO_MBC(B) \
     (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \
-     (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \
-		  (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT))))
+     (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \
+		  (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT))))
 
 #  define FBLK_TO_MBC(B) \
     (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \
@@ -235,7 +235,7 @@ MBC after deallocating first block:
 #  define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
 
 #  define IS_MBC_FIRST_ABLK(AP,B) \
-  ((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \
+  ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \
    && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0)
 
 #  define IS_MBC_FIRST_FBLK(AP,B) \
@@ -652,11 +652,15 @@ alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
 #endif
 
 static ERTS_INLINE void *
-alcu_sys_alloc(Allctr_t *allctr, Uint size)
+alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
 {
     void *res;
-
-    res = erts_sys_alloc(0, NULL, size);
+#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    if (superalign)
+	res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size);
+    else
+#endif
+	res = erts_sys_alloc(0, NULL, size);
     INC_CC(allctr->calls.sys_alloc);
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_alloc(res, allctr->alloc_no, ERTS_ALC_A_SYSTEM, size);
@@ -664,11 +668,16 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size)
 }
 
 static ERTS_INLINE void *
-alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size)
+alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
 {
     void *res;
 
-    res = erts_sys_realloc(0, NULL, ptr, size);
+#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    if (superalign)
+	res = erts_sys_aligned_realloc(ERTS_SACRR_UNIT_SZ, ptr, size, old_size);
+    else
+#endif
+	res = erts_sys_realloc(0, NULL, ptr, size);
     INC_CC(allctr->calls.sys_realloc);
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_realloc(res,
@@ -680,9 +689,14 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size)
 }
 
 static ERTS_INLINE void
-alcu_sys_free(Allctr_t *allctr, void *ptr)
+alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign)
 {
-    erts_sys_free(0, NULL, ptr);
+#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    if (superalign)
+	erts_sys_aligned_free(ERTS_SACRR_UNIT_SZ, ptr);
+    else
+#endif
+	erts_sys_free(0, NULL, ptr);
     INC_CC(allctr->calls.sys_free);
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr);
@@ -1321,7 +1335,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp)
 	    blk = create_sbmbc(allctr, get_blk_sz);
 	else {
 	    blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
-#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY
 	    if (!blk) {
 		/* Emergency! We couldn't create the carrier as we wanted.
 		   Try to place it in a sys_alloced sbc. */
@@ -1927,10 +1941,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 
 #ifdef DEBUG
 
-#if HAVE_ERTS_MSEG
-#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % MSEG_UNIT_SZ == 0)
+#if ERTS_SA_MB_CARRIERS
+#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % ERTS_SACRR_UNIT_SZ == 0)
 #else
-#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ)
+#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ)
 #endif
 
 static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
@@ -1955,7 +1969,7 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
     }
     if ((MSEGED)) {
 	ASSERT(IS_MSEG_CARRIER((C)));
-	ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ));
+	ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ));
     }
     else {
 	ASSERT(IS_SYS_ALLOC_CARRIER((C)));
@@ -2058,7 +2072,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 
 #if HALFWORD_HEAP
     flags |= CFLG_FORCE_MSEG;
-#elif HAVE_SUPER_ALIGNED_MB_CARRIERS
+#elif ERTS_SUPER_ALIGNED_MSEG_ONLY
     if (flags & CFLG_MBC) {
 	flags |= CFLG_FORCE_MSEG;
     }
@@ -2083,7 +2097,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs)
 	    goto try_sys_alloc;
     }
-#if !HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if !ERTS_SUPER_ALIGNED_MSEG_ONLY
     else {
 	if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs)
 	    goto try_sys_alloc;
@@ -2146,12 +2160,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	      ? UNIT_CEILING(bcrr_sz)
 	      : SYS_ALLOC_CARRIER_CEILING(bcrr_sz));
 
-    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz);
+    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
 	
     if (!crr) {
 	if (crr_sz > UNIT_CEILING(bcrr_sz)) {
 	    crr_sz = UNIT_CEILING(bcrr_sz);
-	    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz);
+	    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
 	}
 	if (!crr) {
 #if HAVE_ERTS_MSEG
@@ -2245,7 +2259,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 	if (!(flags & CFLG_FORCE_SYS_ALLOC)) {
 
 	    new_crr_sz = new_blk_sz + SBC_HEADER_SIZE;
-	    new_crr_sz = MSEG_UNIT_CEILING(new_crr_sz);
+	    new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz);
 	    new_crr = (Carrier_t *) alcu_mseg_realloc(allctr,
 						      old_crr,
 						      old_crr_sz,
@@ -2295,7 +2309,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 
 	    new_crr = (Carrier_t *) alcu_sys_realloc(allctr,
 						     (void *) old_crr,
-						     new_crr_sz);
+						     new_crr_sz,
+						     old_crr_sz,
+						     0);
 	    if (new_crr) {
 	    sys_realloc_success:
 		SET_CARRIER_SZ(new_crr, new_crr_sz);
@@ -2314,7 +2330,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		new_crr_sz = UNIT_CEILING(new_crr_sz);
 		new_crr = (Carrier_t *) alcu_sys_realloc(allctr,
 							 (void *) old_crr,
-							 new_crr_sz);
+							 new_crr_sz,
+							 old_crr_sz,
+							 0);
 		if (new_crr)
 		    goto sys_realloc_success;
 	    }
@@ -2333,7 +2351,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		       (void *) BLK2UMEM(old_blk),
 		       MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ);
 	    unlink_carrier(&allctr->sbc_list, old_crr);
-	    alcu_sys_free(allctr, old_crr);
+	    alcu_sys_free(allctr, old_crr, 0);
 	}
 	else {
 	    /* Old carrier unchanged; restore... */
@@ -2350,6 +2368,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 {
     Uint crr_sz;
     Carrier_t *crr;
+    int is_mbc;
 #if HAVE_ERTS_MSEG
     Uint is_mseg = 0;
     Uint mseg_flags = ERTS_MSEG_FLG_NONE;
@@ -2357,6 +2376,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 
     if (IS_SBC_BLK(blk)) {
 	Uint blk_sz = SBC_BLK_SZ(blk);
+	is_mbc = 0;
 	crr = BLK_TO_SBC(blk);
 	crr_sz = CARRIER_SZ(crr);
 
@@ -2367,7 +2387,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(crr)) {
 	    is_mseg++;
-	    ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
+	    ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0);
 	    STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz);
 	}
 	else
@@ -2378,6 +2398,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 
     }
     else {
+	is_mbc = 1;
 	ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
 	crr = FIRST_BLK_TO_MBC(allctr, blk);
 	crr_sz = CARRIER_SZ(crr);
@@ -2397,7 +2418,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(crr)) {
 	    is_mseg++;
-	    ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
+	    ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0);
 	    STAT_MSEG_MBC_FREE(allctr, crr_sz);
 	    mseg_flags = ERTS_MSEG_FLG_2POW;
 	}
@@ -2417,7 +2438,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
     }
     else
 #endif
-	alcu_sys_free(allctr, crr);
+	alcu_sys_free(allctr, crr, is_mbc);
 }
 
 
@@ -3833,7 +3854,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 		crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz);
 #if HAVE_ERTS_MSEG
 	    else
-		crr_sz = MSEG_UNIT_CEILING(used_sz);
+		crr_sz = ERTS_SACRR_UNIT_CEILING(used_sz);
 #endif
 	    diff_sz_val = crr_sz - used_sz;
 	    if (diff_sz_val < (~((Uint) 0) / 100))
@@ -4175,7 +4196,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     allctr->mbc_move_threshold		= init->rmbcmt;
 #if HAVE_ERTS_MSEG
     allctr->max_mseg_sbcs		= init->mmsbc;
-# if HAVE_SUPER_ALIGNED_MB_CARRIERS
+# if ERTS_SUPER_ALIGNED_MSEG_ONLY
     allctr->max_mseg_mbcs		= ~(Uint)0;
 # else
     allctr->max_mseg_mbcs		= init->mmmbc;
@@ -4300,7 +4321,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 			     allctr->main_carrier_size,
 			     CFLG_MBC
 			     | CFLG_FORCE_SIZE
-#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY
 			     | CFLG_FORCE_SYS_ALLOC
 #endif
 			     | CFLG_MAIN_CARRIER);
@@ -4369,9 +4390,9 @@ erts_alcu_init(AlcUInit_t *init)
 {
     ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
 #if HAVE_ERTS_MSEG
-    ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ);
+    ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ);
     max_mseg_carriers = init->mmc;
-    sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs);
+    sys_alloc_carrier_size = ERTS_SACRR_UNIT_CEILING(init->ycs);
 #else /* #if HAVE_ERTS_MSEG */
     sys_alloc_carrier_size = ((init->ycs + 4095) / 4096) * 4096;
 #endif
@@ -4493,7 +4514,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk));
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(sbc)) {
-	    ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0);
+	    ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0);
 	}
 #endif
 	crr = sbc;
@@ -4578,7 +4599,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(crr)) {
-	    ASSERT(CARRIER_SZ(crr) % MSEG_UNIT_SZ == 0);
+	    ASSERT(CARRIER_SZ(crr) % ERTS_SACRR_UNIT_SZ == 0);
 	}
 #endif
 	cl = &allctr->mbc_list;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index e0754e7f69..e975dc34a7 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -76,7 +76,7 @@ typedef struct {
 
 #define ERTS_DEFAULT_ALCU_INIT {                                           \
     1024*1024,		/* (bytes)  ycs:    sys_alloc carrier size       */\
-    1024      		/* (amount) mmc:    max mseg carriers            */\
+    ~((UWord) 0)	/* (amount) mmc:    max mseg carriers            */ \
 }
 
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
@@ -96,7 +96,7 @@ typedef struct {
     50,			/* (%)      rmbcmt: rel mbc move threshold       */\
     1024*1024,		/* (bytes)  mmbcs:  main multiblock carrier size */\
     256,		/* (amount) mmsbc:  max mseg sbcs                */\
-    10,			/* (amount) mmmbc:  max mseg mbcs                */\
+    ~((UWord) 0),	/* (amount) mmmbc:  max mseg mbcs                */ \
     10*1024*1024,	/* (bytes)  lmbcs:  largest mbc size             */\
     1024*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
     10,			/* (amount) mbcgs:  mbc growth stages            */\
@@ -130,7 +130,7 @@ typedef struct {
     80,			/* (%)      rsbcmt: rel sbc move threshold       */\
     128*1024,		/* (bytes)  mmbcs:  main multiblock carrier size */\
     256,		/* (amount) mmsbc:  max mseg sbcs                */\
-    10,			/* (amount) mmmbc:  max mseg mbcs                */\
+    ~((UWord) 0),	/* (amount) mmmbc:  max mseg mbcs                */ \
     1024*1024,		/* (bytes)  lmbcs:  largest mbc size             */\
     128*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
     10,			/* (amount) mbcgs:  mbc growth stages            */\
@@ -221,18 +221,29 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 #define MBC_FBLK_SZ_MASK        UNIT_MASK
 #define CARRIER_SZ_MASK         UNIT_MASK
 
-
-#if HAVE_ERTS_MSEG
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED \
+    || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC)
+#  ifndef MSEG_ALIGN_BITS
+#    define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS
+#  else
+#    define ERTS_SUPER_ALIGN_BITS 18
+#  endif
 #  ifdef ARCH_64 
 #    define MBC_ABLK_OFFSET_BITS   24
-#  elif HAVE_SUPER_ALIGNED_MB_CARRIERS
+#  else
 #    define MBC_ABLK_OFFSET_BITS   9
      /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
 #  endif
-#endif
-#ifndef MBC_ABLK_OFFSET_BITS
+#  define ERTS_SA_MB_CARRIERS 1
+#else
+#  define ERTS_SA_MB_CARRIERS 0
 #  define MBC_ABLK_OFFSET_BITS   0 /* no carrier offset in block header */
 #endif
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+#  define ERTS_SUPER_ALIGNED_MSEG_ONLY 1
+#else
+#  define ERTS_SUPER_ALIGNED_MSEG_ONLY 0
+#endif
 
 #if MBC_ABLK_OFFSET_BITS
 #  define MBC_ABLK_OFFSET_SHIFT  (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS)
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 05bff430e3..096394b878 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1012,6 +1012,9 @@ void erl_bin_write(unsigned char *, int, int);
 #define ERTS_SMALL_ABS(Small) labs(Small) 
 #endif
 
+#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
+#endif
 
 #ifdef __WIN32__
 void call_break_handler(void);
-- 
cgit v1.2.3


From 156b011958a3b80e507039ddc916db039874ada1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 27 Aug 2013 17:23:09 +0200
Subject: erts: Refactor the ASSERT macro

Introduce unconditional ERTS_ASSERT
and use that for both ASSERT and ASSERT_EXPR.
---
 erts/emulator/beam/dist.c                  |  4 ++--
 erts/emulator/beam/erl_alloc_util.c        |  3 ---
 erts/emulator/beam/erl_ao_firstfit_alloc.c |  3 ---
 erts/emulator/beam/erl_bestfit_alloc.c     |  3 ---
 erts/emulator/beam/erl_bif_info.c          |  2 +-
 erts/emulator/beam/erl_binary.h            |  4 ++--
 erts/emulator/beam/erl_db.c                |  4 ++--
 erts/emulator/beam/io.c                    |  2 +-
 erts/emulator/beam/sys.h                   | 16 ++++++----------
 9 files changed, 14 insertions(+), 27 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 44f4eb9d43..aabccac822 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -353,7 +353,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp)
 static void doit_link_net_exits(ErtsLink *lnk, void *vnecp)
 {
     LinkNetExitsContext lnec = {(NetExitsContext *) vnecp, lnk};
-    ASSERT(lnk->type == LINK_PID)
+    ASSERT(lnk->type == LINK_PID);
     erts_sweep_links(ERTS_LINK_ROOT(lnk), &doit_link_net_exits_sub, (void *) &lnec);
 #ifdef DEBUG
     ERTS_LINK_ROOT(lnk) = NULL;
@@ -369,7 +369,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
     Process *rp;
     ErtsLink *rlnk;
     Uint i,n;
-    ASSERT(lnk->type == LINK_NODE)
+    ASSERT(lnk->type == LINK_NODE);
     if (is_internal_pid(lnk->pid)) {
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
 	rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks);
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 825b68bb85..3ea74a12f9 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -87,9 +87,6 @@ static int initialized = 0;
 #define SYS_ALLOC_CARRIER_CEILING(X) \
   SYS_ALLOC_CARRIER_FLOOR((X) + INV_SYS_ALLOC_CARRIER_MASK)
 
-#undef ASSERT
-#define ASSERT ASSERT_EXPR
-
 #if 0
 /* Can be useful for debugging */
 #define MBC_REALLOC_ALWAYS_MOVES
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 4e6c8b317e..396aa88e0b 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -85,9 +85,6 @@
 #define SET_RED(N)		(((AOFF_RBTree_t *) (N))->flags |= RED_FLG)
 #define SET_BLACK(N)		(((AOFF_RBTree_t *) (N))->flags &= ~RED_FLG)
 
-#undef ASSERT
-#define ASSERT ASSERT_EXPR
-
 #if 1
 #define RBT_ASSERT	ASSERT
 #else
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 41f449bb28..59c14899a2 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -75,9 +75,6 @@
 
 #define BF_BLK_SZ(B)            MBC_FBLK_SZ(&(B)->hdr)
 
-#undef ASSERT
-#define ASSERT ASSERT_EXPR
-
 #if 1
 #define RBT_ASSERT	ASSERT
 #else
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 3b25efd9af..a4f9f787cd 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2091,7 +2091,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_sequential_tracer) {
 	val = erts_get_system_seq_tracer();
-	ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false)
+	ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false);
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, am_sequential_tracer, val);
 	BIF_RET(res);
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 506c4813fa..331a12dc1c 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -183,7 +183,7 @@ BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen);
 #endif
 
 #define ERTS_CHK_BIN_ALIGNMENT(B) \
-  do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)) } while(0)
+  do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)); } while(0)
 
 ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr);
 ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 40b8eaf8fb..41e64fcd4f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2236,7 +2236,7 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1)
     CHECK_TABLES();
 
     tptr = tuple_val(a1);
-    ASSERT(arityval(*tptr) >= 1)
+    ASSERT(arityval(*tptr) >= 1);
 
     if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) {
 	BIF_ERROR(p, BADARG);
@@ -2403,7 +2403,7 @@ static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1)
     CHECK_TABLES();
 
     tptr = tuple_val(a1);
-    ASSERT(arityval(*tptr) >= 1)
+    ASSERT(arityval(*tptr) >= 1);
     if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) {
 	BIF_ERROR(p, BADARG);
     }
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c1e66b59af..db19f6c142 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1330,7 +1330,7 @@ force_imm_drv_call(ErtsTryImmDrvCallState *sp)
     erts_aint32_t invalid_state;
     Port *prt = sp->port;
 
-    ASSERT(ERTS_IS_CRASH_DUMPING)
+    ASSERT(ERTS_IS_CRASH_DUMPING);
     ASSERT(is_atom(sp->port_op));
 
     invalid_state = sp->state;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 096394b878..a20106749c 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -149,20 +149,16 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #  define ERTS_EXIT_AFTER_DUMP exit
 #endif
 
-#ifdef DEBUG
-#  define ASSERT(e) \
-  if (e) { \
-     ; \
-  } else { \
-     erl_assert_error(#e, __FILE__, __LINE__); \
-  }
-#  define ASSERT_EXPR(e) \
+#define ERTS_ASSERT(e) \
     ((void) ((e) ? 1 : (erl_assert_error(#e, __FILE__, __LINE__), 0)))
 void erl_assert_error(char* expr, char* file, int line);
+
+#ifdef DEBUG
+#  define ASSERT(e) ERTS_ASSERT(e)
 #else
-#  define ASSERT(e)
-#  define ASSERT_EXPR(e) ((void) 1)
+#  define ASSERT(e) ((void) 1)
 #endif
+#define ASSERT_EXPR ASSERT
 
 /*
  * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
-- 
cgit v1.2.3


From ca1dc60a852c7827c2934ffeacefdd0119e2d776 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 27 Aug 2013 17:36:58 +0200
Subject: erts: Remove ASSERT_EXPR macro

---
 erts/emulator/beam/erl_alloc_util.h           |  6 +++---
 erts/emulator/beam/erl_binary.h               |  2 +-
 erts/emulator/beam/erl_db_util.h              |  2 +-
 erts/emulator/beam/erl_node_container_utils.h | 10 +++++-----
 erts/emulator/beam/erl_process.c              |  2 +-
 erts/emulator/beam/erl_process.h              |  4 ++--
 erts/emulator/beam/erl_vm.h                   | 12 ++++++------
 erts/emulator/beam/external.h                 |  4 ++--
 erts/emulator/beam/global.h                   |  6 +++---
 erts/emulator/beam/sys.h                      |  1 -
 10 files changed, 24 insertions(+), 25 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 70ecf28172..222f137024 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -254,9 +254,9 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 #  define MBC_ABLK_SZ_MASK	(~FLG_MASK)
 #endif
 
-#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
-#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
-#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
+#define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
+#define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
+#define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
 
 #define CARRIER_SZ(C) \
   ((C)->chdr & CARRIER_SZ_MASK)
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 331a12dc1c..f7dc20f5e6 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -153,7 +153,7 @@ do {									\
 #define binary_bytes(Bin)						\
   (*binary_val(Bin) == HEADER_PROC_BIN ?				\
    ((ProcBin *) binary_val(Bin))->bytes :				\
-   (ASSERT_EXPR(thing_subtag(*binary_val(Bin)) == HEAP_BINARY_SUBTAG),	\
+   (ASSERT(thing_subtag(*binary_val(Bin)) == HEAP_BINARY_SUBTAG),	\
    (byte *)(&(((ErlHeapBin *) binary_val(Bin))->data))))
 
 void erts_init_binary(void);
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 90b79e6044..328b19dfc9 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -457,7 +457,7 @@ int erts_db_is_compiled_ms(Eterm term);
    && ERTS_MAGIC_BIN_DESTRUCTOR((BP)) == erts_db_match_prog_destructor)
 
 #define Binary2MatchProg(BP) \
-  (ASSERT_EXPR(IsMatchProgBinary((BP))), \
+  (ASSERT(IsMatchProgBinary((BP))), \
    ((MatchProg *) ERTS_MAGIC_BIN_DATA((BP))))
 /*
 ** Debugging 
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 0f93a3a9f0..17f6b32bb1 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -106,7 +106,7 @@
 #define dist_entry_channel_no(x)				\
   ((x) == erts_this_dist_entry					\
    ? ((Uint) 0)							\
-   : (ASSERT_EXPR(is_atom((x)->sysname)),			\
+   : (ASSERT(is_atom((x)->sysname)),			        \
       (Uint) atom_val((x)->sysname)))
 #define internal_channel_no(x) ((Uint) ERST_INTERNAL_CHANNEL_NO)
 #define external_channel_no(x) \
@@ -122,10 +122,10 @@ extern ErtsPTab erts_proc;
 							  (D), \
 							  _TAG_IMMED1_PID)
 
-#define internal_pid_index(PID)		(ASSERT_EXPR(is_internal_pid((PID))), \
+#define internal_pid_index(PID)		(ASSERT(is_internal_pid((PID))), \
 					 erts_ptab_id2pix(&erts_proc, (PID)))
 
-#define internal_pid_data(PID)		(ASSERT_EXPR(is_internal_pid((PID))), \
+#define internal_pid_data(PID)		(ASSERT(is_internal_pid((PID))), \
 					 erts_ptab_id2data(&erts_proc, (PID)))
 
 #define internal_pid_number(x)		_GET_PID_NUM(internal_pid_data((x)))
@@ -193,10 +193,10 @@ extern ErtsPTab erts_port;
 							  (D), \
 							  _TAG_IMMED1_PORT)
 
-#define internal_port_index(PRT)	(ASSERT_EXPR(is_internal_port((PRT))), \
+#define internal_port_index(PRT)	(ASSERT(is_internal_port((PRT))), \
 					 erts_ptab_id2pix(&erts_port, (PRT)))
 
-#define internal_port_data(PRT)		(ASSERT_EXPR(is_internal_port((PRT))), \
+#define internal_port_data(PRT)		(ASSERT(is_internal_port((PRT))), \
 					 erts_ptab_id2data(&erts_port, (PRT)))
 
 #define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x)))
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 434d5ca147..79f382674a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -294,7 +294,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
 				 ERTS_ALC_T_PROC_LIST)
 
 #define ERTS_SCHED_SLEEP_INFO_IX(IX)					\
-    (ASSERT_EXPR(-1 <= ((int) (IX))					\
+    (ASSERT(-1 <= ((int) (IX))					        \
 		 && ((int) (IX)) < ((int) erts_no_schedulers)),		\
      &aligned_sched_sleep_info[(IX)].ssi)
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8e5467f196..8d136f6e8b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1135,10 +1135,10 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
     } while (0)
 
 #define ERTS_RUNQ_IX(IX)						\
-  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues),			\
+  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
    &erts_aligned_run_queues[(IX)].runq)
 #define ERTS_SCHEDULER_IX(IX)						\
-  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers),			\
+  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
    &erts_aligned_scheduler_data[(IX)].esd)
 
 void erts_pre_init_process(void);
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index c962955de9..8026243555 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -98,7 +98,7 @@
  * failing that, in a heap fragment.
  */
 #define HAllocX(p, sz, xtra)		                              \
-    (ASSERT_EXPR((sz) >= 0),					      \
+  (ASSERT((sz) >= 0),					              \
      ErtsHAllocLockCheck(p),					      \
      (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \
       ? erts_heap_alloc((p),(sz),(xtra))                              \
@@ -135,14 +135,14 @@
  */
 #ifdef CHECK_FOR_HOLES
 # define HeapOnlyAlloc(p, sz)					\
-    (ASSERT_EXPR((sz) >= 0),					\
-     (ASSERT_EXPR(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))),	\
+    (ASSERT((sz) >= 0),					        \
+     (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))),	        \
       (erts_set_hole_marker(HEAP_TOP(p), (sz)),			\
        (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))))
 #else
 # define HeapOnlyAlloc(p, sz)					\
-    (ASSERT_EXPR((sz) >= 0),					\
-     (ASSERT_EXPR(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))),	\
+    (ASSERT((sz) >= 0),					        \
+     (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))),	        \
       (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))
 #endif
 
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index e37d47919e..ff29e84972 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -138,8 +138,8 @@ typedef struct {
 #define ERTS_DIST_EXT_SIZE(EDEP) \
   (sizeof(ErtsDistExternal) \
    - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \
-      ? (ASSERT_EXPR(0 <= (EDEP)->attab.size \
-		     && (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \
+      ? (ASSERT(0 <= (EDEP)->attab.size \
+		&& (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \
 	 sizeof(Eterm)*(ERTS_ATOM_CACHE_SIZE - (EDEP)->attab.size)) \
       : sizeof(ErtsAtomTranslationTable)))
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bacd5a5752..063d16c0c7 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -866,13 +866,13 @@ Eterm store_external_or_ref_in_proc_(Process *, Eterm);
 Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm);
 
 #define NC_HEAP_SIZE(NC) \
- (ASSERT_EXPR(is_node_container((NC))), \
+ (ASSERT(is_node_container((NC))), \
   IS_CONST((NC)) ? 0 : (thing_arityval(*boxed_val((NC))) + 1))
 #define STORE_NC(Hpp, ETpp, NC) \
- (ASSERT_EXPR(is_node_container((NC))), \
+ (ASSERT(is_node_container((NC))), \
   IS_CONST((NC)) ? (NC) : store_external_or_ref_((Hpp), (ETpp), (NC)))
 #define STORE_NC_IN_PROC(Pp, NC) \
- (ASSERT_EXPR(is_node_container((NC))), \
+ (ASSERT(is_node_container((NC))), \
   IS_CONST((NC)) ? (NC) : store_external_or_ref_in_proc_((Pp), (NC)))
 
 /* duplicates from big.h */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index a20106749c..97e6ed8410 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -158,7 +158,6 @@ void erl_assert_error(char* expr, char* file, int line);
 #else
 #  define ASSERT(e) ((void) 1)
 #endif
-#define ASSERT_EXPR ASSERT
 
 /*
  * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
-- 
cgit v1.2.3


From c2dbcb69929ac18e7687f1df1de6613b34e2897b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 30 Aug 2013 11:59:49 +0200
Subject: erts: Prepare erl_mmap with tree structures for free seg storage

---
 erts/emulator/beam/erl_alloc.c | 37 +++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/sys.h       |  6 ++++++
 2 files changed, 43 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 6dec383cee..2babe2f416 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1174,6 +1174,25 @@ get_kb_value(char *param_end, char** argv, int* ip)
 	return ((Uint) tmp)*1024;
 }
 
+static UWord
+get_mb_value(char *param_end, char** argv, int* ip)
+{
+    SWord tmp;
+    UWord max = ((~((Uint) 0))/(1024*1024)) + 1;
+    char *rest;
+    char *param = argv[*ip]+1;
+    char *value = get_value(param_end, argv, ip);
+    errno = 0;
+    tmp = (SWord) ErtsStrToSint(value, &rest, 10);
+    if (errno != 0 || rest == value || tmp < 0 || max < ((UWord) tmp))
+	bad_value(param, param_end, value);
+    if (max == (UWord) tmp)
+	return ~((UWord) 0);
+    else
+	return ((UWord) tmp)*1024*1024;
+}
+
+
 #if 0
 static Uint
 get_byte_value(char *param_end, char** argv, int* ip)
@@ -1448,6 +1467,24 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 #endif
 			    get_amount_value(argv[i]+6, argv, &i);
 		    }
+		    else if (has_prefix("scs", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+			init->mseg.mmap.scs =
+#endif
+			    get_mb_value(argv[i]+6, argv, &i);
+		    }
+		    else if (has_prefix("sco", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+			init->mseg.mmap.sco =
+#endif
+			    get_bool_value(argv[i]+6, argv, &i);
+		    }
+		    else if (has_prefix("scmgc", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+			init->mseg.mmap.scmgc =
+#endif
+			    get_amount_value(argv[i]+8, argv, &i);
+		    }
 		    else {
 			bad_param(param, param+2);
 		    }
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 97e6ed8410..dfe60d8ea0 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -277,16 +277,19 @@ typedef unsigned long UWord;
 typedef long          SWord;
 #define SWORD_CONSTANT(Const) Const##L
 #define UWORD_CONSTANT(Const) Const##UL
+#define ERTS_SWORD_MAX LONG_MAX
 #elif SIZEOF_VOID_P == SIZEOF_INT
 typedef unsigned int UWord;
 typedef int          SWord;
 #define SWORD_CONSTANT(Const) Const
 #define UWORD_CONSTANT(Const) Const##U
+#define ERTS_SWORD_MAX INT_MAX
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
 typedef unsigned long long UWord;
 typedef long long          SWord;
 #define SWORD_CONSTANT(Const) Const##LL
 #define UWORD_CONSTANT(Const) Const##ULL
+#define ERTS_SWORD_MAX LLONG_MAX
 #else
 #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
 #endif
@@ -299,6 +302,7 @@ typedef unsigned long Uint;
 typedef long          Sint;
 #define SWORD_CONSTANT(Const) Const##L
 #define UWORD_CONSTANT(Const) Const##UL
+#define ERTS_SWORD_MAX LONG_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_INT
@@ -307,6 +311,7 @@ typedef unsigned int Uint;
 typedef int          Sint;
 #define SWORD_CONSTANT(Const) Const
 #define UWORD_CONSTANT(Const) Const##U
+#define ERTS_SWORD_MAX INT_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
@@ -315,6 +320,7 @@ typedef unsigned long long Uint;
 typedef long long          Sint;
 #define SWORD_CONSTANT(Const) Const##LL
 #define UWORD_CONSTANT(Const) Const##ULL
+#define ERTS_SWORD_MAX LLONG_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
 #if defined(__WIN32__)
 #define ErtsStrToSint _strtoi64
-- 
cgit v1.2.3


From ef3da907bd566b43a4022f1cbb1ae3d103b9ec3e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 23 Aug 2013 17:29:58 +0200
Subject: erts: erts_mmap supercarrier management and erts_mseg usage

* Coalescing and trimming of free segments in supercarrier
* Management of super aligned and super unaligned areas in
  supercarrier
* Management of reservation of physical memory
* erts_mseg usage of erts_mmap
---
 erts/emulator/beam/erl_alloc.c      |  9 ++++++++-
 erts/emulator/beam/erl_alloc_util.c | 17 ++++++++++-------
 erts/emulator/beam/erl_lock_check.c |  3 ++-
 erts/emulator/beam/erl_unicode.c    |  3 +++
 erts/emulator/beam/erl_vm.h         |  2 +-
 5 files changed, 24 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 2babe2f416..e30b3e7b51 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -718,6 +718,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     init.mseg.nos = erts_no_schedulers;
     erts_mseg_init(&init.mseg);
 #endif
+
     erts_alcu_init(&init.alloc_util);
     erts_afalc_init();
     erts_bfalc_init();
@@ -1178,7 +1179,7 @@ static UWord
 get_mb_value(char *param_end, char** argv, int* ip)
 {
     SWord tmp;
-    UWord max = ((~((Uint) 0))/(1024*1024)) + 1;
+    UWord max = ((~((UWord) 0))/(1024*1024)) + 1;
     char *rest;
     char *param = argv[*ip]+1;
     char *value = get_value(param_end, argv, ip);
@@ -1479,6 +1480,12 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 #endif
 			    get_bool_value(argv[i]+6, argv, &i);
 		    }
+		    else if (has_prefix("scrpm", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+			init->mseg.mmap.scrpm =
+#endif
+			    get_bool_value(argv[i]+8, argv, &i);
+		    }
 		    else if (has_prefix("scmgc", argv[i]+3)) {
 #if HAVE_ERTS_MSEG
 			init->mseg.mmap.scmgc =
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 3ea74a12f9..1fdee4db2c 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -756,8 +756,9 @@ static ERTS_INLINE void *
 alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
 {
     void *res;
-
-    res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, flags, &allctr->mseg_opt);
+    UWord size = (UWord) *size_p;
+    res = erts_mseg_alloc_opt(allctr->alloc_no, &size, flags, &allctr->mseg_opt);
+    *size_p = (Uint) size;
     INC_CC(allctr->calls.mseg_alloc);
     return res;
 }
@@ -766,9 +767,10 @@ static ERTS_INLINE void *
 alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p)
 {
     void *res;
-
-    res = erts_mseg_realloc_opt(allctr->alloc_no, seg, old_size, new_size_p,
+    UWord new_size = (UWord) *new_size_p;
+    res = erts_mseg_realloc_opt(allctr->alloc_no, seg, (UWord) old_size, &new_size,
 				ERTS_MSEG_FLG_NONE, &allctr->mseg_opt);
+    *new_size_p = (Uint) new_size;
     INC_CC(allctr->calls.mseg_realloc);
     return res;
 }
@@ -776,7 +778,7 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p)
 static ERTS_INLINE void
 alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
 {
-    erts_mseg_dealloc_opt(allctr->alloc_no, seg, size, flags, &allctr->mseg_opt);
+    erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt);
     INC_CC(allctr->calls.mseg_dealloc);
 }
 
@@ -3223,10 +3225,12 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
 	ASSERT(IS_MBC_BLK((B)));
 	ASSERT(IS_MB_CARRIER((C)));
 	ASSERT(FBLK_TO_MBC(B) == (C));
+	if ((MSEGED)) {
+	    ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ));
+	}
     }
     if ((MSEGED)) {
 	ASSERT(IS_MSEG_CARRIER((C)));
-	ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ));
     }
     else {
 	ASSERT(IS_SYS_ALLOC_CARRIER((C)));
@@ -3598,7 +3602,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(crr)) {
-	    ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0);
 	    STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz);
 	}
 	else
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 2114d0c001..1e9cef3759 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -185,7 +185,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {   "sys_gethrtime",                        NULL                    },
 #endif
 #endif
-    {	"erts_alloc_hard_debug",		NULL			}
+    {	"erts_alloc_hard_debug",		NULL			},
+    {	"erts_mmap",				NULL			}
 };
 
 #define ERTS_LOCK_ORDER_SIZE \
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index e00440b905..569c0a7d31 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1476,6 +1476,9 @@ static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint s
     Uint16 savepoints[4];
     int numpoints = 0;
 
+    if (num == 0)
+	return NIL;
+
     ASSERT(num > 0);
 
     hp = HAlloc(p,num * 2); /* May be to much */
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 8026243555..337422eead 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -80,7 +80,7 @@
 #  ifdef CHECK_FOR_HOLES
 #    define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz))
 #  else
-#    define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),DEBUG_BAD_BYTE,(sz)*sizeof(Eterm*))
+#    define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*))
 #  endif
 #else
 #  define INIT_HEAP_MEM(p,sz) ((void)0)
-- 
cgit v1.2.3


From e0e17136506f9c8363b46a991012422980925dd1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 4 Sep 2013 14:56:49 +0200
Subject: erts: Add __func__ to ERTS_ASSERT macro

---
 erts/emulator/beam/sys.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index dfe60d8ea0..9561c0be96 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -150,8 +150,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #endif
 
 #define ERTS_ASSERT(e) \
-    ((void) ((e) ? 1 : (erl_assert_error(#e, __FILE__, __LINE__), 0)))
-void erl_assert_error(char* expr, char* file, int line);
+    ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))
+void erl_assert_error(const char* expr, const char *func, const char* file, int line);
 
 #ifdef DEBUG
 #  define ASSERT(e) ERTS_ASSERT(e)
-- 
cgit v1.2.3


From 0820017c421bfab27d23aff4da474974f988006c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 5 Sep 2013 19:23:07 +0200
Subject: erts: Add mmap argument to erts_debug:get_internal_state

---
 erts/emulator/beam/erl_bif_info.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a4f9f787cd..7aa439f2e6 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3289,6 +3289,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    erts_smp_thr_progress_unblock();
 	    BIF_RET(res);
 	}
+        else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) {
+            BIF_RET(erts_mmap_info(BIF_P));
+        }
     }
     else if (is_tuple(BIF_ARG_1)) {
 	Eterm* tp = tuple_val(BIF_ARG_1);
-- 
cgit v1.2.3


From b9e82ba0be1364c64e90274d5e9bf37f78b676ee Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Sep 2013 13:31:05 +0200
Subject: erts: Add HARD_DBG_MSEG

---
 erts/emulator/beam/erl_lock_check.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 1e9cef3759..87efbdbc3e 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -186,6 +186,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #endif
 #endif
     {	"erts_alloc_hard_debug",		NULL			},
+    {	"hard_dbg_mseg",		        NULL	                },
     {	"erts_mmap",				NULL			}
 };
 
-- 
cgit v1.2.3


From d6d531012957f8e3315d44c2bcb10938ab5d6e72 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 25 Sep 2013 17:23:55 +0200
Subject: erts: Rename erts_bld_atom_uint_2tup_list to *_uword_*

and change from Uint to UWord values
---
 erts/emulator/beam/erl_trace.c | 38 +++++++++++++++++++-------------------
 erts/emulator/beam/erl_utils.h |  6 +++---
 erts/emulator/beam/utils.c     |  4 ++--
 3 files changed, 24 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index fa015ee4b9..ff7fdfcfca 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2184,7 +2184,7 @@ trace_gc(Process *p, Eterm what)
 	AM_bin_old_vheap_block_size
     };
 
-    Uint values[] = {
+    UWord values[] = {
 	OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
 	HEAP_SIZE(p),
 	MBUF_SIZE(p),
@@ -2198,7 +2198,7 @@ trace_gc(Process *p, Eterm what)
 	BIN_OLD_VHEAP_SZ(p)
     };
 #define LOCAL_HEAP_SIZE						\
-    (sizeof(values)/sizeof(Eterm)) *				\
+    (sizeof(values)/sizeof(*values)) *				\
 	(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) +	\
 	5/*4-tuple */ + TS_HEAP_WORDS
     DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
@@ -2206,7 +2206,7 @@ trace_gc(Process *p, Eterm what)
     Eterm* limit;
 #endif
 
-    ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm));
+    ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm));
 
     UseTmpHeap(LOCAL_HEAP_SIZE,p);
 
@@ -2214,9 +2214,9 @@ trace_gc(Process *p, Eterm what)
 	hp = local_heap;
 #ifdef DEBUG
 	size = 0;
-	(void) erts_bld_atom_uint_2tup_list(NULL,
+	(void) erts_bld_atom_uword_2tup_list(NULL,
 					    &size,
-					    sizeof(values)/sizeof(Uint),
+					    sizeof(values)/sizeof(*values),
 					    tags,
 					    values);
 	size += 5/*4-tuple*/ + TS_SIZE(p);
@@ -2229,9 +2229,9 @@ trace_gc(Process *p, Eterm what)
 			    ERTS_TRACE_FLAGS(p));
 
 	size = 0;
-	(void) erts_bld_atom_uint_2tup_list(NULL,
+	(void) erts_bld_atom_uword_2tup_list(NULL,
 					    &size,
-					    sizeof(values)/sizeof(Uint),
+					    sizeof(values)/sizeof(*values),
 					    tags,
 					    values);
 	size += 5/*4-tuple*/ + TS_SIZE(p);
@@ -2244,9 +2244,9 @@ trace_gc(Process *p, Eterm what)
     ASSERT(size <= LOCAL_HEAP_SIZE);
 #endif
 
-    msg = erts_bld_atom_uint_2tup_list(&hp,
+    msg = erts_bld_atom_uword_2tup_list(&hp,
 				       NULL,
-				       sizeof(values)/sizeof(Uint),
+				       sizeof(values)/sizeof(*values),
 				       tags,
 				       values);
 
@@ -2415,7 +2415,7 @@ monitor_long_gc(Process *p, Uint time) {
 	am_old_heap_size,
 	am_heap_size
     };
-    Eterm values[] = {
+    UWord values[] = {
 	time,
 	OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
 	HEAP_SIZE(p),
@@ -2436,9 +2436,9 @@ monitor_long_gc(Process *p, Uint time) {
 #endif
 
     hsz = 0;
-    (void) erts_bld_atom_uint_2tup_list(NULL,
+    (void) erts_bld_atom_uword_2tup_list(NULL,
 					&hsz,
-					sizeof(values)/sizeof(Uint),
+					sizeof(values)/sizeof(*values),
 					tags,
 					values);
     hsz += 5 /* 4-tuple */;
@@ -2449,9 +2449,9 @@ monitor_long_gc(Process *p, Uint time) {
     hp_end = hp + hsz;
 #endif
 
-    list = erts_bld_atom_uint_2tup_list(&hp,
+    list = erts_bld_atom_uword_2tup_list(&hp,
 					NULL,
-					sizeof(values)/sizeof(Uint),
+					sizeof(values)/sizeof(*values),
 					tags,
 					values);
     msg = TUPLE4(hp, am_monitor, p->common.id, am_long_gc, list); 
@@ -2489,7 +2489,7 @@ monitor_large_heap(Process *p) {
 	am_old_heap_size,
 	am_heap_size
     };
-    Uint values[] = {
+    UWord values[] = {
 	OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
 	HEAP_SIZE(p),
 	MBUF_SIZE(p),
@@ -2511,9 +2511,9 @@ monitor_large_heap(Process *p) {
 #endif
 
     hsz = 0;
-    (void) erts_bld_atom_uint_2tup_list(NULL,
+    (void) erts_bld_atom_uword_2tup_list(NULL,
 					&hsz,
-					sizeof(values)/sizeof(Uint),
+					sizeof(values)/sizeof(*values),
 					tags,
 					values);
     hsz += 5 /* 4-tuple */;
@@ -2524,9 +2524,9 @@ monitor_large_heap(Process *p) {
     hp_end = hp + hsz;
 #endif
 
-    list = erts_bld_atom_uint_2tup_list(&hp,
+    list = erts_bld_atom_uword_2tup_list(&hp,
 					NULL,
-					sizeof(values)/sizeof(Uint),
+					sizeof(values)/sizeof(*values),
 					tags,
 					values);
     msg = TUPLE4(hp, am_monitor, p->common.id, am_large_heap, list); 
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 80d29d554a..f85c7410c4 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2013. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -175,8 +175,8 @@ Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]);
 Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp,
 			 Sint length, Eterm terms1[], Uint terms2[]);
 Eterm
-erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp,
-			     Sint length, Eterm atoms[], Uint uints[]);
+erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp,
+			     Sint length, Eterm atoms[], UWord uints[]);
 Eterm
 erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
 			      Eterm atoms[], Uint uints1[], Uint uints2[]);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index bd2be7afca..0d75bbcc77 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -576,8 +576,8 @@ erts_bld_2tup_list(Uint **hpp, Uint *szp,
 }
 
 Eterm
-erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp,
-			     Sint length, Eterm atoms[], Uint uints[])
+erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp,
+                              Sint length, Eterm atoms[], UWord uints[])
 {
     Sint i;
     Eterm res = THE_NON_VALUE;
-- 
cgit v1.2.3


From 8d5b9a53a1fd5e2264d705911af23cd484ccead0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 26 Sep 2013 22:30:26 +0200
Subject: erts: Add erts_bld_tupleX macros

for compile time argument checking
---
 erts/emulator/beam/erl_alloc.c | 12 ++++++------
 erts/emulator/beam/erl_utils.h |  4 ++++
 2 files changed, 10 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index e30b3e7b51..ad1020d7d6 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3060,13 +3060,13 @@ reply_alloc_info(void *vair)
 			     ? NIL
 			     : erts_mseg_info(0, NULL, NULL, hpp != NULL,
 					      hpp, szp));
-		    ainfo = erts_bld_tuple(hpp, szp, 3,
-					   alloc_atom,
-					   make_small(0),
-					   ainfo);
+		    ainfo = erts_bld_tuple3(hpp, szp,
+                                            alloc_atom,
+                                            make_small(0),
+                                            ainfo);
 #else
-		    ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
-					   am_false);
+		    ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
+                                            am_false);
 #endif
 			break;
 		default:
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index f85c7410c4..292d135946 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -168,6 +168,10 @@ Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64);
 Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64);
 Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr);
 Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...);
+#define erts_bld_tuple2(H,S,E1,E2) erts_bld_tuple(H,S,2,E1,E2)
+#define erts_bld_tuple3(H,S,E1,E2,E3) erts_bld_tuple(H,S,3,E1,E2,E3)
+#define erts_bld_tuple4(H,S,E1,E2,E3,E4) erts_bld_tuple(H,S,4,E1,E2,E3,E4)
+#define erts_bld_tuple5(H,S,E1,E2,E3,E4,E5) erts_bld_tuple(H,S,5,E1,E2,E3,E4,E5)
 Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]);
 Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len);
 #define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str))
-- 
cgit v1.2.3


From 059d8b76011f960cc5938501a33002b051b0bca2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 26 Sep 2013 22:36:34 +0200
Subject: erts: Add erts_mmap stats

As part of erlang:system_info({allocator,mseg_alloc})
and erl_crash.dump
---
 erts/emulator/beam/erl_alloc.c    | 14 +++++++++++++-
 erts/emulator/beam/erl_bif_info.c |  2 +-
 2 files changed, 14 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index ad1020d7d6..d8da616d05 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2712,6 +2712,7 @@ erts_allocator_info(int to, void *arg)
 
 #if HAVE_ERTS_MSEG
     {
+	struct erts_mmap_info_struct emis;
 #ifdef ERTS_SMP
 	int max = (int) erts_no_schedulers;
 #else
@@ -2722,6 +2723,8 @@ erts_allocator_info(int to, void *arg)
 	    erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
 	    erts_mseg_info(i, &to, arg, 0, NULL, NULL);
 	}
+	erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n");
+	erts_mmap_info(&to, arg, NULL, NULL, &emis);
     }
 #endif
 
@@ -2948,6 +2951,7 @@ reply_alloc_info(void *vair)
     Uint sz, *szp;
     ErlOffHeap *ohp = NULL;
     ErlHeapFragment *bp = NULL;
+    struct erts_mmap_info_struct emis;
     int i;
     Eterm (*info_func)(Allctr_t *,
 		       int,
@@ -3064,11 +3068,19 @@ reply_alloc_info(void *vair)
                                             alloc_atom,
                                             make_small(0),
                                             ainfo);
+
+		    ai_list = erts_bld_cons(hpp, szp,
+					    ainfo, ai_list);
+		    ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis));
+		    ainfo = erts_bld_tuple3(hpp, szp,
+                                            alloc_atom,
+                                            erts_bld_atom(hpp,szp,"erts_mmap"),
+                                            ainfo);
 #else
 		    ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
                                             am_false);
 #endif
-			break;
+		    break;
 		default:
 		    alloc_atom = erts_bld_atom(hpp, szp,
 					       (char *) ERTS_ALC_A2AD(ai));
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 7aa439f2e6..5fbcbbe250 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3290,7 +3290,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    BIF_RET(res);
 	}
         else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) {
-            BIF_RET(erts_mmap_info(BIF_P));
+            BIF_RET(erts_mmap_debug_info(BIF_P));
         }
     }
     else if (is_tuple(BIF_ARG_1)) {
-- 
cgit v1.2.3


From ff95e85937007a7952477c7acc7619791405ab1c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 1 Oct 2013 18:12:43 +0200
Subject: erts: Add mutex to init_atoms in erts_mmap.c

---
 erts/emulator/beam/erl_lock_check.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 87efbdbc3e..0dd83fa6ed 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -132,6 +132,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #endif /* __WIN32__ */
     {	"alcu_init_atoms",			NULL			},
     {	"mseg_init_atoms",			NULL			},
+    {	"mmap_init_atoms",			NULL			},
     {	"drv_tsd",				NULL			},
     {	"async_enq_mtx",			NULL			},
 #ifdef ERTS_SMP
-- 
cgit v1.2.3


From 54aecc2a73a1398d141c7f8e9a96c24a5a5731cf Mon Sep 17 00:00:00 2001
From: Lars Hesel Christensen <larshesel@gmail.com>
Date: Tue, 24 Sep 2013 11:22:48 +0200
Subject: Fix bsr bug

Fix bsr bug occurring when shifting a huge number a huge number of
bits to the right. The bug can occur if Sint is 64 bits and int is 32
bits, causing a truncation in the big.c:I_lshift function.
---
 erts/emulator/beam/big.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 6b43c53985..2b27b111d8 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1325,9 +1325,9 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y,
 	return 1;
     }
     else {
-	SWord ay = (y < 0) ? -y : y;
-	int bw = ay / D_EXP;
-	int sw = ay % D_EXP;
+	Uint ay = (y < 0) ? -y : y;
+	Uint bw = ay / D_EXP;
+	Uint sw = ay % D_EXP;
 	dsize_t rl;
 	ErtsDigit a1=0;
 	ErtsDigit a0=0;
@@ -1368,7 +1368,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y,
 	    }
 
 	    if (sign) {
-		int zl = bw;
+		Uint zl = bw;
 		ErtsDigit* z = x;
 
 		while(zl--) {
-- 
cgit v1.2.3


From cf2159e5e43bb989f5a3f5f1b038bfa5ce33057d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 17 Oct 2013 15:10:48 +0200
Subject: erts: Fix memory leak for distributed monitors

---
 erts/emulator/beam/dist.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 44f4eb9d43..95d2c5b362 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1509,12 +1509,12 @@ int erts_net_message(Port *prt,
 	    break;
 	}
 	rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
+
+	erts_destroy_monitor(mon);
 	if (rp == NULL) {
 	    break;
 	}
 
-	erts_destroy_monitor(mon);
-
 	mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
 
 	if (mon == NULL) {
-- 
cgit v1.2.3


From e25f74afd0705f686d0fc949e4362c73d6da15fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 18 Oct 2013 16:47:19 +0200
Subject: erts: Fix segfaulting crashdump writing

Crashdumps initiated by out-of-memory on spawn could cause the beam
to segfault during crashdump writing due to invalid pointers.

The pointers are invalid since the process creation never finished.
This commit remedies this problem by removing the process from
crashdump printout.
---
 erts/emulator/beam/break.c       | 5 ++++-
 erts/emulator/beam/erl_process.c | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index ad9a89b642..99604fa3bc 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -76,7 +76,10 @@ process_info(int to, void *to_arg)
     for (i = 0; i < max; i++) {
 	Process *p = erts_pix2proc(i);
 	if (p && p->i != ENULL) {
-	    if (!ERTS_PROC_IS_EXITING(p))
+	    /* Do not include processes with no heap,
+	     * they are most likely just created and has invalid data
+	     */
+	    if (!ERTS_PROC_IS_EXITING(p) && p->heap != NULL)
 		print_process_info(to, to_arg, p);
 	}
     }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 434d5ca147..5cfaf1b5ee 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7471,6 +7471,7 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
     
     p->approx_started = erts_get_approx_time();
     p->rcount = 0;
+    p->heap = NULL;
 
 
     ASSERT(p == (Process *) (erts_ptab_pix2intptr_nob(
@@ -7583,7 +7584,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     hipe_init_process_smp(&p->hipe_smp);
 #endif
 #endif
-
     p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz);
     p->old_hend = p->old_htop = p->old_heap = NULL;
     p->high_water = p->heap;
-- 
cgit v1.2.3


From a51730d1f0ef5a9825998a9e4c2cd0556de9a18b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 30 Oct 2013 13:20:42 +0100
Subject: Fix truncated pointers in erl_crash.dump

When dumping the heaps and stacks in the erl_crash.dump file, only
the lower 32 bits of pointers would be printed. Ensure that all bits
are printed.

While at it, make sure that printing sub binaries with a debug compiled
run-time system will work.
---
 erts/emulator/beam/break.c            |  2 +-
 erts/emulator/beam/erl_process_dump.c | 46 ++++++++++++++++++++++++++---------
 2 files changed, 35 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 99604fa3bc..b7e1092907 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -757,7 +757,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 	return; /* Can't create the crash dump, skip it */
     
     time(&now);
-    erts_fdprintf(fd, "=erl_crash_dump:0.2\n%s", ctime(&now));
+    erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now));
 
     if (file != NULL)
        erts_fdprintf(fd, "The error occurred in file %s, line %d\n", file, line);
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 6cd0d23b97..2f3cf23b00 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -34,8 +34,8 @@
 #define ERTS_WANT_EXTERNAL_TAGS
 #include "external.h"
 
-#define WORD_FMT "%X"
-#define ADDR_FMT "%X"
+#define PTR_FMT "%bpX"
+#define ETERM_FMT "%beX"
 
 #define OUR_NIL	_make_header(0,_TAG_HEADER_FLOAT)
 
@@ -210,9 +210,9 @@ static void
 dump_element(int to, void *to_arg, Eterm x)
 {
     if (is_list(x)) {
-	erts_print(to, to_arg, "H" WORD_FMT, list_val(x));
+	erts_print(to, to_arg, "H" PTR_FMT, list_val(x));
     } else if (is_boxed(x)) {
-	erts_print(to, to_arg, "H" WORD_FMT, boxed_val(x));
+	erts_print(to, to_arg, "H" PTR_FMT, boxed_val(x));
     } else if (is_immed(x)) {
 	if (is_atom(x)) {
 	    unsigned char* s = atom_tab(atom_val(x))->name;
@@ -311,7 +311,7 @@ heap_dump(int to, void *to_arg, Eterm x)
 	} else if (is_list(x)) {
 	    ptr = list_val(x);
 	    if (ptr[0] != OUR_NIL) {
-		erts_print(to, to_arg, ADDR_FMT ":l", ptr);
+		erts_print(to, to_arg, PTR_FMT ":l", ptr);
 		dump_element(to, to_arg, ptr[0]);
 		erts_putc(to, to_arg, '|');
 		dump_element(to, to_arg, ptr[1]);
@@ -330,12 +330,12 @@ heap_dump(int to, void *to_arg, Eterm x)
 	    ptr = boxed_val(x);
 	    hdr = *ptr;
 	    if (hdr != OUR_NIL) {	/* If not visited */
-		erts_print(to, to_arg, ADDR_FMT ":", ptr);
+		erts_print(to, to_arg, PTR_FMT ":", ptr);
 	        if (is_arity_value(hdr)) {
 		    Uint i;
 		    Uint arity = arityval(hdr);
 
-		    erts_print(to, to_arg, "t" WORD_FMT ":", arity);
+		    erts_print(to, to_arg, "t" ETERM_FMT ":", arity);
 		    for (i = 1; i <= arity; i++) {
 			dump_element(to, to_arg, ptr[i]);
 			if (is_immed(ptr[i])) {
@@ -388,21 +388,43 @@ heap_dump(int to, void *to_arg, Eterm x)
 			    val->flags = (UWord) all_binaries;
 			    all_binaries = val;
 			}
-			erts_print(to, to_arg, "Yc%X:%X:%X", val,
+			erts_print(to, to_arg,
+				   "Yc" PTR_FMT ":" PTR_FMT ":" PTR_FMT,
+				   val,
 				   pb->bytes - (byte *)val->orig_bytes,
 				   size);
 		    } else if (tag == SUB_BINARY_SUBTAG) {
 			ErlSubBin* Sb = (ErlSubBin *) binary_val(x);
-			Eterm* real_bin = binary_val(Sb->orig);
+			Eterm* real_bin;
 			void* val;
 
+			/*
+			 * Must use boxed_val() here, because the original
+			 * binary may have been visited and have had its
+			 * header word changed to OUR_NIL (in which case
+			 * binary_val() will cause an assertion failure in
+			 * the DEBUG emulator).
+			 */
+
+			real_bin = boxed_val(Sb->orig);
+
 			if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
+			    /*
+			     * Unvisited REFC_BINARY: Point directly to
+			     * the binary.
+			     */
 			    ProcBin* pb = (ProcBin *) real_bin;
 			    val = pb->val;
-			} else {	/* Heap binary */
+			} else {
+			    /*
+			     * Heap binary or visited REFC binary: Point
+			     * to heap binary or ProcBin on the heap.
+			     */
 			    val = real_bin;
 			}
-			erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size);
+			erts_print(to, to_arg,
+				   "Ys" PTR_FMT ":" PTR_FMT ":" PTR_FMT,
+				   val, Sb->offs, size);
 		    }
 		    erts_putc(to, to_arg, '\n');
 		    *ptr = OUR_NIL;
@@ -438,7 +460,7 @@ dump_binaries(int to, void *to_arg, Binary* current)
 	long size = current->orig_size;
 	byte* bytes = (byte*) current->orig_bytes;
 
-	erts_print(to, to_arg, "=binary:%X\n", current);
+	erts_print(to, to_arg, "=binary:" PTR_FMT "\n", current);
 	erts_print(to, to_arg, "%X:", size);
 	for (i = 0; i < size; i++) {
 	    erts_print(to, to_arg, "%02X", bytes[i]);
-- 
cgit v1.2.3


From fb3aaa41a046e317609c39c9b8281263a250ef2f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 1 Nov 2013 14:32:00 +0100
Subject: Ensure carrier pool only accessed by schedulers

Disable carrier pool for the thread safe allocator instance 0.
This since non-managed threads allocates and deallocates memory
in this instance, and only managed threads are allowed to
access the carrier pool.
---
 erts/emulator/beam/erl_alloc_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 1fdee4db2c..a6f256a15e 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -5512,7 +5512,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0);
     erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0);
     allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT;
-    allctr->cpool.util_limit = init->acul;
+    allctr->cpool.util_limit = init->ts ? 0 : init->acul;
 #endif
 
     allctr->sbc_threshold		= init->sbct;
-- 
cgit v1.2.3


From e94f730bdda9137ce5f436256ac7fdbdcd0be14a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 17 Oct 2013 15:25:52 +0200
Subject: erts: Prevent valgrind from repeating same memory leaks reports

by using the macro VALGRIND_DO_ADDED_LEAK_CHECK if it exists
for system_info({error_checker,memory})
---
 erts/emulator/beam/erl_bif_info.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 673dfc658c..6479320a83 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1771,7 +1771,11 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 #if defined(PURIFY)
 	    BIF_RET(erts_make_integer(purify_new_leaks(), BIF_P));
 #elif defined(VALGRIND)
+#  ifdef VALGRIND_DO_ADDED_LEAK_CHECK
+	    VALGRIND_DO_ADDED_LEAK_CHECK;
+#  else
 	    VALGRIND_DO_LEAK_CHECK;
+#  endif
 	    BIF_RET(make_small(0));
 #endif
 	} else if (*tp == am_fd) {
-- 
cgit v1.2.3


From f17532112c6b723a7b025c7d74565e7ac2588cbb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 4 Nov 2013 17:18:52 +0100
Subject: Add support for locking mappings to physical memory

Using "+Mlpm all" switch all mappings made by the emulator will
be locked into physical memory.
---
 erts/emulator/beam/erl_alloc.c | 40 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d8da616d05..8e20d58161 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -100,6 +100,8 @@ static Uint install_debug_functions(void);
 #endif
 #endif
 
+static int lock_all_physical_memory = 0;
+
 ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
 ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
 ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
@@ -618,6 +620,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     hdbg_init();
 #endif
 
+    lock_all_physical_memory = 0;
+
     ncpu = eaiop->ncpu;
     if (ncpu < 1)
 	ncpu = 1;
@@ -641,6 +645,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     if (argc && argv)
 	handle_args(argc, argv, &init);
 
+    if (lock_all_physical_memory) {
+#ifdef HAVE_MLOCKALL
+	errno = 0;
+	if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) {
+	    int err = errno;
+	    char *errstr = err ? strerror(err) : "unknown";
+	    erl_exit(-1, "Failed to lock physical memory: %s (%d)\n",
+		     errstr, err);
+	}
+#else
+	erl_exit(-1, "Failed to lock physical memory: Not supported\n");
+#endif
+    }
+
 #ifndef ERTS_SMP
     init.sl_alloc.thr_spec = 0;
     init.std_alloc.thr_spec = 0;
@@ -1630,6 +1648,19 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 			bad_param(param, param+2);
 		    }
 		    break;
+		case 'l':
+		    if (has_prefix("pm", param+2)) {
+			arg = get_value(argv[i]+5, argv, &i);
+			if (strcmp("all", arg) == 0)
+			    lock_all_physical_memory = 1;
+			else if (strcmp("no", arg) == 0)
+			    lock_all_physical_memory = 0;
+			else
+			    bad_value(param, param+4, arg);
+			break;
+		    }
+		    bad_param(param, param+2);
+		    break;
 		case 'u':
 		    if (has_prefix("ycs", argv[i]+3)) {
 			init->alloc_util.ycs
@@ -2749,8 +2780,8 @@ erts_allocator_options(void *proc)
 #endif
     Uint sz, *szp, *hp, **hpp;
     Eterm res, features, settings;
-    Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6];
-    Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6];
+    Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
+    Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
     int a, length;
     SysAllocStat sas;
     Uint *endp = NULL;
@@ -2848,6 +2879,11 @@ erts_allocator_options(void *proc)
 	terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v);
     }
 
+    atoms[length] = am_atom_put("lock_physical_memory", 20);
+    terms[length++] = (lock_all_physical_memory
+		       ? am_atom_put("all", 3)
+		       : am_atom_put("no", 2));
+
     settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms);
 
     length = 0;
-- 
cgit v1.2.3


From bb59a8fcf1f6cf4162a2a97c13087843d1b9dac4 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 5 Nov 2013 15:37:49 +0100
Subject: Add switch for disabling sys_alloc carriers

The switch "+Musac <boolean>" controls if sys_alloc carriers
are allowed.
---
 erts/emulator/beam/erl_alloc.c      |  4 ++++
 erts/emulator/beam/erl_alloc_util.c | 25 ++++++++++++++++++-------
 erts/emulator/beam/erl_alloc_util.h |  7 +++++--
 3 files changed, 27 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 8e20d58161..fb0f47d1b7 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1670,6 +1670,10 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 			init->alloc_util.mmc
 			    = get_amount_value(argv[i]+6, argv, &i);
 		    }
+		    else if (has_prefix("sac", argv[i]+3)) {
+			init->alloc_util.sac
+			    = get_bool_value(argv[i]+6, argv, &i);
+		    }
 		    else {
 			int a;
 			int start = i;
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 1fdee4db2c..b0fc653f32 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -98,6 +98,7 @@ static Uint sys_alloc_carrier_size;
 #if HAVE_ERTS_MSEG
 static Uint max_mseg_carriers;
 #endif
+static int allow_sys_alloc_carriers;
 
 #define ONE_GIGA (1000000000)
 
@@ -3256,13 +3257,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
     int is_mseg = 0;
 #endif
 
-#if HALFWORD_HEAP
-    flags |= CFLG_FORCE_MSEG;
-#elif ERTS_SUPER_ALIGNED_MSEG_ONLY
-    if (flags & CFLG_MBC) {
+    if (HALFWORD_HEAP
+	|| (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC))
+	|| !allow_sys_alloc_carriers) {
 	flags |= CFLG_FORCE_MSEG;
-    }
+	flags &= ~CFLG_FORCE_SYS_ALLOC;
+#if !HAVE_ERTS_MSEG
+	return NULL;
 #endif
+    }
 
     ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC))
 	   || (flags & CFLG_MBC && !(flags & CFLG_SBC)));
@@ -3697,6 +3700,7 @@ static struct {
     Eterm mmc;
 #endif
     Eterm ycs;
+    Eterm sac;
 
     Eterm fix_types;
 
@@ -3789,6 +3793,7 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(mmc);
 #endif
 	AM_INIT(ycs);
+	AM_INIT(sac);
 
 	AM_INIT(fix_types);
 
@@ -4509,16 +4514,21 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg,
 #if HAVE_ERTS_MSEG
 		   "option mmc: %beu\n"
 #endif
-		   "option ycs: %beu\n",
+		   "option ycs: %beu\n"
+		   "option sac: %s\n",
 #if HAVE_ERTS_MSEG
 		   max_mseg_carriers,
 #endif
-		   sys_alloc_carrier_size);
+		   sys_alloc_carrier_size,
+		   allow_sys_alloc_carriers ? "true" : "false");
     }
 
     if (hpp || szp) {
 	res = NIL;
 	ensure_atoms_initialized(NULL);
+	add_2tup(hpp, szp, &res,
+		 am.sac,
+		 allow_sys_alloc_carriers ? am_true : am_false);
 	add_2tup(hpp, szp, &res,
 		 am.ycs,
 		 bld_uint(hpp, szp, sys_alloc_carrier_size));
@@ -5681,6 +5691,7 @@ erts_alcu_init(AlcUInit_t *init)
 #else /* #if HAVE_ERTS_MSEG */
     sys_alloc_carrier_size = ((init->ycs + 4095) / 4096) * 4096;
 #endif
+    allow_sys_alloc_carriers = init->sac;
 
 #ifdef DEBUG
     carrier_alignment = sizeof(Unit_t);
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 222f137024..7be6b1ed9d 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -32,6 +32,7 @@ typedef struct Allctr_t_ Allctr_t;
 typedef struct {
     UWord ycs;
     UWord mmc;
+    int   sac;
 } AlcUInit_t;
 
 typedef struct {
@@ -75,7 +76,8 @@ typedef struct {
 
 #define ERTS_DEFAULT_ALCU_INIT {                                           \
     1024*1024,		/* (bytes)  ycs:    sys_alloc carrier size       */\
-    ~((UWord) 0)	/* (amount) mmc:    max mseg carriers            */ \
+    ~((UWord) 0),	/* (amount) mmc:    max mseg carriers            */\
+    1			/* (bool)   sac:    sys_alloc carriers           */\
 }
 
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
@@ -109,7 +111,8 @@ typedef struct {
 
 #define ERTS_DEFAULT_ALCU_INIT {                                           \
     128*1024,		/* (bytes)  ycs:    sys_alloc carrier size       */\
-    1024      		/* (amount) mmc:    max mseg carriers            */\
+    1024,      		/* (amount) mmc:    max mseg carriers            */\
+    1			/* (bool)   sac:    sys_alloc carriers           */\
 }
 
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
-- 
cgit v1.2.3


From e526cb32336afc4feb4b92665d31ccc2af3e741c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 5 Nov 2013 17:00:38 +0100
Subject: Replace the +MMscmgc switch with +MMscrfsd

Replaced the +MMscmgc switch with the +MMscrfsd switch. The old switch
didn't reflect what it controlled.
---
 erts/emulator/beam/erl_alloc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index fb0f47d1b7..b5ba9bb94a 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1504,11 +1504,11 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 #endif
 			    get_bool_value(argv[i]+8, argv, &i);
 		    }
-		    else if (has_prefix("scmgc", argv[i]+3)) {
+		    else if (has_prefix("scrfsd", argv[i]+3)) {
 #if HAVE_ERTS_MSEG
-			init->mseg.mmap.scmgc =
+			init->mseg.mmap.scrfsd =
 #endif
-			    get_amount_value(argv[i]+8, argv, &i);
+			    get_amount_value(argv[i]+9, argv, &i);
 		    }
 		    else {
 			bad_param(param, param+2);
-- 
cgit v1.2.3


From bb52546e9a671ed0fd55d2e5274f299a853bed51 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Tue, 5 Nov 2013 15:52:56 +0100
Subject: Implement prim_inet:socknames/1,2 and prim_inet:peernames/1,2

---
 erts/emulator/beam/io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c1e66b59af..aeecea1ff5 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4078,7 +4078,7 @@ erts_port_control(Process* c_p,
 	    copy = 1;
 	else {
 	    binp = ((ProcBin *) ebinp)->val;
-	    ASSERT(bufp < bufp + size);
+	    ASSERT(bufp <= bufp + size);
 	    ASSERT(binp->orig_bytes <= bufp
 		   && bufp + size <= binp->orig_bytes + binp->orig_size);
 	    erts_refc_inc(&binp->refc, 1);
-- 
cgit v1.2.3


From 7b739330bb459401f9c11f0f84912aedc7ee22cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20H=C3=A4ssler?= <mhssler@gmail.com>
Date: Sat, 9 Nov 2013 21:04:06 +0100
Subject: Add os:unsetenv/1

New BIF os:unsetenv/1 which deletes an environment variable and
returns 'true'.

Does not change any old functionality.

Calls the libc function unsetenv(3) on UNIX and
SetEnvironmentVariableW(key, NULL) on Windows. The unicode support
is the same as for os:getenv and os:putenv.
---
 erts/emulator/beam/bif.tab      |  1 +
 erts/emulator/beam/erl_bif_os.c | 22 ++++++++++++++++++++++
 erts/emulator/beam/sys.h        |  2 ++
 3 files changed, 25 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index dc8e9101de..6037c08dd8 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -572,6 +572,7 @@ bif erlang:float_to_binary/2
 bif erlang:binary_to_float/1
 
 bif io:printable_range/0
+bif os:unsetenv/1
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index 1062d4379b..e07c622928 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -180,3 +180,25 @@ BIF_RETTYPE os_putenv_2(BIF_ALIST_2)
     BIF_RET(am_true);
 }
 
+BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1)
+{
+    char *key_buf;
+    char buf[STATIC_BUF_SIZE];
+
+    key_buf = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE,
+					      ERTS_ALC_T_TMP,0,0,NULL);
+    if (!key_buf) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    if (erts_sys_unsetenv(key_buf)) {
+	if (key_buf != buf) {
+	    erts_free(ERTS_ALC_T_TMP, key_buf);
+	}
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    if (key_buf != buf) {
+	erts_free(ERTS_ALC_T_TMP, key_buf);
+    }
+    BIF_RET(am_true);
+}
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 9561c0be96..d22f125945 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -750,6 +750,8 @@ int erts_sys_getenv(char *key, char *value, size_t *size);
 int erts_sys_getenv_raw(char *key, char *value, size_t *size);
 /* erts_sys_getenv__() is only allowed to be used in early init phase */
 int erts_sys_getenv__(char *key, char *value, size_t *size);
+/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
+int erts_sys_unsetenv(char *key);
 
 /* Easier to use, but not as efficient, environment functions */
 char *erts_read_env(char *key);
-- 
cgit v1.2.3


From ee0ca14382e76d97285e64b3396fbb87f33e23da Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 Nov 2013 16:56:40 +0100
Subject: erts: Fix bugs in binary_to_term for invalid bitstrings

<<131, 77, Len:32, Bits:8, Data/binary>>

badarg if Bits > 8
	Used to return internally inconsistent bitstring

badarg if Len==0 and Bits > 0
	Used to return invalid *huge* binary (size = (Uint)-1)

badarg if Bits==0 and Len > 0
	Used to return valid binary as if Bits was 8
---
 erts/emulator/beam/erl_binary.h | 2 +-
 erts/emulator/beam/external.c   | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 506c4813fa..24b10cffef 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -225,7 +225,7 @@ erts_free_aligned_binary_bytes(byte* buf)
 ** These extra bytes where earlier (< R13B04) added by an alignment-bug
 ** in this code. Do we dare remove this in some major release (R14?) maybe?
 */
-#ifdef DEBUG
+#if defined(DEBUG) || defined(VALGRIND)
 #  define CHICKEN_PAD 0
 #else
 #  define CHICKEN_PAD (sizeof(void*) - 1)
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 1c88765381..cfdd38df73 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3007,7 +3007,9 @@ dec_term_atom_common:
 
 		n = get_int32(ep);
 		bitsize = ep[4];
-		ep += 5;
+                if (((bitsize==0) != (n==0)) || bitsize > 8)
+                    goto error;
+                ep += 5;
 		if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		    ErlHeapBin* hb = (ErlHeapBin *) hp;
 
@@ -3035,10 +3037,10 @@ dec_term_atom_common:
 		    hp += PROC_BIN_SIZE;
 		}
 		ep += n;
-		if (bitsize == 0) {
+		if (bitsize == 8 || n == 0) {
 		    *objp = bin;
 		} else {
-		    sb = (ErlSubBin *) hp;
+                    sb = (ErlSubBin *)hp;
 		    sb->thing_word = HEADER_SUB_BIN;
 		    sb->orig = bin;
 		    sb->size = n - 1;
-- 
cgit v1.2.3


From f32368c4b34c86aa772a372cdb3c306a79127185 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 Nov 2013 16:58:43 +0100
Subject: erts: Fix bug in binary_to_term for binaries larger than 2^31

---
 erts/emulator/beam/external.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index cfdd38df73..22b0a02937 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2970,7 +2970,7 @@ dec_term_atom_common:
 		n = get_int32(ep);
 		ep += 4;
 	    
-		if (n <= ERL_ONHEAP_BIN_LIMIT) {
+		if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) {
 		    ErlHeapBin* hb = (ErlHeapBin *) hp;
 
 		    hb->thing_word = header_heap_bin(n);
@@ -3010,7 +3010,7 @@ dec_term_atom_common:
                 if (((bitsize==0) != (n==0)) || bitsize > 8)
                     goto error;
                 ep += 5;
-		if (n <= ERL_ONHEAP_BIN_LIMIT) {
+		if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) {
 		    ErlHeapBin* hb = (ErlHeapBin *) hp;
 
 		    hb->thing_word = header_heap_bin(n);
-- 
cgit v1.2.3


From ca0425c6ff85262bc15367f5fd9cbc51cde52b20 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 2 Oct 2013 10:07:27 +0200
Subject: Execution of system tasks in context of another process

A process requesting a system task to be executed in the context of
another process will be notified by a message when the task has
executed. This message will be on the form:
  {RequestType, RequestId, Pid, Result}.

A process requesting a system task to be executed can set priority
on the system task. The requester typically set the same priority
on the task as its own process priority, and by this avoiding
priority inversion. A request for execution of a system task is
made by calling the statically linked in NIF
erts_internal:request_system_task(Pid, Prio, Request). This is an
undocumented ERTS internal function that should remain so. It
should *only* be called from BIF implementations.

Currently defined system tasks are:
* garbage_collect
* check_process_code

Further system tasks can and will be implemented in the future.

The erlang:garbage_collect/[1,2] and erlang:check_process_code/[2,3]
BIFs are now implemented using system tasks. Both the
'garbage_collect' and the 'check_process_code' operations perform
or may perform garbage_collections. By doing these via the
system task functionality all garbage collect operations in the
system will be performed solely in the context of the process
being garbage collected. This makes it possible to later implement
functionality for disabling garbage collection of a process over
context switches.

Newly introduced BIFs:

* erlang:garbage_collect/2 - The new second argument is an option
  list. Introduced option:
  * {async, RequestId} - making it possible for users to issue
    asynchronous garbage collect requests.

* erlang:check_process_code/3 -  The new third argument is an
  option list. Introduced options:
  * {async, RequestId} - making it possible for users to issue
    asynchronous check process code requests.
  * {allow_gc, boolean()} - making it possible to issue requests
    that aren't allowed to garbage collect (operation will abort
    if gc should be needed).
  These options have been introduced as a preparation for
  parallelization of check_process_code operations when the
  code_server is about to purge a module.
---
 erts/emulator/beam/atom.names      |    1 +
 erts/emulator/beam/beam_bif_load.c |  133 ++--
 erts/emulator/beam/bif.c           |   39 --
 erts/emulator/beam/bif.tab         |    6 +-
 erts/emulator/beam/break.c         |   10 +-
 erts/emulator/beam/erl_alloc.types |    2 +
 erts/emulator/beam/erl_message.h   |    5 +
 erts/emulator/beam/erl_process.c   | 1224 ++++++++++++++++++++++++++++++++----
 erts/emulator/beam/erl_process.h   |   74 ++-
 erts/emulator/beam/global.h        |    4 +
 erts/emulator/beam/ops.tab         |   14 +-
 erts/emulator/beam/utils.c         |    2 +-
 12 files changed, 1263 insertions(+), 251 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index c2f32ba089..8433fd826a 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -78,6 +78,7 @@ atom allocated_areas
 atom allocator
 atom allocator_sizes
 atom alloc_util_allocators
+atom allow_gc
 atom allow_passive_connect
 atom already_loaded
 atom amd64
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 649594a334..5239940a50 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -36,7 +36,7 @@
 #include "erl_thr_progress.h"
 
 static void set_default_trace_pattern(Eterm module);
-static Eterm check_process_code(Process* rp, Module* modp);
+static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp);
 static void delete_code(Module* modp);
 static void decrement_refc(BeamInstr* code);
 static int is_native(BeamInstr* code);
@@ -427,69 +427,80 @@ check_old_code_1(BIF_ALIST_1)
 }
 
 Eterm
-check_process_code_2(BIF_ALIST_2)
+erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp)
 {
-    Process* rp;
     Module* modp;
+    Eterm res;
+    ErtsCodeIndex code_ix;
 
-    if (is_not_atom(BIF_ARG_2)) {
-	goto error;
-    }
-    if (is_internal_pid(BIF_ARG_1)) {
-	Eterm res;
-	ErtsCodeIndex code_ix;
-
-	code_ix = erts_active_code_ix();
-	modp = erts_get_module(BIF_ARG_2, code_ix);
-	if (modp == NULL) {		/* Doesn't exist. */
-	    return am_false;
-	}
-	erts_rlock_old_code(code_ix);
-	if (modp->old.code == NULL) { /* No old code. */
-	    erts_runlock_old_code(code_ix);
-	    return am_false;
-	}
-	erts_runlock_old_code(code_ix);
-	
-#ifdef ERTS_SMP
-	rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN,
-				   BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
-#else
-	rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0);
-#endif
-	if (!rp) {
-	    BIF_RET(am_false);
-	}
-	if (rp == ERTS_PROC_LOCK_BUSY) {
-	    ERTS_BIF_YIELD2(bif_export[BIF_check_process_code_2], BIF_P,
-			    BIF_ARG_1, BIF_ARG_2);
-	}
-	erts_rlock_old_code(code_ix);
-	if (modp->old.code != NULL) { /* must check again */
-	    res = check_process_code(rp, modp);
-	}
-	else {
-	    res = am_false;
-	}
-	erts_runlock_old_code(code_ix);
-#ifdef ERTS_SMP
-	if (BIF_P != rp) {
-	    erts_resume(rp, ERTS_PROC_LOCK_MAIN);
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+    (*redsp)++;
+
+    ASSERT(is_atom(module));
+
+    code_ix = erts_active_code_ix();
+    modp = erts_get_module(module, code_ix);
+    if (!modp)
+	return am_false;
+    erts_rlock_old_code(code_ix);
+    res = modp->old.code ? check_process_code(c_p, modp, allow_gc, redsp) : am_false;
+    erts_runlock_old_code(code_ix);
+
+    return res;
+}
+
+BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2)
+{
+    int reds = 0;
+    Eterm res;
+    Eterm olist = BIF_ARG_2;
+    int allow_gc = 1;
+
+    if (is_not_atom(BIF_ARG_1))
+	goto badarg;
+
+    while (is_list(olist)) {
+	Eterm *lp = list_val(olist);
+	Eterm opt = CAR(lp);
+	if (is_tuple(opt)) {
+	    Eterm* tp = tuple_val(opt);
+	    switch (arityval(tp[0])) {
+	    case 2:
+		switch (tp[1]) {
+		case am_allow_gc:
+		    switch (tp[2]) {
+		    case am_false:
+			allow_gc = 0;
+			break;
+		    case am_true:
+			allow_gc = 1;
+			break;
+		    default:
+			goto badarg;
+		    }
+		    break;
+		default:
+		    goto badarg;
+		}
+		break;
+	    default:
+		goto badarg;
+	    }
 	}
-#endif
-	BIF_RET(res);
-    }
-    else if (is_external_pid(BIF_ARG_1)
-	     && external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) {
-	BIF_RET(am_false);
+	else
+	    goto badarg;
+	olist = CDR(lp);
     }
+    if (is_not_nil(olist))
+	goto badarg;
 
- error:
+    res = erts_check_process_code(BIF_P, BIF_ARG_1, allow_gc, &reds);
+
+    BIF_RET2(res, reds);
+
+badarg:
     BIF_ERROR(BIF_P, BADARG);
 }
 
-
 BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 {
     ErtsCodeIndex code_ix;
@@ -710,7 +721,7 @@ set_default_trace_pattern(Eterm module)
 }
 
 static Eterm
-check_process_code(Process* rp, Module* modp)
+check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 {
     BeamInstr* start;
     char* mod_start;
@@ -786,6 +797,8 @@ check_process_code(Process* rp, Module* modp)
 		if (done_gc) {
 		    return am_true;
 		} else {
+		    if (!allow_gc)
+			return am_aborted;
 		    /*
 		    * Try to get rid of this fun by garbage collecting.
 		    * Clear both fvalue and ftrace to make sure they
@@ -796,7 +809,7 @@ check_process_code(Process* rp, Module* modp)
 		    rp->ftrace = NIL;
 		    done_gc = 1;
 		    FLAGS(rp) |= F_NEED_FULLSWEEP;
-		    (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+		    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
 		    goto rescan;
 		}
 	    }
@@ -850,6 +863,9 @@ check_process_code(Process* rp, Module* modp)
 	    Uint lit_size;
 	    struct erl_off_heap_header* oh;
 
+	    if (!allow_gc)
+		return am_aborted;
+
 	    /*
 	     * Try to get rid of constants by by garbage collecting.
 	     * Clear both fvalue and ftrace.
@@ -859,11 +875,12 @@ check_process_code(Process* rp, Module* modp)
 	    rp->ftrace = NIL;
 	    done_gc = 1;
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
-	    (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+	    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
 	    literals = (Eterm *) modp->old.code[MI_LITERALS_START];
 	    lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals;
 	    oh = (struct erl_off_heap_header *)
 		modp->old.code[MI_LITERALS_OFF_HEAP];
+	    *redsp += lit_size / 10; /* Need, better value... */
 	    erts_garbage_collect_literals(rp, literals, lit_size, oh);
 	}
     }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 755c5e6882..58bd6aac0b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3741,45 +3741,6 @@ BIF_RETTYPE now_0(BIF_ALIST_0)
 
 /**********************************************************************/
 
-BIF_RETTYPE garbage_collect_1(BIF_ALIST_1)
-{
-    int reds;
-    Process *rp;
-
-    if (is_not_pid(BIF_ARG_1)) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-
-    if (BIF_P->common.id == BIF_ARG_1)
-	rp = BIF_P;
-    else {
-#ifdef ERTS_SMP
-	rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN,
-				   BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
-	if (rp == ERTS_PROC_LOCK_BUSY)
-	    ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1);
-#else
-	rp = erts_proc_lookup(BIF_ARG_1);
-#endif
-	if (!rp)
-	    BIF_RET(am_false);
-    }
-
-    /* The GC cost is taken for the process executing this BIF. */
-
-    FLAGS(rp) |= F_NEED_FULLSWEEP;
-    reds = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
-
-#ifdef ERTS_SMP
-    if (BIF_P != rp) {
-	erts_resume(rp, ERTS_PROC_LOCK_MAIN);
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
-    }
-#endif
-
-    BIF_RET2(am_true, reds);
-}
-
 BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
 {
     int reds;
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index dc8e9101de..3b7bb56885 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -46,7 +46,6 @@ bif erlang:atom_to_list/1
 bif erlang:binary_to_list/1
 bif erlang:binary_to_list/3
 bif erlang:binary_to_term/1	
-bif erlang:check_process_code/2
 bif erlang:crc32/1
 bif erlang:crc32/2
 bif erlang:crc32_combine/3
@@ -67,7 +66,6 @@ bif erlang:float_to_list/1
 bif erlang:float_to_list/2
 bif erlang:fun_info/2
 bif erlang:garbage_collect/0
-bif erlang:garbage_collect/1
 bif erlang:get/0
 bif erlang:get/1
 bif erlang:get_keys/1
@@ -155,6 +153,10 @@ bif erts_internal:port_control/3
 bif erts_internal:port_close/1
 bif erts_internal:port_connect/2
 
+bif erts_internal:request_system_task/3
+bif erts_internal:check_process_code/2
+
+
 # inet_db support
 bif erlang:port_set_data/2
 bif erlang:port_get_data/1
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index ad9a89b642..acbd700cc4 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -109,10 +109,12 @@ process_killer(void)
 		    erts_smp_proc_lock(rp, rp_locks);
 		    state = erts_smp_atomic32_read_acqb(&rp->state);
 		    if (state & (ERTS_PSFLG_FREE
-				  | ERTS_PSFLG_EXITING
-				  | ERTS_PSFLG_ACTIVE
-				  | ERTS_PSFLG_IN_RUNQ
-				  | ERTS_PSFLG_RUNNING)) {
+				 | ERTS_PSFLG_EXITING
+				 | ERTS_PSFLG_ACTIVE
+				 | ERTS_PSFLG_ACTIVE_SYS
+				 | ERTS_PSFLG_IN_RUNQ
+				 | ERTS_PSFLG_RUNNING
+				 | ERTS_PSFLG_RUNNING_SYS)) {
 			erts_printf("Can only kill WAITING processes this way\n");
 		    }
 		    else {
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index bb5eba80be..32308fae9b 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -268,6 +268,8 @@ type	PROC_INTERVAL	LONG_LIVED	SYSTEM		process_interval
 type    BUSY_CALLER_TAB	SHORT_LIVED	SYSTEM		busy_caller_table
 type    BUSY_CALLER	SHORT_LIVED	SYSTEM		busy_caller
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
+type	PROC_SYS_TSK	SHORT_LIVED	PROCESSES	proc_sys_task
+type	PROC_SYS_TSK_QS	SHORT_LIVED	PROCESSES	proc_sys_task_queues
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 771eba431f..0f3bb8d281 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -46,6 +46,11 @@ typedef struct erl_off_heap {
     Uint64 overhead;     /* Administrative overhead (used to force GC). */
 } ErlOffHeap;
 
+#define ERTS_INIT_OFF_HEAP(OHP)			\
+    do {					\
+	(OHP)->first = NULL;			\
+	(OHP)->overhead = 0;			\
+    } while (0)
 #include "external.h"
 #include "erl_process.h"
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 434d5ca147..ad806da660 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -283,6 +283,40 @@ struct erts_system_profile_flags_t erts_system_profile_flags;
 #error "Need to store process_count in another type"
 #endif
 
+typedef enum {
+    ERTS_PSTT_GC,	/* Garbage Collect */
+    ERTS_PSTT_CPC	/* Check Process Code */
+} ErtsProcSysTaskType;
+
+#define ERTS_MAX_PROC_SYS_TASK_ARGS 2
+
+struct ErtsProcSysTask_ {
+    ErtsProcSysTask *next;
+    ErtsProcSysTask *prev;
+    ErtsProcSysTaskType type;
+    Eterm requester;
+    Eterm reply_tag;
+    Eterm req_id;
+    Uint req_id_sz;
+    Eterm arg[ERTS_MAX_PROC_SYS_TASK_ARGS];
+    ErlOffHeap off_heap;
+    Eterm heap[1];
+};
+
+#define ERTS_PROC_SYS_TASK_SIZE(HSz) \
+    (sizeof(ErtsProcSysTask) - sizeof(Eterm) + sizeof(Eterm)*(HSz))
+
+struct ErtsProcSysTaskQs_ {
+    int qmask;
+    int ncount;
+    ErtsProcSysTask *q[ERTS_NO_PROC_PRIO_LEVELS];
+};
+
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proc_sys_task_queues,
+				 ErtsProcSysTaskQs,
+				 50,
+				 ERTS_ALC_T_PROC_SYS_TSK_QS)
+
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_op_list,
 				 ErtsMiscOpList,
 				 10,
@@ -353,6 +387,14 @@ static void aux_work_timeout_early_init(int no_schedulers);
 static void aux_work_timeout_late_init(void);
 static void setup_aux_work_timer(void);
 
+static int execute_sys_tasks(Process *c_p,
+			     erts_aint32_t *statep,
+			     int in_reds);
+static int cleanup_sys_tasks(Process *c_p,
+			     erts_aint32_t in_state,
+			     int in_reds);
+
+
 #if defined(DEBUG) || 0
 #define ERTS_DBG_CHK_AUX_WORK_VAL(V) dbg_chk_aux_work_val((V))
 static void
@@ -2848,7 +2890,7 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep)
     if (statep)
 	*statep = state;
 
-    prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
+    prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 
     rqi = &runq->procs.prio_info[prio];
 
@@ -2997,7 +3039,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 		    erts_aint32_t state;
 		    state = erts_smp_atomic32_read_acqb(&proc->state);
 		    if (!(ERTS_PSFLG_BOUND & state)
-			&& (prio == (int) (ERTS_PSFLG_PRIO_MASK & state))) {
+			&& (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) {
 			ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio];
 			unqueue_process(rq, rpq, rqi, prio, prev_proc, proc);
 			erts_smp_runq_unlock(rq);
@@ -3079,7 +3121,7 @@ schedule_bound_processes(ErtsRunQueue *rq,
     while (proc) {
 	erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state);
 	next = proc->next;
-	enqueue_process(rq, (int) (ERTS_PSFLG_PRIO_MASK & state), proc);
+	enqueue_process(rq, (int) ERTS_PSFLGS_GET_PRQ_PRIO(state), proc);
 	proc = next;
     }
 }
@@ -3184,7 +3226,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		sbpp->last = proc;
 	    }
 	    else {
-		int prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
+		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		erts_smp_runq_unlock(rq);
 
 		to_rq = mp->prio[prio].runq;
@@ -3257,7 +3299,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq,
 	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state);
 	    if (!(ERTS_PSFLG_BOUND & state)) {
 		/* Steal process */
-		int prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
+		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio];
 		unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc);
 		erts_smp_runq_unlock(vrq);
@@ -4575,6 +4617,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 #endif
 
     init_misc_op_list_alloc();
+    init_proc_sys_task_queues_alloc();
 
 #ifdef ERTS_SMP
     set_wakeup_other_data();
@@ -4858,46 +4901,166 @@ erts_get_scheduler_data(void)
 
 #endif
 
+static Process *
+make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
+{
+    erts_aint32_t state;
+    Process *proxy;
+#ifdef ERTS_SMP
+    ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue);
+#endif
+
+    state = (ERTS_PSFLG_PROXY
+	     | ERTS_PSFLG_IN_RUNQ
+	     | (((erts_aint32_t) 1) << (prio + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET))
+	     | (prio << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
+	     | (prio << ERTS_PSFLGS_USR_PRIO_OFFSET)
+	     | (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET));
+
+    if (prev_proxy) {
+	proxy = prev_proxy;
+	ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
+	erts_smp_atomic32_set_nob(&proxy->state, state);
+#ifdef ERTS_SMP
+	RUNQ_SET_RQ(&proc->run_queue, rq);
+#endif
+    }
+    else {
+	proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process));
+#ifdef DEBUG
+	{
+	    int i;
+	    Uint32 *ui32 = (Uint32 *) (char *) proxy;
+	    for (i = 0; i < sizeof(Process)/sizeof(Uint32); i++)
+		ui32[i] = (Uint32) 0xdeadbeef;
+	}
+#endif
+	erts_smp_atomic32_init_nob(&proxy->state, state);
+#ifdef ERTS_SMP
+	erts_smp_atomic_init_nob(&proxy->run_queue,
+				 erts_smp_atomic_read_nob(&proc->run_queue));
+#endif
+    }
+
+    proxy->common.id = proc->common.id;
+
+    return proxy;
+}
+
+static ERTS_INLINE void
+free_proxy_proc(Process *proxy)
+{
+    ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
+    erts_free(ERTS_ALC_T_PROC, proxy);
+}
+
+
+static ERTS_INLINE int
+check_enqueue_in_prio_queue(erts_aint32_t *prq_prio_p,
+			    erts_aint32_t *newp,
+			    erts_aint32_t actual)
+{
+    erts_aint32_t aprio, qbit, max_qbit;
+
+    aprio = (actual >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK;
+    qbit = 1 << aprio;
+
+    *prq_prio_p = aprio;
+
+    max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK;
+    max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
+    max_qbit &= -max_qbit;
+    /*
+     * max_qbit now either contain bit set for highest prio queue or a bit
+     * out of range (which will have a value larger than valid range).
+     */
+
+    if (qbit >= max_qbit)
+	return 0; /* Already queued in higher or equal prio */
+
+    /* Need to enqueue (if already enqueued, it is in lower prio) */
+    *newp |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET;
+
+    if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK))
+	!= (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) {
+	/*
+	 * Process struct already enqueued, or actual prio not
+	 * equal to user prio, i.e., enqueue using proxy.
+	 */
+	return -1;
+    }
+
+    /*
+     * Enqueue using process struct.
+     */
+    *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK;
+    *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET);
+    return 1;
+}
+
 /*
  * scheduler_out_process() return with c_rq locked.
  */
 static ERTS_INLINE int
-schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p)
+schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Process *proxy)
 {
-    erts_aint32_t a, e, n;
+    erts_aint32_t a, e, n, enq_prio = -1;
     int res = 0;
+    int enqueue; /* < 0 -> use proxy */
 
     a = state;
 
     while (1) {
 	n = e = a;
 
-	ASSERT(a & ERTS_PSFLG_RUNNING);
-	ASSERT(!(a & ERTS_PSFLG_IN_RUNQ));
+	ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
 
-	n &= ~ERTS_PSFLG_RUNNING;
-	if ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE)
-	    n |= ERTS_PSFLG_IN_RUNQ;
+	enqueue = 0;
+
+	n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS);
+	if (a & ERTS_PSFLG_ACTIVE_SYS
+	    || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
+	    enqueue = check_enqueue_in_prio_queue(&enq_prio, &n, a);
+	}
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
     }
 
-    if (!(n & ERTS_PSFLG_IN_RUNQ)) {
-	if (erts_system_profile_flags.runnable_procs)
-	    profile_runnable_proc(p, am_inactive);
+    if (!enqueue) {
+
+	if (erts_system_profile_flags.runnable_procs) {
+
+	    if (!(a & ERTS_PSFLG_ACTIVE_SYS)
+		&& (!(a & ERTS_PSFLG_ACTIVE)
+		    || (a & ERTS_PSFLG_SUSPENDED))) {
+		/* Process inactive */
+		profile_runnable_proc(p, am_inactive);
+	    }
+	}
+
+	if (proxy)
+	    free_proxy_proc(proxy);
     }
     else {
-	int prio = (int) (ERTS_PSFLG_PRIO_MASK & n);
+	Process *sched_p;
 	ErtsRunQueue *runq = erts_get_runq_proc(p);
 
-	ASSERT(!(n & ERTS_PSFLG_SUSPENDED));
+	ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS));
+
+	if (enqueue < 0)
+	    sched_p = make_proxy_proc(proxy, p, enq_prio);
+	else {
+	    sched_p = p;
+	    if (proxy)
+		free_proxy_proc(proxy);
+	}
 
 #ifdef ERTS_SMP
 	if (!(ERTS_PSFLG_BOUND & n)) {
-	    ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio);
+	    ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
 	    if (new_runq) {
-		RUNQ_SET_RQ(&p->run_queue, new_runq);
+		RUNQ_SET_RQ(&sched_p->run_queue, new_runq);
 		runq = new_runq;
 	    }
 	}
@@ -4908,7 +5071,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p)
 	erts_smp_runq_lock(runq);
 
 	/* Enqueue the process */
-	enqueue_process(runq, prio, p);
+	enqueue_process(runq, (int) enq_prio, sched_p);
 
 	if (runq == c_rq)
 	    return res;
@@ -4920,14 +5083,13 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p)
 }
 
 static ERTS_INLINE void
-add2runq(Process *p, erts_aint32_t state)
+add2runq(Process *p, erts_aint32_t state, erts_aint32_t prio)
 {
-    int prio = (int) (ERTS_PSFLG_PRIO_MASK & state);
     ErtsRunQueue *runq = erts_get_runq_proc(p);
 
 #ifdef ERTS_SMP
     if (!(ERTS_PSFLG_BOUND & state)) {
-	ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio);
+	ErtsRunQueue *new_runq = erts_check_emigration_need(runq, (int) prio);
 	if (new_runq) {
 	    RUNQ_SET_RQ(&p->run_queue, new_runq);
 	    runq = new_runq;
@@ -4939,101 +5101,236 @@ add2runq(Process *p, erts_aint32_t state)
     erts_smp_runq_lock(runq);
 
     /* Enqueue the process */
-    enqueue_process(runq, prio, p);
+    enqueue_process(runq, (int) prio, p);
 
     erts_smp_runq_unlock(runq);
     smp_notify_inc_runq(runq);
 
 }
 
-static ERTS_INLINE void
-schedule_process(Process *p, erts_aint32_t state, int active_enq)
+static ERTS_INLINE int
+change_proc_schedule_state(Process *p,
+			   erts_aint32_t clear_state_flags,
+			   erts_aint32_t set_state_flags,
+			   erts_aint32_t *statep,
+			   erts_aint32_t *enq_prio_p)
 {
-    erts_aint32_t a = state, n;
+    /*
+     * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS and
+     *       ERTS_PSFLG_ACTIVE_SYS are not allowed to be
+     *       altered by this function!
+     */
+    erts_aint32_t a = *statep, n;
+    int enqueue; /* < 0 -> use proxy */
+
+    ASSERT(!(a & ERTS_PSFLG_PROXY));
+    ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING
+				 | ERTS_PSFLG_RUNNING_SYS
+				 | ERTS_PSFLG_ACTIVE_SYS)) == 0);
+    ASSERT((set_state_flags & (ERTS_PSFLG_RUNNING
+			       | ERTS_PSFLG_RUNNING_SYS
+			       | ERTS_PSFLG_ACTIVE_SYS)) == 0);
 
     while (1) {
 	erts_aint32_t e;
 	n = e = a;
 
+	enqueue = 0;
+
 	if (a & ERTS_PSFLG_FREE)
-	    return; /* We don't want to schedule free processes... */
-	n |= ERTS_PSFLG_ACTIVE;
-	if (!(a & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_RUNNING)))
-	    n |= ERTS_PSFLG_IN_RUNQ;
+	    break; /* We don't want to schedule free processes... */
+
+	if (clear_state_flags)
+	    n &= ~clear_state_flags;
+
+	if (set_state_flags)
+	    n |= set_state_flags;
+
+	if ((n & (ERTS_PSFLG_SUSPENDED
+		  | ERTS_PSFLG_RUNNING
+		  | ERTS_PSFLG_RUNNING_SYS
+		  | ERTS_PSFLG_IN_RUNQ
+		  | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) {
+	    /*
+	     * Active and seemingly need to be enqueued, but
+	     * process may be in a run queue via proxy, need
+	     * further inspection...
+	     */
+	    enqueue = check_enqueue_in_prio_queue(enq_prio_p, &n, a);
+	}
+
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
-	if (!active_enq && (a & ERTS_PSFLG_ACTIVE))
-	    return; /* Someone else activated process ... */
+	if (enqueue == 0 && n == a)
+	    break;
     }
 
-    if (erts_system_profile_flags.runnable_procs
-	&& !(a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) {
-    	profile_runnable_proc(p, am_active);
+    if (erts_system_profile_flags.runnable_procs) {
+
+	if (((n & (ERTS_PSFLG_SUSPENDED
+		   | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE)
+	    && (!(a & (ERTS_PSFLG_ACTIVE_SYS
+		       | ERTS_PSFLG_RUNNING
+		       | ERTS_PSFLG_RUNNING_SYS)
+		  && (!(a & ERTS_PSFLG_ACTIVE)
+		      || (a & ERTS_PSFLG_SUSPENDED))))) {
+	    /* We activated a prevously inactive process */
+	    profile_runnable_proc(p, am_active);
+	}
+
     }
 
-    if ((n & ERTS_PSFLG_IN_RUNQ) && !(a & ERTS_PSFLG_IN_RUNQ))
-	add2runq(p, n);
+    *statep = a;
+
+    return enqueue;
+}
+
+static ERTS_INLINE void
+schedule_process(Process *p, erts_aint32_t in_state)
+{
+    erts_aint32_t enq_prio  = -1;
+    erts_aint32_t state = in_state;
+    int enqueue = change_proc_schedule_state(p,
+					     0,
+					     ERTS_PSFLG_ACTIVE,
+					     &state,
+					     &enq_prio);
+    if (enqueue)
+	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
+		 state,
+		 enq_prio);
 }
 
 void
 erts_schedule_process(Process *p, erts_aint32_t state)
 {
-    schedule_process(p, state, 0);
+    schedule_process(p, state);
+}
+
+static void
+schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
+{
+    erts_aint32_t a = state, n, enq_prio = -1;
+    int enqueue; /* < 0 -> use proxy */
+
+    ASSERT(!(state & ERTS_PSFLG_PROXY));
+
+    while (1) {
+	erts_aint32_t e;
+	n = e = a;
+
+	if (a & ERTS_PSFLG_FREE)
+	    return; /* We don't want to schedule free processes... */
+
+	enqueue = 0;
+	n |= ERTS_PSFLG_ACTIVE_SYS;
+	if (!(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)))
+	    enqueue = check_enqueue_in_prio_queue(&enq_prio, &n, a);
+	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+	if (a == e)
+	    break;
+	if (a == n && !enqueue)
+	    goto cleanup;
+    }
+
+    if (erts_system_profile_flags.runnable_procs) {
+
+	if (!(a & (ERTS_PSFLG_ACTIVE_SYS
+		   | ERTS_PSFLG_RUNNING
+		   | ERTS_PSFLG_RUNNING_SYS))
+	    && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) {
+	    /* We activated a prevously inactive process */
+	    profile_runnable_proc(p, am_active);
+	}
+
+    }
+
+    if (enqueue) {
+	Process *sched_p;
+	if (enqueue > 0)
+	    sched_p = p;
+	else {
+	    sched_p = make_proxy_proc(proxy, p, enq_prio);
+	    proxy = NULL;
+	}
+	add2runq(sched_p, n, enq_prio);
+    }
+
+cleanup:
+    if (proxy)
+	free_proxy_proc(proxy);
 }
 
 static ERTS_INLINE int
 suspend_process(Process *c_p, Process *p)
 {
-    erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+    erts_aint32_t state;
     int suspended = 0;
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
+    state = erts_smp_atomic32_read_acqb(&p->state);
+
     if ((state & ERTS_PSFLG_SUSPENDED))
 	suspended = -1;
     else {
 	if (c_p == p) {
 	    state = erts_smp_atomic32_read_bor_relb(&p->state,
 						    ERTS_PSFLG_SUSPENDED);
-	    state |= ERTS_PSFLG_SUSPENDED;
 	    ASSERT(state & ERTS_PSFLG_RUNNING);
-	    suspended = 1;
+	    suspended = (state & ERTS_PSFLG_SUSPENDED) ? -1: 1;
 	}
 	else {
 	    while (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_EXITING))) {
-		erts_aint32_t e, n;
+		erts_aint32_t n, e;
+
 		n = e = state;
 		n |= ERTS_PSFLG_SUSPENDED;
 		state = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e);
 		if (state == e) {
-		    state = n;
 		    suspended = 1;
 		    break;
 		}
+		if (state & ERTS_PSFLG_SUSPENDED) {
+		    suspended = -1;
+		    break;
+		}
 	    }
 	}
     }
 
-    if (state & ERTS_PSFLG_SUSPENDED) {
+    if (suspended) {
 
 	ASSERT(!(ERTS_PSFLG_RUNNING & state)
 	       || p == erts_get_current_process());
 
-	if (erts_system_profile_flags.runnable_procs
-	    && (p->rcount == 0)
-	    && (state & ERTS_PSFLG_ACTIVE)) {
-	    profile_runnable_proc(p, am_inactive);
+	if (suspended > 0 && erts_system_profile_flags.runnable_procs) {
+
+	    /* 'state' is before our change... */
+
+	    if ((state & (ERTS_PSFLG_ACTIVE
+			  | ERTS_PSFLG_ACTIVE_SYS
+			  | ERTS_PSFLG_RUNNING
+			  | ERTS_PSFLG_RUNNING_SYS
+			  | ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
+		/* We made process inactive */
+		profile_runnable_proc(p, am_inactive);
+	    }
+
 	}
 
 	p->rcount++;  /* count number of suspend */
     }
+
     return suspended;
 }
 
 static ERTS_INLINE void
 resume_process(Process *p)
 {
-    erts_aint32_t state;
+    erts_aint32_t state, enq_prio = -1;
+    int enqueue;
+
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
     ASSERT(p->rcount > 0);
@@ -5041,14 +5338,16 @@ resume_process(Process *p)
     if (--p->rcount > 0)  /* multiple suspend */
 	return;
 
-    state = erts_smp_atomic32_read_band_mb(&p->state, ~ERTS_PSFLG_SUSPENDED);
-    state &= ~ERTS_PSFLG_SUSPENDED;
-    if ((state & (ERTS_PSFLG_EXITING
-		  | ERTS_PSFLG_ACTIVE
-		  | ERTS_PSFLG_IN_RUNQ
-		  | ERTS_PSFLG_RUNNING)) == ERTS_PSFLG_ACTIVE) {
-	schedule_process(p, state, 1);
-    }
+    state = erts_smp_atomic32_read_nob(&p->state);
+    enqueue = change_proc_schedule_state(p,
+					 ERTS_PSFLG_SUSPENDED,
+					 0,
+					 &state,
+					 &enq_prio);
+    if (enqueue)
+	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
+		 state,
+		 enq_prio);
 }
 
 int
@@ -6032,7 +6331,8 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 		goto done;
 	}
 	else {
-	    if (!(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&rp->state)))
+	    if (!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
+		  & erts_smp_atomic32_read_acqb(&rp->state)))
 		goto done;
 
 	}
@@ -6695,7 +6995,7 @@ Eterm
 erts_get_process_priority(Process *p)
 {
     erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
-    switch (state & ERTS_PSFLG_PRIO_MASK) {
+    switch (ERTS_PSFLGS_GET_USR_PRIO(state)) {
     case PRIORITY_MAX:		return am_max;
     case PRIORITY_HIGH:		return am_high;
     case PRIORITY_NORMAL:	return am_normal;
@@ -6718,18 +7018,60 @@ erts_set_process_priority(Process *p, Eterm value)
     }
 
     a = erts_smp_atomic32_read_nob(&p->state);
-    if (nprio == (a & ERTS_PSFLG_PRIO_MASK))
+    if (nprio == ERTS_PSFLGS_GET_USR_PRIO(a))
 	oprio = nprio;
     else {
-	erts_aint32_t e, n;
+	int slocked = 0;
+	erts_aint32_t e, n, aprio;
+
+	if (a & ERTS_PSFLG_ACTIVE_SYS) {
+	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	    slocked = 1;
+	}
+
 	do {
-	    oprio = a & ERTS_PSFLG_PRIO_MASK;
+	    oprio = ERTS_PSFLGS_GET_USR_PRIO(a);
 	    n = e = a;
 
-	    ASSERT(!(a & ERTS_PSFLG_IN_RUNQ));
+	    if (!(a & ERTS_PSFLG_ACTIVE_SYS))
+		aprio = nprio;
+	    else {
+		int max_qbit;
+
+		if (!slocked) {
+		    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+		    slocked = 1;
+		}
+
+		max_qbit = p->sys_task_qs->qmask;
+		max_qbit &= -max_qbit;
+		switch (max_qbit) {
+		case MAX_BIT:
+		    aprio = PRIORITY_MAX;
+		    break;
+		case HIGH_BIT:
+		    aprio = PRIORITY_HIGH;
+		    break;
+		case NORMAL_BIT:
+		    aprio = PRIORITY_NORMAL;
+		    break;
+		case LOW_BIT:
+		    aprio = PRIORITY_LOW;
+		    break;
+		default:
+		    ERTS_INTERNAL_ERROR("Invalid qmask");
+		    aprio = -1;
+		}
+
+		if (aprio > nprio) /* low value -> high prio */
+		    aprio = nprio;
+	    }
+
+	    n &= ~(ERTS_PSFLGS_USR_PRIO_MASK
+		   | ERTS_PSFLGS_ACT_PRIO_MASK);
+	    n |= ((nprio << ERTS_PSFLGS_USR_PRIO_OFFSET)
+		  | (aprio << ERTS_PSFLGS_ACT_PRIO_OFFSET));
 
-	    n &= ~ERTS_PSFLG_PRIO_MASK;
-	    n |= nprio;
 	    a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	} while (a != e);
     }
@@ -6763,6 +7105,7 @@ erts_set_process_priority(Process *p, Eterm value)
 
 Process *schedule(Process *p, int calls)
 {
+    Process *proxy_p = NULL;
     ErtsRunQueue *rq;
     erts_aint_t dt;
     ErtsSchedulerData *esdp;
@@ -6805,6 +7148,8 @@ Process *schedule(Process *p, int calls)
 	actual_reds = reds = 0;
 	erts_smp_runq_lock(rq);
     } else {
+    sched_out_proc:
+
 #ifdef ERTS_SMP
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 	esdp = p->scheduler_data;
@@ -6858,10 +7203,11 @@ Process *schedule(Process *p, int calls)
 
 	esdp->reductions += reds;
 
-	schedule_out_process(rq, state, p); /* Returns with rq locked! */
+	schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */
+	proxy_p = NULL;
 
 	ERTS_PROC_REDUCTIONS_EXECUTED(rq,
-				      (int) (state & ERTS_PSFLG_PRIO_MASK),
+				      (int) ERTS_PSFLGS_GET_USR_PRIO(state),
 				      reds,
 				      actual_reds);
 
@@ -6870,18 +7216,19 @@ Process *schedule(Process *p, int calls)
 	p->scheduler_data = NULL;
 #endif
 
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 
 	if (state & ERTS_PSFLG_FREE) {
 #ifdef ERTS_SMP
 	    ASSERT(esdp->free_process == p);
 	    esdp->free_process = NULL;
-#else	    
-	    erts_free_proc(p);
+#else
+	    state = erts_smp_atomic32_read_nob(&p->state);
+	    if (!(state & ERTS_PSFLG_IN_RUNQ))
+		erts_free_proc(p);
 #endif
 	}
 
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-
 #ifdef ERTS_SMP
 	ASSERT(!esdp->free_process);
 #endif
@@ -7088,6 +7435,8 @@ Process *schedule(Process *p, int calls)
 	 * Find a new process to run.
 	 */
     pick_next_process: {
+	    erts_aint32_t psflg_band_mask;
+	    erts_aint32_t running_flag;
 	    int prio_q;
 	    int qmask;
 
@@ -7121,18 +7470,62 @@ Process *schedule(Process *p, int calls)
 
 	    ASSERT(p); /* Wrong qmask in rq->flags? */
 
+	    psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state)
+							+ ERTS_PSFLGS_IN_PRQ_MASK_OFFSET));
+
+	    if (!(state & ERTS_PSFLG_PROXY))
+		psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ;
+	    else {
+		proxy_p = p;
+		p = erts_proc_lookup_raw(proxy_p->common.id);
+		if (!p) {
+		    free_proxy_proc(proxy_p);
+		    proxy_p = NULL;
+		    goto pick_next_process;
+		}
+		state = erts_smp_atomic32_read_nob(&p->state);
+	    }
+
+
+	    if (state & ERTS_PSFLG_ACTIVE_SYS)
+		running_flag = ERTS_PSFLG_RUNNING_SYS;
+	    else
+		running_flag = ERTS_PSFLG_RUNNING;
+
 	    while (1) {
 		erts_aint32_t exp, new, tmp;
 		tmp = new = exp = state;
-		new &= ~ERTS_PSFLG_IN_RUNQ;
-		tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT);
-		if (tmp != ERTS_PSFLG_SUSPENDED)
-		    new |= ERTS_PSFLG_RUNNING;
+		new &= psflg_band_mask;
+		if (!(state & (ERTS_PSFLG_RUNNING
+			       | ERTS_PSFLG_RUNNING_SYS))) {
+		    tmp = state & (ERTS_PSFLG_SUSPENDED
+				   | ERTS_PSFLG_PENDING_EXIT
+				   | ERTS_PSFLG_ACTIVE_SYS);
+		    if (tmp != ERTS_PSFLG_SUSPENDED)
+			new |= running_flag;
+		}
 		state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp);
 		if (state == exp) {
-		    tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT);
-		    if (tmp == ERTS_PSFLG_SUSPENDED)
+		    if ((state & (ERTS_PSFLG_RUNNING
+				  | ERTS_PSFLG_RUNNING_SYS
+				  | ERTS_PSFLG_FREE))
+			|| ((state & (ERTS_PSFLG_SUSPENDED
+				      | ERTS_PSFLG_PENDING_EXIT
+				      | ERTS_PSFLG_ACTIVE_SYS))
+			    == ERTS_PSFLG_SUSPENDED)) {
+			if (state & ERTS_PSFLG_FREE) {
+#ifdef ERTS_SMP
+			    erts_smp_proc_dec_refc(p);
+#else
+			    erts_free_proc(p);
+#endif
+			}
+			if (proxy_p) {
+			    free_proxy_proc(proxy_p);
+			    proxy_p = NULL;
+			}
 			goto pick_next_process;
+		    }
 		    state = new;
 		    break;
 		}
@@ -7162,7 +7555,7 @@ Process *schedule(Process *p, int calls)
 					  (UWord) esdp->no);
 	    int migrated = old && old != esdp->no;
 
-	    prio = (int) (state & ERTS_PSFLG_PRIO_MASK);
+	    prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state);
 
 	    erts_smp_spin_lock(&erts_sched_stat.lock);
 	    erts_sched_stat.prio[prio].total_executed++;
@@ -7182,9 +7575,6 @@ Process *schedule(Process *p, int calls)
 	ASSERT(!p->scheduler_data);
 	p->scheduler_data = esdp;
 #endif
-	/* Never run a suspended process */
-	ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
-
 	reds = context_reds;
 
 	if (IS_TRACED(p)) {
@@ -7210,21 +7600,574 @@ Process *schedule(Process *p, int calls)
 	    erts_check_my_tracer_proc(p);
 #endif
 
+	if (state & ERTS_PSFLG_RUNNING_SYS) {
+	    reds -= execute_sys_tasks(p, &state, reds);
+	    if (reds <= 0) {
+		p->fcalls = reds;
+		goto sched_out_proc;
+	    }
+
+	    ASSERT(state & ERTS_PSFLG_RUNNING_SYS);
+	    ASSERT(!(state & ERTS_PSFLG_RUNNING));
+
+	    while (1) {
+		erts_aint32_t n, e;
+
+		if (((state & (ERTS_PSFLG_SUSPENDED
+			       | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
+		    && !(state & ERTS_PSFLG_EXITING))
+		    goto sched_out_proc;
+
+		n = e = state;
+		n &= ~ERTS_PSFLG_RUNNING_SYS;
+		n |= ERTS_PSFLG_RUNNING;
+
+		state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+		if (state == e) {
+		    state = n;
+		    break;
+		}
+
+		ASSERT(state & ERTS_PSFLG_RUNNING_SYS);
+		ASSERT(!(state & ERTS_PSFLG_RUNNING));
+	    }
+	}
+
 	if (!(state & ERTS_PSFLG_EXITING)
 	    && ((FLAGS(p) & F_FORCE_GC)
 		|| (MSO(p).overhead > BIN_VHEAP_SZ(p)))) {
 	    reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity);
-	    if (reds < 0) {
-		reds = 1;
+	    if (reds <= 0) {
+		p->fcalls = reds;
+		goto sched_out_proc;
 	    }
 	}
+
+	if (proxy_p) {
+	    free_proxy_proc(proxy_p);
+	    proxy_p = NULL;
+	}
 	    
 	p->fcalls = reds;
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+
+	/* Never run a suspended process */
+	ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
+
 	return p;
     }
 }
 
+static int
+notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
+{
+    Process *rp = erts_proc_lookup(st->requester);
+    if (rp) {
+	ErtsProcLocks rp_locks;
+	ErlOffHeap *ohp;
+	ErlHeapFragment* bp;
+	Eterm *hp, msg, req_id, result;
+	Uint st_result_sz, hsz;
+#ifdef DEBUG
+	Eterm *hp_start;
+#endif
+
+	rp_locks = (c_p == rp) ? ERTS_PROC_LOCK_MAIN : 0;
+
+	st_result_sz = is_immed(st_result) ? 0 : size_object(st_result);
+	hsz = st->req_id_sz + st_result_sz + 4 /* 3-tuple */;
+
+	hp = erts_alloc_message_heap(hsz,
+				     &bp,
+				     &ohp,
+				     rp,
+				     &rp_locks);
+
+#ifdef DEBUG
+	hp_start = hp;
+#endif
+
+	req_id = st->req_id_sz == 0 ? st->req_id : copy_struct(st->req_id,
+							       st->req_id_sz,
+							       &hp,
+							       ohp);
+
+	result = st_result_sz == 0 ? st_result :  copy_struct(st_result,
+							      st_result_sz,
+							      &hp,
+							      ohp);
+
+	ASSERT(is_immed(st->reply_tag));
+
+	msg = TUPLE3(hp, st->reply_tag, req_id, result);
+
+#ifdef DEBUG
+	hp += 4;
+	ASSERT(hp_start + hsz == hp);
+#endif
+
+	erts_queue_message(rp,
+			   &rp_locks,
+			   bp,
+			   msg,
+			   NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+	    );
+
+	if (c_p == rp)
+	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+
+	if (rp_locks)
+	    erts_smp_proc_unlock(rp, rp_locks);
+    }
+
+    erts_cleanup_offheap(&st->off_heap);
+
+    erts_free(ERTS_ALC_T_PROC_SYS_TSK, st);
+
+    return rp ? 1 : 0;
+}
+
+static ERTS_INLINE ErtsProcSysTask *
+fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp)
+{
+    ErtsProcSysTaskQs *unused_qs = NULL;
+    int qbit, qmask;
+    ErtsProcSysTask *st, **qp;
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+
+    if (!c_p->sys_task_qs) {
+	qmask = 0;
+	st = NULL;
+	goto update_state;
+    }
+
+    qmask = c_p->sys_task_qs->qmask;
+
+    if ((state & (ERTS_PSFLG_ACTIVE
+		  | ERTS_PSFLG_EXITING
+		  | ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
+	/* No sys tasks if we got exclusively higher prio user work to do */
+	st = NULL;
+	switch (ERTS_PSFLGS_GET_USR_PRIO(state)) {
+	case PRIORITY_MAX:
+	    if (!(qmask & MAX_BIT))
+		goto done;
+	    break;
+	case PRIORITY_HIGH:
+	    if (!(qmask & (MAX_BIT|HIGH_BIT)))
+		goto done;
+	    break;
+	default:
+	    break;
+	}
+    }
+
+    qbit = qmask & -qmask;
+    switch (qbit) {
+    case MAX_BIT:
+	qp = &c_p->sys_task_qs->q[PRIORITY_MAX];
+	break;
+    case HIGH_BIT:
+	qp = &c_p->sys_task_qs->q[PRIORITY_HIGH];
+	break;
+    case NORMAL_BIT:
+	if (!(qmask & PRIORITY_LOW)
+	    || ++c_p->sys_task_qs->ncount <= RESCHEDULE_LOW) {
+	    qp = &c_p->sys_task_qs->q[PRIORITY_NORMAL];
+	    break;
+	}
+	c_p->sys_task_qs->ncount = 0;
+	/* Fall through */
+    case LOW_BIT:
+	qp = &c_p->sys_task_qs->q[PRIORITY_LOW];
+	break;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid qmask");
+    }
+
+    st = *qp;
+    ASSERT(st);
+    if (st->next != st) {
+	*qp = st->next;
+	st->next->prev = st->prev;
+	st->prev->next = st->next;
+    }
+    else {
+	erts_aint32_t a, e, n, st_prio, qmask2;
+
+	*qp = NULL;
+	qmask &= ~qbit;
+	c_p->sys_task_qs->qmask = qmask;
+
+    update_state:
+
+	qmask2 = qmask;
+
+	switch (qmask2 & -qmask2) {
+	case MAX_BIT:
+	    st_prio = PRIORITY_MAX;
+	    break;
+	case HIGH_BIT:
+	    st_prio = PRIORITY_HIGH;
+	    break;
+	case NORMAL_BIT:
+	    st_prio = PRIORITY_NORMAL;
+	    break;
+	case LOW_BIT:
+	    st_prio = PRIORITY_LOW;
+	    break;
+	case 0:
+	    st_prio = PRIORITY_LOW;
+	    unused_qs = c_p->sys_task_qs;
+	    c_p->sys_task_qs = NULL;
+	    break;
+	default:
+	    ERTS_INTERNAL_ERROR("Invalid qmask");
+	}
+
+	a = state;
+	do {
+	    erts_aint32_t prio = ERTS_PSFLGS_GET_USR_PRIO(a);
+
+	    if (prio > st_prio)
+		prio = st_prio;
+
+	    n = e = a;
+
+	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
+	    n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
+
+	    if (!qmask)
+		n &= ~ERTS_PSFLG_ACTIVE_SYS;
+
+	    if (a == n)
+		break;
+	    a = erts_smp_atomic32_cmpxchg_nob(&c_p->state, n, e);
+	} while (a != e);
+    }
+
+done:
+
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+
+    if (unused_qs)
+	proc_sys_task_queues_free(unused_qs);
+
+    *qmaskp = qmask;
+
+    return st;
+}
+
+static int
+execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
+{
+    int garbage_collected = 0;
+    erts_aint32_t state = *statep;
+    int max_reds = in_reds;
+    int reds = 0;
+    int qmask = 0;
+
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+
+    do {
+	ErtsProcSysTask *st;
+	Eterm st_res;
+
+	if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
+#ifdef ERTS_SMP
+	    if (state & ERTS_PSFLG_PENDING_EXIT)
+		erts_handle_pending_exit(c_p, ERTS_PROC_LOCK_MAIN);
+#endif
+	    ASSERT(ERTS_PROC_IS_EXITING(c_p));
+	    break;
+	}
+
+	st = fetch_sys_task(c_p, state, &qmask);
+	if (!st)
+	    break;
+
+	switch (st->type) {
+	case ERTS_PSTT_GC:
+	    if (!garbage_collected) {
+		FLAGS(c_p) |= F_NEED_FULLSWEEP;
+		reds += erts_garbage_collect(c_p, 0, c_p->arg_reg, c_p->arity);
+		garbage_collected = 1;
+	    }
+	    st_res = am_true;
+	    break;
+	case ERTS_PSTT_CPC:
+	    st_res = erts_check_process_code(c_p,
+					     st->arg[0],
+					     st->arg[1] == am_true,
+					     &reds);
+	    break;
+	default:
+	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
+	    st_res = am_false;
+	}
+
+	if (st)
+	    reds += notify_sys_task_executed(c_p, st, st_res);
+
+	state = erts_smp_atomic32_read_acqb(&c_p->state);
+    } while (qmask && reds < max_reds);
+
+    *statep = state;
+
+    return reds;
+}
+
+static int
+cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
+{
+    erts_aint32_t state = in_state;
+    int max_reds = in_reds;
+    int reds = 0;
+    int qmask = 0;
+
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+
+    do {
+	ErtsProcSysTask *st;
+	Eterm st_res;
+
+	st = fetch_sys_task(c_p, state, &qmask);
+	if (!st)
+	    break;
+
+	switch (st->type) {
+	case ERTS_PSTT_GC:
+	    st_res = am_false;
+	    break;
+	case ERTS_PSTT_CPC:
+	    st_res = am_false;
+	    break;
+	default:
+	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
+	    st_res = am_false;
+	    break;
+	}
+
+	reds += notify_sys_task_executed(c_p, st, st_res);
+
+	state = erts_smp_atomic32_read_acqb(&c_p->state);
+    } while (qmask && reds < max_reds);
+
+    return reds;
+}
+
+BIF_RETTYPE
+erts_internal_request_system_task_3(BIF_ALIST_3)
+{
+    Process *rp = erts_proc_lookup(BIF_ARG_1);
+    ErtsProcSysTaskQs *stqs, *free_stqs = NULL;
+    ErtsProcSysTask *st = NULL;
+    erts_aint32_t prio, rp_state;
+    int rp_locked;
+    Eterm noproc_res, req_type;
+
+    if (!rp && !is_internal_pid(BIF_ARG_1)) {
+	if (!is_external_pid(BIF_ARG_1))
+	    goto badarg;
+	if (external_pid_dist_entry(BIF_ARG_1) != erts_this_dist_entry)
+	    goto badarg;
+    }
+
+    switch (BIF_ARG_2) {
+    case am_max:	prio = PRIORITY_MAX;	break;
+    case am_high:	prio = PRIORITY_HIGH;	break;
+    case am_normal:	prio = PRIORITY_NORMAL;	break;
+    case am_low:	prio = PRIORITY_LOW;	break;
+    default: goto badarg;
+    }
+
+    if (is_not_tuple(BIF_ARG_3))
+	goto badarg;
+    else {
+	int i;
+	Eterm *tp = tuple_val(BIF_ARG_3);
+	Uint arity = arityval(*tp);
+	Eterm req_id;
+	Uint req_id_sz;
+	Eterm arg[ERTS_MAX_PROC_SYS_TASK_ARGS];
+	Uint arg_sz[ERTS_MAX_PROC_SYS_TASK_ARGS];
+	Uint tot_sz;
+	Eterm *hp;
+
+	if (arity < 2)
+	    goto badarg;
+	if (arity > 2 + ERTS_MAX_PROC_SYS_TASK_ARGS)
+	    goto badarg;
+	req_type = tp[1];
+	req_id = tp[2];
+	req_id_sz = is_immed(req_id) ? req_id : size_object(req_id);
+	tot_sz = req_id_sz;
+	for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) {
+	    int tix = 3 + i;
+	    if (tix > arity) {
+		arg[i] = THE_NON_VALUE;
+		arg_sz[i] = 0;
+	    }
+	    else {
+		arg[i] = tp[tix];
+		if (is_immed(arg[i]))
+		    arg_sz[i] = 0;
+		else {
+		    arg_sz[i] = size_object(arg[i]);
+		    tot_sz += arg_sz[i];
+		}
+	    }
+	}
+	st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK,
+			ERTS_PROC_SYS_TASK_SIZE(tot_sz));
+	st->next = st->prev = st; /* Prep for empty prio queue */
+	ERTS_INIT_OFF_HEAP(&st->off_heap);
+	hp = &st->heap[0];
+
+	st->requester = BIF_P->common.id;
+	st->reply_tag = req_type;
+	st->req_id_sz = req_id_sz;
+	st->req_id = req_id_sz == 0 ? req_id : copy_struct(req_id,
+							   req_id_sz,
+							   &hp,
+							   &st->off_heap);
+
+	for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++)
+	    st->arg[i] = arg_sz[i] == 0 ? arg[i] : copy_struct(arg[i],
+							       arg_sz[i],
+							       &hp,
+							       &st->off_heap);
+	ASSERT(&st->heap[0] + tot_sz == hp);
+    }
+
+    switch (req_type) {
+
+    case am_garbage_collect:
+	st->type = ERTS_PSTT_GC;
+	noproc_res = am_false;
+	if (!rp)
+	    goto noproc;
+	break;
+
+    case am_check_process_code:
+	if (is_not_atom(st->arg[0]))
+	    goto badarg;
+	if (st->arg[1] != am_true && st->arg[1] != am_false)
+	    goto badarg;
+	noproc_res = am_false;
+	st->type = ERTS_PSTT_CPC;
+	if (!rp)
+	    goto noproc;
+	break;
+
+    default:
+	goto badarg;
+    }
+
+    rp_state = erts_smp_atomic32_read_nob(&rp->state);
+
+    rp_locked = 0;
+
+    free_stqs = NULL;
+    if (rp_state & ERTS_PSFLG_ACTIVE_SYS)
+	stqs = NULL;
+    else {
+    alloc_qs:
+	stqs = proc_sys_task_queues_alloc();
+	stqs->qmask = 1 << prio;
+	stqs->ncount = 0;
+	stqs->q[PRIORITY_MAX] = NULL;
+	stqs->q[PRIORITY_HIGH] = NULL;
+	stqs->q[PRIORITY_NORMAL] = NULL;
+	stqs->q[PRIORITY_LOW] = NULL;
+	stqs->q[prio] = st;
+    }
+
+    if (!rp_locked) {
+	rp_locked = 1;
+	erts_smp_proc_lock(rp, ERTS_PROC_LOCK_STATUS);
+
+	rp_state = erts_smp_atomic32_read_nob(&rp->state);
+	if (rp_state & ERTS_PSFLG_EXITING) {
+	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+	    rp = NULL;
+	    free_stqs = stqs;
+	    goto noproc;
+	}
+    }
+
+    if (!rp->sys_task_qs) {
+	if (stqs)
+	    rp->sys_task_qs = stqs;
+	else
+	    goto alloc_qs;
+    }
+    else {
+	if (stqs)
+	    free_stqs = stqs;
+	stqs = rp->sys_task_qs;
+	if (!stqs->q[prio]) {
+	    stqs->q[prio] = st;
+	    stqs->qmask |= 1 << prio;
+	}
+	else {
+	    st->next = stqs->q[prio];
+	    st->prev = stqs->q[prio]->prev;
+	    st->next->prev = st;
+	    st->prev->next = st;
+	    ASSERT(stqs->qmask & (1 << prio));
+	}
+    }
+
+    if (ERTS_PSFLGS_GET_ACT_PRIO(rp_state) > prio) {
+	erts_aint32_t n, a, e;
+	/* Need to elevate actual prio */
+
+	a = rp_state;
+	do {
+	    if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) {
+		n = a;
+		break;
+	    }
+	    n = e = a;
+	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
+	    n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
+	    a = erts_smp_atomic32_cmpxchg_nob(&rp->state, n, e);    
+	} while (a != e);
+	rp_state = n;
+    }
+
+    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+
+    schedule_process_sys_task(rp, rp_state, NULL);
+
+    if (free_stqs)
+	proc_sys_task_queues_free(free_stqs);
+
+    BIF_RET(am_ok);
+
+noproc:
+
+    notify_sys_task_executed(BIF_P, st, noproc_res);
+    if (free_stqs)
+	proc_sys_task_queues_free(free_stqs);
+    BIF_RET(am_ok);
+
+badarg:
+
+    if (st) {
+	erts_cleanup_offheap(&st->off_heap);
+	erts_free(ERTS_ALC_T_PROC_SYS_TSK, st);
+    }
+    if (free_stqs)
+	proc_sys_task_queues_free(free_stqs);
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 void
 erts_sched_stat_modify(int what)
 {
@@ -7520,7 +8463,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	prio = (erts_aint32_t) so->priority;
     }
 
-    state |= (prio & ERTS_PSFLG_PRIO_MASK);
+    state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET)
+	      | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET));
 
     if (!rq)
 	rq = erts_get_runq_proc(parent);
@@ -7599,6 +8543,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->bin_old_vheap    = 0;
     p->bin_vheap_mature = 0;
 
+    p->sys_task_qs = NULL;
+
     /* No need to initialize p->fcalls. */
 
     p->current = p->initial+INITIAL_MOD;
@@ -7764,7 +8710,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
      * Schedule process for execution.
      */
 
-    schedule_process(p, state, 0);
+    schedule_process(p, state);
 
     VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id));
 
@@ -7815,6 +8761,7 @@ void erts_init_empty_process(Process *p)
     p->bin_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap = 0;
+    p->sys_task_qs = NULL;
     p->bin_vheap_mature = 0;
 #ifdef ERTS_SMP
     p->common.u.alive.ptimer = NULL;
@@ -8070,33 +9017,21 @@ delete_process(Process* p)
     p->fvalue = NIL;
 }
 
-static ERTS_INLINE erts_aint32_t
-set_proc_exiting_state(Process *p, erts_aint32_t state)
-{
-    erts_aint32_t a, n, e;
-    a = state;
-    while (1) {
-	n = e = a;
-	n &= ~(ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT);
-	n |= ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE;
-	if (!(a & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING)))
-	    n |= ERTS_PSFLG_IN_RUNQ;
-	a = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e);
-	if (a == e)
-	    break;
-    }
-    return a;
-}
-
 static ERTS_INLINE void
 set_proc_exiting(Process *p,
-		 erts_aint32_t state,
+		 erts_aint32_t in_state,
 		 Eterm reason,
 		 ErlHeapFragment *bp)
 {
+    erts_aint32_t state = in_state, enq_prio = -1;
+    int enqueue;
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL);
 
-    state = set_proc_exiting_state(p, state);
+    enqueue = change_proc_schedule_state(p,
+					 ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT,
+					 ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
+					 &state,
+					 &enq_prio);
 
     p->fvalue = reason;
     if (bp)
@@ -8111,15 +9046,37 @@ set_proc_exiting(Process *p,
     cancel_timer(p);
     p->i = (BeamInstr *) beam_exit;
 
-    if (erts_system_profile_flags.runnable_procs
-	&& !(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) {
-    	profile_runnable_proc(p, am_active);
-    }
-
-    if (!(state & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING)))
-	add2runq(p, state);
+    if (enqueue)
+	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
+		 state,
+		 enq_prio);
 }
 
+static ERTS_INLINE erts_aint32_t
+set_proc_self_exiting(Process *c_p)
+{
+#ifdef DEBUG
+    int enqueue;
+#endif
+    erts_aint32_t state, enq_prio = -1;
+
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL);
+
+    state = erts_smp_atomic32_read_nob(&c_p->state);
+    ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
+
+#ifdef DEBUG
+    enqueue =
+#endif
+	change_proc_schedule_state(c_p,
+				   ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT,
+				   ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
+				   &state,
+				   &enq_prio);
+
+    ASSERT(!enqueue);
+    return state;
+}
 
 #ifdef ERTS_SMP
 
@@ -8167,7 +9124,7 @@ handle_pending_exiters(ErtsProcList *pnd_xtrs)
 	if (p) {
 	    if (erts_proclist_same(plp, p)) {
 		erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
-		if (!(state & ERTS_PSFLG_RUNNING)) {
+		if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
 		    ASSERT(state & ERTS_PSFLG_PENDING_EXIT);
 		    erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL);
 		}
@@ -8388,7 +9345,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		}
 		set_proc_exiting(c_p, state, rsn, NULL);
 	    }
-	    else if (!(state & ERTS_PSFLG_RUNNING)) {
+	    else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
 		/* Process not running ... */
 		ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
 		if (need_locks
@@ -8765,9 +9722,6 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p)
 void 
 erts_do_exit_process(Process* p, Eterm reason)
 {
-#ifdef ERTS_SMP
-    erts_aint32_t state;
-#endif
     p->arity = 0;		/* No live registers */
     p->fvalue = reason;
 
@@ -8792,10 +9746,9 @@ erts_do_exit_process(Process* p, Eterm reason)
 #endif
 
 #ifndef ERTS_SMP
-    set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state));
+    set_proc_self_exiting(p);
 #else
-    state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state));
-    if (state & ERTS_PSFLG_PENDING_EXIT) {
+    if (ERTS_PSFLG_PENDING_EXIT & set_proc_self_exiting(p)) {
 	/* Process exited before pending exit was received... */
 	p->pending_exit.reason = THE_NON_VALUE;
 	if (p->pending_exit.bp) {
@@ -8849,6 +9802,7 @@ erts_continue_exit_process(Process *p)
     DistEntry *dep;
     struct saved_calls *scb;
     process_breakpoint_time_t *pbt;
+    erts_aint32_t state;
 
 #ifdef DEBUG
     int yield_allowed = 1;
@@ -8885,6 +9839,12 @@ erts_continue_exit_process(Process *p)
 	p->flags &= ~F_USING_DB;
     }
 
+    state = erts_smp_atomic32_read_acqb(&p->state);
+    if (state & ERTS_PSFLG_ACTIVE_SYS) {
+	if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2)
+	    goto yield;
+    }
+
     if (p->flags & F_USING_DDLL) {
 	erts_ddll_proc_dead(p, ERTS_PROC_LOCK_MAIN);
 	p->flags &= ~F_USING_DDLL;
@@ -8962,17 +9922,31 @@ erts_continue_exit_process(Process *p)
     {
 	/* Inactivate and notify free */
 	erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state);
+#ifdef ERTS_SMP
+	int refc_inced = 0;
+#endif
 	while (1) {
 	    n = e = a;
 	    ASSERT(a & ERTS_PSFLG_EXITING);
 	    n |= ERTS_PSFLG_FREE;
 	    n &= ~ERTS_PSFLG_ACTIVE;
+#ifdef ERTS_SMP
+	    if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) {
+		erts_smp_proc_inc_refc(p);
+		refc_inced = 1;
+	    }
+#endif
 	    a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	    if (a == e)
 		break;
 	}
-    }
 
+#ifdef ERTS_SMP
+	if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ))
+	    erts_smp_proc_dec_refc(p);
+#endif
+    }
+    
     dep = ((p->flags & F_DISTRIBUTION)
 	   ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL)
 	   : NULL);
@@ -9075,7 +10049,7 @@ timeout_proc(Process* p)
 
     state = erts_smp_atomic32_read_acqb(&p->state);
     if (!(state & ERTS_PSFLG_ACTIVE))
-	schedule_process(p, state, 0);
+	schedule_process(p, state);
 }
 
 
@@ -9153,7 +10127,9 @@ erts_program_counter_info(int to, void *to_arg, Process *p)
     print_function_from_pc(to, to_arg, p->cp);
     erts_print(to, to_arg, ")\n");
     state = erts_smp_atomic32_read_acqb(&p->state);
-    if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_GC))) {
+    if (!(state & (ERTS_PSFLG_RUNNING
+		   | ERTS_PSFLG_RUNNING_SYS
+		   | ERTS_PSFLG_GC))) {
         erts_print(to, to_arg, "arity = %d\n",p->arity);
 	if (!ERTS_IS_CRASH_DUMPING) {
 	    /*
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8e5467f196..814e3d34df 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -688,6 +688,9 @@ typedef struct {
     ErlHeapFragment *bp;
 } ErtsPendExit;
 
+typedef struct ErtsProcSysTask_ ErtsProcSysTask;
+typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;
+
 #ifdef ERTS_SMP
 
 typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
@@ -855,6 +858,7 @@ struct process {
     Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
     Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
+    ErtsProcSysTaskQs *sys_task_qs;
     erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
 
 #ifdef ERTS_SMP
@@ -924,24 +928,66 @@ void erts_check_for_holes(Process* p);
 #  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
 #endif
 
-#define ERTS_PSFLG_PRIO_SHIFT 2
+#define ERTS_PSFLGS_PRIO_BITS 2
+#define ERTS_PSFLGS_PRIO_MASK \
+    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)
 
-#define ERTS_PSFLG_BIT(N) \
-    (((erts_aint32_t) 1) << (ERTS_PSFLG_PRIO_SHIFT + (N)))
+#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
+#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
+#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
+#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)
 
-#define ERTS_PSFLG_PRIO_MASK 		(ERTS_PSFLG_BIT(0) - 1)
+#define ERTS_PSFLGS_QMASK_BITS 4
+#define ERTS_PSFLGS_QMASK \
+    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
+#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
+    ERTS_PSFLGS_ZERO_BIT_OFFSET
 
-#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(0)
-#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(1)
-#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(2)
-#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(3)
-#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(4)
-#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(5)
-#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(6)
-#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(7)
-#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(8)
-#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(9)
+#define ERTS_PSFLG_BIT(N) \
+    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))
 
+/*
+ * ACT_PRIO -> Active prio, i.e., currently active prio. This
+ *             prio may be higher than user prio.
+ * USR_PRIO -> User prio. i.e., prio the user has set.
+ * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
+ *             enqueued in. 
+ */
+#define ERTS_PSFLGS_ACT_PRIO_MASK \
+    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
+#define ERTS_PSFLGS_USR_PRIO_MASK \
+    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
+#define ERTS_PSFLGS_PRQ_PRIO_MASK \
+    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
+#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
+#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
+#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
+#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
+#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
+#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
+#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
+#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
+#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
+#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
+#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
+#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
+#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
+#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
+#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
+#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
+#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
+
+#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
+					 | ERTS_PSFLG_IN_PRQ_HIGH	\
+					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
+					 | ERTS_PSFLG_IN_PRQ_LOW)
+
+#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
+    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
+#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
+    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
+#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
+    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
 
 /* The sequential tracing token is a tuple of size 5:
  *
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bacd5a5752..885f092aca 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -655,6 +655,10 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg);
 
 Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 
+/* beam_bif_load.c */
+Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp);
+
+
 /* beam_load.c */
 typedef struct {
     BeamInstr* current;		/* Pointer to: Mod, Name, Arity */
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 1e5ae46bfa..c29f3f9b1b 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -763,17 +763,17 @@ allocate_init t I y
 #################################################################
 
 #
-# The BIFs erlang:check_process_code/2 must be called like a function,
+# The BIFs erts_internal:check_process_code/2 must be called like a function,
 # to ensure that c_p->i (program counter) is set correctly (an ordinary
 # BIF call doesn't set it).
 #
 
-call_ext u==2 Bif=u$bif:erlang:check_process_code/2 => i_call_ext Bif
-call_ext_last u==2 Bif=u$bif:erlang:check_process_code/2 D => i_call_ext_last Bif D
-call_ext_only u==2 Bif=u$bif:erlang:check_process_code/2 => i_call_ext_only Bif
+call_ext u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext Bif
+call_ext_last u==2 Bif=u$bif:erts_internal:check_process_code/2 D => i_call_ext_last Bif D
+call_ext_only u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext_only Bif
 
 #
-# The BIFs erlang:garbage_collect/0,1 must be called like functions,
+# The BIFs erlang:garbage_collect/0 must be called like a function,
 # to allow them to invoke the garbage collector. (The stack pointer must
 # be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.)
 #
@@ -782,10 +782,6 @@ call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif
 call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D
 call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif
 
-call_ext u==1 Bif=u$bif:erlang:garbage_collect/1 => i_call_ext Bif
-call_ext_last u==1 Bif=u$bif:erlang:garbage_collect/1 D => i_call_ext_last Bif D
-call_ext_only u==1 Bif=u$bif:erlang:garbage_collect/1 => i_call_ext_only Bif
-
 #
 # put/2 and erase/1 must be able to do garbage collection, so we must call
 # them like functions.
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index bd2be7afca..a5b2cc589c 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1675,7 +1675,7 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
 	p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
 	if (p) {
 	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
-	    if (state & ERTS_PSFLG_RUNNING)
+	    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))
 		p = NULL;
 	}
     }
-- 
cgit v1.2.3


From c6cb0293ba33e1671f8ed670d5add082e5ee674a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 30 Oct 2013 17:29:18 +0100
Subject: Functionality for disabling garbage collection

Being able to disable garbage collection over context
switches vastly simplifies implementation of yielding
native code that builds large or complex data structures
on the heap. This since the heap can be left in an
inconsistent state over the context switch.
---
 erts/emulator/beam/beam_bif_load.c |  12 ++
 erts/emulator/beam/erl_bif_info.c  |  13 ++
 erts/emulator/beam/erl_debug.c     |   3 +
 erts/emulator/beam/erl_gc.c        |  13 +-
 erts/emulator/beam/erl_process.c   | 251 ++++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_process.h   |  15 ++-
 erts/emulator/beam/global.h        |   5 +
 7 files changed, 296 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5239940a50..3f92c5b025 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -495,6 +495,8 @@ BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2)
 
     res = erts_check_process_code(BIF_P, BIF_ARG_1, allow_gc, &reds);
 
+    ASSERT(is_value(res));
+
     BIF_RET2(res, reds);
 
 badarg:
@@ -784,6 +786,16 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	}
     }
 
+    if (rp->flags & F_DISABLE_GC) {
+	/*
+	 * Cannot proceed. Process has disabled gc in order to
+	 * safely leave inconsistent data on the heap and/or
+	 * off heap lists. Need to wait for gc to be enabled
+	 * again.
+	 */ 
+	return THE_NON_VALUE;
+    }
+
     /*
      * See if there are funs that refer to the old version of the module.
      */
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 673dfc658c..f5893f9291 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3593,6 +3593,19 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		BIF_RET(am_true);
 	    }
 	}
+	else if (ERTS_IS_ATOM_STR("gc_state", BIF_ARG_1)) {
+	    /* Used by process_SUITE (emulator) */
+	    int res, enable;
+
+	    switch (BIF_ARG_2) {
+	    case am_true: enable = 1; break;
+	    case am_false: enable = 0; break;
+	    default: BIF_ERROR(BIF_P, BADARG); break;
+	    }
+ 
+	    res = erts_set_gc_state(BIF_P, enable);
+	    BIF_RET(res ? am_true : am_false);
+	}
 	else if (ERTS_IS_ATOM_STR("send_fake_exit_signal", BIF_ARG_1)) {
 	    /* Used by signal_SUITE (emulator) */
 
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index b90d00f236..dc79d45be7 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -299,6 +299,9 @@ void erts_check_for_holes(Process* p)
     ErlHeapFragment* hf;
     Eterm* start;
 
+    if (p->flags & F_DISABLE_GC)
+	return;
+
     start = p->last_htop ? p->last_htop : HEAP_START(p);
     check_memory(start, HEAP_TOP(p));
     p->last_htop = HEAP_TOP(p);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8ba94d89e9..da254286c4 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -400,10 +400,16 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     Uint reclaimed_now = 0;
     int done = 0;
     Uint ms1, s1, us1;
-    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsSchedulerData *esdp;
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
+
+    if (p->flags & F_DISABLE_GC)
+	return 1;
+
+    esdp = erts_get_scheduler_data();
+
     if (IS_TRACED_FL(p, F_TRACE_GC)) {
         trace_gc(p, am_gc_start);
     }
@@ -532,6 +538,9 @@ erts_garbage_collect_hibernate(Process* p)
     Uint area_size;
     Sint offs;
 
+    if (p->flags & F_DISABLE_GC)
+	ERTS_INTERNAL_ERROR("GC disabled");
+
     /*
      * Preliminaries.
      */
@@ -667,6 +676,8 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     Uint n;
     struct erl_off_heap_header** prev;
 
+    if (p->flags & F_DISABLE_GC)
+	return;
     /*
      * Set GC state.
      */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ad806da660..13b18e9e0e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -513,6 +513,11 @@ erts_pre_init_process(void)
      erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
 	 = ERTS_PSD_CALL_TIME_BP_SET_LOCKS;
 
+     erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks
+	 = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS;
+     erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks
+	 = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS;
+
      /* Check that we have locks for all entries */
      for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
 	 ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks);
@@ -7033,7 +7038,7 @@ erts_set_process_priority(Process *p, Eterm value)
 	    oprio = ERTS_PSFLGS_GET_USR_PRIO(a);
 	    n = e = a;
 
-	    if (!(a & ERTS_PSFLG_ACTIVE_SYS))
+	    if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DELAYED_SYS)))
 		aprio = nprio;
 	    else {
 		int max_qbit;
@@ -7043,7 +7048,15 @@ erts_set_process_priority(Process *p, Eterm value)
 		    slocked = 1;
 		}
 
-		max_qbit = p->sys_task_qs->qmask;
+		max_qbit = 0;
+		if (a & ERTS_PSFLG_ACTIVE_SYS)
+		    max_qbit |= p->sys_task_qs->qmask;
+		if (a & ERTS_PSFLG_DELAYED_SYS) {
+		    ErtsProcSysTaskQs *qs;
+		    qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(p);
+		    ASSERT(qs);
+		    max_qbit |= qs->qmask;
+		}
 		max_qbit &= -max_qbit;
 		switch (max_qbit) {
 		case MAX_BIT:
@@ -7731,12 +7744,14 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 }
 
 static ERTS_INLINE ErtsProcSysTask *
-fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp)
+fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop)
 {
     ErtsProcSysTaskQs *unused_qs = NULL;
     int qbit, qmask;
     ErtsProcSysTask *st, **qp;
 
+    *priop = -1; /* Shut up annoying erroneous warning */
+
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
 
     if (!c_p->sys_task_qs) {
@@ -7770,20 +7785,24 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp)
     switch (qbit) {
     case MAX_BIT:
 	qp = &c_p->sys_task_qs->q[PRIORITY_MAX];
+	*priop = PRIORITY_MAX;
 	break;
     case HIGH_BIT:
 	qp = &c_p->sys_task_qs->q[PRIORITY_HIGH];
+	*priop = PRIORITY_HIGH;
 	break;
     case NORMAL_BIT:
 	if (!(qmask & PRIORITY_LOW)
 	    || ++c_p->sys_task_qs->ncount <= RESCHEDULE_LOW) {
 	    qp = &c_p->sys_task_qs->q[PRIORITY_NORMAL];
+	    *priop = PRIORITY_NORMAL;
 	    break;
 	}
 	c_p->sys_task_qs->ncount = 0;
 	/* Fall through */
     case LOW_BIT:
 	qp = &c_p->sys_task_qs->q[PRIORITY_LOW];
+	*priop = PRIORITY_LOW;
 	break;
     default:
 	ERTS_INTERNAL_ERROR("Invalid qmask");
@@ -7807,6 +7826,12 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp)
 
 	qmask2 = qmask;
 
+	if (state & ERTS_PSFLG_DELAYED_SYS) {
+	    ErtsProcSysTaskQs *qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p);
+	    ASSERT(qs);
+	    qmask2 |= qs->qmask;
+	}
+
 	switch (qmask2 & -qmask2) {
 	case MAX_BIT:
 	    st_prio = PRIORITY_MAX;
@@ -7818,17 +7843,18 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp)
 	    st_prio = PRIORITY_NORMAL;
 	    break;
 	case LOW_BIT:
-	    st_prio = PRIORITY_LOW;
-	    break;
 	case 0:
 	    st_prio = PRIORITY_LOW;
-	    unused_qs = c_p->sys_task_qs;
-	    c_p->sys_task_qs = NULL;
 	    break;
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid qmask");
 	}
 
+	if (!qmask) {
+	    unused_qs = c_p->sys_task_qs;
+	    c_p->sys_task_qs = NULL;
+	}
+
 	a = state;
 	do {
 	    erts_aint32_t prio = ERTS_PSFLGS_GET_USR_PRIO(a);
@@ -7862,6 +7888,8 @@ done:
     return st;
 }
 
+static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio);
+
 static int
 execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 {
@@ -7875,6 +7903,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 
     do {
 	ErtsProcSysTask *st;
+	int st_prio;
 	Eterm st_res;
 
 	if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
@@ -7886,24 +7915,39 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    break;
 	}
 
-	st = fetch_sys_task(c_p, state, &qmask);
+	st = fetch_sys_task(c_p, state, &qmask, &st_prio);
 	if (!st)
 	    break;
 
 	switch (st->type) {
 	case ERTS_PSTT_GC:
-	    if (!garbage_collected) {
-		FLAGS(c_p) |= F_NEED_FULLSWEEP;
-		reds += erts_garbage_collect(c_p, 0, c_p->arg_reg, c_p->arity);
-		garbage_collected = 1;
+	    if (c_p->flags & F_DISABLE_GC) {
+		save_gc_task(c_p, st, st_prio);
+		st = NULL;
+		reds++;
+	    }
+	    else {
+		if (!garbage_collected) {
+		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
+		    reds += erts_garbage_collect(c_p,
+						 0,
+						 c_p->arg_reg,
+						 c_p->arity);
+		    garbage_collected = 1;
+		}
+		st_res = am_true;
 	    }
-	    st_res = am_true;
 	    break;
 	case ERTS_PSTT_CPC:
 	    st_res = erts_check_process_code(c_p,
 					     st->arg[0],
 					     st->arg[1] == am_true,
 					     &reds);
+	    if (is_non_value(st_res)) {
+		/* Needed gc, but gc was disabled */
+		save_gc_task(c_p, st, st_prio);
+		st = NULL;
+	    }
 	    break;
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
@@ -7934,8 +7978,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
     do {
 	ErtsProcSysTask *st;
 	Eterm st_res;
+	int st_prio;
 
-	st = fetch_sys_task(c_p, state, &qmask);
+	st = fetch_sys_task(c_p, state, &qmask, &st_prio);
 	if (!st)
 	    break;
 
@@ -8168,6 +8213,183 @@ badarg:
     BIF_ERROR(BIF_P, BADARG);
 }
 
+static void
+save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
+{
+    erts_aint32_t state;
+    ErtsProcSysTaskQs *qs;
+
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+
+    qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p);
+    if (!qs) {
+	st->next = st->prev = st;
+	qs = proc_sys_task_queues_alloc();
+	qs->qmask = 1 << prio;
+	qs->ncount = 0;
+	qs->q[PRIORITY_MAX] = NULL;
+	qs->q[PRIORITY_HIGH] = NULL;
+	qs->q[PRIORITY_NORMAL] = NULL;
+	qs->q[PRIORITY_LOW] = NULL;
+	qs->q[prio] = st;
+	(void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, qs);
+    }
+    else {
+	if (!qs->q[prio]) {
+	    st->next = st->prev = st;
+	    qs->q[prio] = st;
+	    qs->qmask |= 1 << prio;
+	}
+	else {
+	    st->next = qs->q[prio];
+	    st->prev = qs->q[prio]->prev;
+	    st->next->prev = st;
+	    st->prev->next = st;
+	    ASSERT(qs->qmask & (1 << prio));
+	}
+    }
+
+    state = erts_smp_atomic32_read_nob(&c_p->state);
+    ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) & state);
+
+    while (!(state & ERTS_PSFLG_DELAYED_SYS)
+	   || prio < ERTS_PSFLGS_GET_ACT_PRIO(state)) {
+	erts_aint32_t n, e;
+
+	n = e = state;
+	n |= ERTS_PSFLG_DELAYED_SYS;
+	if (prio < ERTS_PSFLGS_GET_ACT_PRIO(state)) {
+	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
+	    n |= prio << ERTS_PSFLGS_ACT_PRIO_OFFSET;
+	}
+	state = erts_smp_atomic32_cmpxchg_relb(&c_p->state, n, e);
+	if (state == e)
+	    break;
+    }
+}
+
+int
+erts_set_gc_state(Process *c_p, int enable)
+{
+    int res;
+    ErtsProcSysTaskQs *dgc_tsk_qs;
+    ASSERT(c_p == erts_get_current_process());
+    ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
+	   & erts_smp_atomic32_read_nob(&c_p->state));
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+
+    res = !(c_p->flags & F_DISABLE_GC);
+
+    if (!enable) {
+	c_p->flags |= F_DISABLE_GC;
+	return res;
+    }
+
+    c_p->flags &= ~F_DISABLE_GC;
+
+    dgc_tsk_qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p);
+    if (!dgc_tsk_qs)
+	return res;
+
+    /* Move delayed gc tasks into sys tasks queues. */
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+
+    if (!c_p->sys_task_qs) {
+	c_p->sys_task_qs = dgc_tsk_qs;
+	dgc_tsk_qs = NULL;
+    }
+    else {
+	ErtsProcSysTaskQs *stsk_qs;
+	int prio;
+
+	/*
+	 * We push delayed tasks to the front of the queue
+	 * since they have already made it to the front
+	 * once and then been delayed after that.
+	 */
+
+	stsk_qs = c_p->sys_task_qs;
+
+	while (dgc_tsk_qs->qmask) {
+	    int qbit = dgc_tsk_qs->qmask & -dgc_tsk_qs->qmask;
+	    dgc_tsk_qs->qmask &= ~qbit;
+	    switch (qbit) {
+	    case MAX_BIT:
+		prio = PRIORITY_MAX;
+		break;
+	    case HIGH_BIT:
+		prio = PRIORITY_HIGH;
+		break;
+	    case NORMAL_BIT:
+		prio = PRIORITY_NORMAL;
+		break;
+	    case LOW_BIT:
+		prio = PRIORITY_LOW;
+		break;
+	    default:
+		ERTS_INTERNAL_ERROR("Invalid qmask");
+		prio = -1;
+		break;
+	    }
+
+	    ASSERT(dgc_tsk_qs->q[prio]);
+
+	    if (!stsk_qs->q[prio]) {
+		stsk_qs->q[prio] = dgc_tsk_qs->q[prio];
+		stsk_qs->qmask |= 1 << prio;
+	    }
+	    else {
+		ErtsProcSysTask *first1, *last1, *first2, *last2;
+
+		ASSERT(stsk_qs->qmask & (1 << prio));
+		first1 = dgc_tsk_qs->q[prio];
+		last1 = first1->prev;
+		first2 = stsk_qs->q[prio];
+		last2 = first1->prev;
+
+		last1->next = first2;
+		first2->prev = last1;
+
+		first1->prev = last2;
+		last2->next = first1;
+
+		stsk_qs->q[prio] = first1;
+	    }
+
+	}
+    }
+
+#ifdef DEBUG
+    {
+	int qmask;
+	erts_aint32_t aprio, state =
+#endif
+
+	    erts_smp_atomic32_read_bset_nob(&c_p->state,
+					    (ERTS_PSFLG_DELAYED_SYS
+					     | ERTS_PSFLG_ACTIVE_SYS),
+					    ERTS_PSFLG_ACTIVE_SYS);
+
+#ifdef DEBUG
+	ASSERT(state & ERTS_PSFLG_DELAYED_SYS);
+	qmask = c_p->sys_task_qs->qmask;
+	aprio = ERTS_PSFLGS_GET_ACT_PRIO(state);
+	ASSERT(ERTS_PSFLGS_GET_USR_PRIO(state) >= aprio);
+	ASSERT((qmask & -qmask) >= (1 << aprio));
+    }
+#endif
+
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+
+    (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+
+    if (dgc_tsk_qs)
+	proc_sys_task_queues_free(dgc_tsk_qs);
+
+    return res;
+}
+
 void
 erts_sched_stat_modify(int what)
 {
@@ -9839,6 +10061,7 @@ erts_continue_exit_process(Process *p)
 	p->flags &= ~F_USING_DB;
     }
 
+    erts_set_gc_state(p, 1);
     state = erts_smp_atomic32_read_acqb(&p->state);
     if (state & ERTS_PSFLG_ACTIVE_SYS) {
 	if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2)
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 814e3d34df..0b8d2976ed 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -631,8 +631,9 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_SCHED_ID			2
 #define ERTS_PSD_DIST_ENTRY			3
 #define ERTS_PSD_CALL_TIME_BP			4
+#define ERTS_PSD_DELAYED_GC_TASK_QS		5
 
-#define ERTS_PSD_SIZE				5
+#define ERTS_PSD_SIZE				6
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -656,6 +657,9 @@ typedef struct {
 #define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
+#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
+
 typedef struct {
     ErtsProcLocks get_locks;
     ErtsProcLocks set_locks;
@@ -859,6 +863,7 @@ struct process {
     Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     ErtsProcSysTaskQs *sys_task_qs;
+
     erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
 
 #ifdef ERTS_SMP
@@ -976,6 +981,7 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
+#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
 					 | ERTS_PSFLG_IN_PRQ_HIGH	\
@@ -1102,6 +1108,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
+#define F_DISABLE_GC         (1 << 11) /* Disable GC */
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
@@ -1192,6 +1199,7 @@ void erts_late_init_process(void);
 void erts_early_init_scheduling(int);
 void erts_init_scheduling(int, int);
 
+int erts_set_gc_state(Process *c_p, int enable);
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
 Eterm erts_gc_info_request(Process *c_p);
 Uint64 erts_get_proc_interval(void);
@@ -1637,6 +1645,11 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
   ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))
 
+#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
+    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
+#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
+    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))
+
 
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 885f092aca..6b720b53d8 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1114,7 +1114,12 @@ erts_alloc_message_heap_state(Uint size,
 	if (statep)
 	    *statep = state;
 	if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+	    || (receiver->flags & F_DISABLE_GC)
 	    || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
+	    /*
+	     * The heap is either potentially in an inconsistent
+	     * state, or not large enough.
+	     */
 #ifdef ERTS_SMP
 	    if (locked_main) {
 		*receiver_locks &= ~ERTS_PROC_LOCK_MAIN;
-- 
cgit v1.2.3


From 7f17d16335231b4bedd9d2cdd7507ab88a5bf021 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 8 Oct 2013 12:18:46 +0200
Subject: trapping binary_to_term passing binary_SUITE

---
 erts/emulator/beam/atom.names  |   1 +
 erts/emulator/beam/erl_debug.c |   2 +-
 erts/emulator/beam/external.c  | 299 ++++++++++++++++++++++++++++++++++-------
 erts/emulator/beam/external.h  |   1 +
 erts/emulator/beam/sys.h       |   6 +
 5 files changed, 256 insertions(+), 53 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 8433fd826a..71addedfdc 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -115,6 +115,7 @@ atom binary_longest_prefix_trap
 atom binary_longest_suffix_trap
 atom binary_match_trap
 atom binary_matches_trap
+atom binary_to_term_trap
 atom block
 atom blocked
 atom bm
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index dc79d45be7..873a9860da 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2013. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 1c88765381..b1cdad1343 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -61,6 +61,9 @@
  */
 #    define ERTS_DEBUG_USE_DIST_SEP
 #  endif
+#  define IF_DEBUG(X) X
+#else
+#  define IF_DEBUG(X)
 #endif
 
 /* Does Sint fit in Sint32?
@@ -89,7 +92,8 @@ static int enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 static Uint is_external_string(Eterm obj, int* p_is_string);
 static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
-static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
+struct B2TContext_t;
+static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
 static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
 static Sint decoded_size(byte *ep, byte* endp, int internal_tags);
@@ -102,11 +106,19 @@ static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
 static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, 
 				  unsigned dflags, Sint *reds, Uint *res);
 
+static Export binary_to_term_trap_export;
+static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
+static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b);
+
 void erts_init_external(void) {
 #if 1 /* In R16 */
     erts_init_trap_export(&term_to_binary_trap_export,
 			  am_erlang, am_term_to_binary_trap, 1,
 			  &term_to_binary_trap_1);
+
+    erts_init_trap_export(&binary_to_term_trap_export,
+			  am_erlang, am_binary_to_term_trap, 1,
+			  &binary_to_term_trap_1);
 #else
     sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export));
     term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3];
@@ -927,7 +939,7 @@ erts_decode_dist_ext(Eterm** hpp,
 	    goto error;
 	ep++;
     }
-    ep = dec_term(edep, hpp, ep, off_heap, &obj);
+    ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL);
     if (!ep)
 	goto error;
 
@@ -948,7 +960,7 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
     byte *ep = *ext;
     if (*ep++ != VERSION_MAGIC)
 	return THE_NON_VALUE;
-    ep = dec_term(NULL, hpp, ep, off_heap, &obj);
+    ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL);
     if (!ep) {
 #ifdef DEBUG
 	bin_write(ERTS_PRINT_STDERR,NULL,*ext,500);
@@ -962,7 +974,7 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
 Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
 {
     Eterm obj;
-    ext = dec_term(NULL, hpp, ext, off_heap, &obj);
+    ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL);
     ASSERT(ext);
     return obj;
 }
@@ -1112,6 +1124,46 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
     }
 }
 
+
+enum B2TState { /*B2TUncompress,*/ B2TSize, B2TDecodeInit, B2TDecode, B2TDecodeFail, B2TBadArg, B2TDone };
+
+typedef struct {
+} B2TSizeContext;
+
+typedef struct {
+    byte*  ep;
+    Eterm  res;
+    Eterm* next;
+    Eterm* hp_start;
+    Eterm* hp;
+    Eterm* hp_end;
+} B2TDecodeContext;
+
+typedef struct {
+    /*Uint real_size;
+    Uint dest_len;
+    byte *dbytes;
+    Binary *result_bin;
+    Binary *destination_bin;
+    z_stream stream;*/
+} B2TUncompressContext;
+
+typedef struct B2TContext_t {
+    Sint heap_size;
+    byte* aligned_alloc;
+    ErtsBinary2TermState b2ts;
+    //Uint ext_size;
+    Uint reds;
+    Eterm trap_bin;
+    enum B2TState state;
+    union {
+	B2TSizeContext sc;
+	B2TDecodeContext dc;
+	B2TUncompressContext uc;
+    } u;
+} B2TContext;
+
+
 static uLongf binary2term_uncomp_size(byte* data, Sint size)
 {
     z_stream stream;
@@ -1153,7 +1205,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
     if (size < 1 || *bytes != VERSION_MAGIC) {
     error:
 	if (state->exttmp)
-	    erts_free(ERTS_ALC_T_TMP, state->extp);
+	    erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
 	state->extp = NULL;
 	state->exttmp = 0;
 	return -1;
@@ -1168,17 +1220,18 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 	bytes += 5;
 	size -= 5;	
 	if (dest_len > 32*1024*1024
-	    || (state->extp = erts_alloc_fnf(ERTS_ALC_T_TMP, dest_len)) == NULL) {
+	    || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) {
 	    if (dest_len != binary2term_uncomp_size(bytes, size)) {
 		goto error;
 	    }
-	    state->extp = erts_alloc(ERTS_ALC_T_TMP, dest_len);
+	    state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len);
 	}
 	state->exttmp = 1;
 	if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK)
 	    goto error;
 	size = (Sint) dest_len;
     }
+    state->extsize = size;
     res = decoded_size(state->extp, state->extp + size, 0);
     if (res < 0)
 	goto error;
@@ -1190,7 +1243,7 @@ binary2term_abort(ErtsBinary2TermState *state)
 {
     if (state->exttmp) {
 	state->exttmp = 0;
-	erts_free(ERTS_ALC_T_TMP, state->extp);
+	erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
     }
 }
 
@@ -1198,11 +1251,11 @@ static ERTS_INLINE Eterm
 binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
 {
     Eterm res;
-    if (!dec_term(edep, hpp, state->extp, ohp, &res))
+    if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL))
 	res = THE_NON_VALUE;
     if (state->exttmp) {
 	state->exttmp = 0;
-	erts_free(ERTS_ALC_T_TMP, state->extp);
+	erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
     }
     return res;
 }
@@ -1225,46 +1278,155 @@ erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *oh
     return binary2term_create(NULL,state, hpp, ohp);
 }
 
+static void b2t_destroy_context(B2TContext* context)
+{
+    erts_free_aligned_binary_bytes_extra(context->aligned_alloc,
+                                         ERTS_ALC_T_EXT_TERM_DATA);
+    context->aligned_alloc = NULL;
+    binary2term_abort(&context->b2ts);
+}
+
+static void b2t_context_destructor(Binary *context_bin)
+{
+    B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin);
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
+
+    b2t_destroy_context(ctx);
+}
+
+static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
+{
+    Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
+
+    return binary_to_term_int(BIF_P, THE_NON_VALUE, context_bin);
+}
+
 BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
 {
-    Sint heap_size;
-    Eterm res;
-    Eterm* hp;
-    Eterm* endp;
-    Sint size;
+    return binary_to_term_int(BIF_P, BIF_ARG_1, NULL);
+}
+
+#define B2T_BYTES_PER_REDUCTION 100
+
+
+static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
+{
     byte* bytes;
-    byte* temp_alloc = NULL;
-    ErtsBinary2TermState b2ts;
+    B2TContext c_buff;
+    B2TContext *ctx;
+    Eterm* magic_space = NULL;
 
-    if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) {
-    error:
-	erts_free_aligned_binary_bytes(temp_alloc);
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    size = binary_size(BIF_ARG_1);
+    if (context_b == NULL) {
+	/* Setup enough to get started */
+        ctx = &c_buff;
+	ctx->state = B2TSize;
+        ctx->aligned_alloc = NULL;
+        IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
+    } else {
+	ctx = ERTS_MAGIC_BIN_DATA(context_b);
+    }
+    ctx->reds = 1; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/
+
+    do {
+        switch (ctx->state) {
+        case B2TSize:
+            bytes = erts_get_aligned_binary_bytes_extra(bin,
+                                                        &ctx->aligned_alloc,
+                                                        ERTS_ALC_T_EXT_TERM_DATA,
+                                                        0);
+            if (bytes == NULL) {
+                ctx->b2ts.exttmp = 0;
+                ctx->state = B2TBadArg;
+                break;
+            }
 
-    heap_size = binary2term_prepare(&b2ts, bytes, size);
-    if (heap_size < 0)
-	goto error;
+            ctx->heap_size = binary2term_prepare(&ctx->b2ts, bytes,
+                                                 binary_size(bin));
+            if (ctx->heap_size < 0) {
+                ctx->state = B2TBadArg;
+                break;
+            }
 
-    hp = HAlloc(BIF_P, heap_size);
-    endp = hp + heap_size;
 
-    res = binary2term_create(NULL, &b2ts, &hp, &MSO(BIF_P));
 
-    erts_free_aligned_binary_bytes(temp_alloc);
+            ctx->state = B2TDecodeInit;
 
-    if (hp > endp) {
-	erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
-		 __FILE__, __LINE__, hp-endp);
-    }
+            if (ctx->b2ts.extsize >= ctx->reds) {
+                ctx->reds = 0;
+                break;
+            }
+            ctx->reds -= ctx->b2ts.extsize;
+            /*fall through*/
+        case B2TDecodeInit:
+            if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) {
+                /* dec_term will probably trap, allocate space for magic bin
+                   before result term to make it easy to trim with HRelease.
+                 */
+                magic_space = HAlloc(p, PROC_BIN_SIZE);
+                *magic_space = make_pos_bignum_header(PROC_BIN_SIZE-1);
+            }
+            ctx->u.dc.ep = ctx->b2ts.extp;
+            ctx->u.dc.res = (Eterm) (UWord) NULL;
+            ctx->u.dc.next = &ctx->u.dc.res;
+            ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size);
+            ctx->u.dc.hp       = ctx->u.dc.hp_start;
+            ctx->u.dc.hp_end   = ctx->u.dc.hp_start + ctx->heap_size;
+            ctx->state = B2TDecode;
+            /*fall through*/
+        case B2TDecode:
+            dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx);
+            break;
+
+        case B2TDecodeFail:
+            HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start);
+            /*fall through*/
+        case B2TBadArg:
+            b2t_destroy_context(ctx);
+            if (context_b) {
+                erts_set_gc_state(p, 1);
+            }
+            BIF_ERROR(p, BADARG);
 
-    HRelease(BIF_P, endp, hp);
+        case B2TDone:
+            b2t_destroy_context(ctx);
 
-    if (res == THE_NON_VALUE)
-	goto error;
+            if (ctx->u.dc.hp > ctx->u.dc.hp_end) {
+                erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
+                         __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end);
+            }
+            HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp);
 
-    return res;
+            if (context_b) {
+                erts_set_gc_state(p, 1);
+            }
+
+            return ctx->u.dc.res;
+
+        }
+    }while (ctx->reds);
+
+    if (context_b == NULL) {
+        ASSERT(ctx->trap_bin == THE_NON_VALUE);
+
+        context_b = erts_create_magic_binary(sizeof(B2TContext),
+                                             b2t_context_destructor);
+        ctx = ERTS_MAGIC_BIN_DATA(context_b);
+        sys_memcpy(ctx, &c_buff, sizeof(B2TContext));
+
+        if (!magic_space) {
+            ASSERT(ctx->state != B2TDecode);
+            magic_space = HAlloc(p, PROC_BIN_SIZE);
+        }
+        ctx->trap_bin = erts_mk_magic_binary_term(&magic_space, &MSO(p), context_b);
+
+        erts_set_gc_state(p, 0);
+    }
+    ASSERT(context_b);
+    ASSERT(ctx->trap_bin != THE_NON_VALUE);
+
+    BUMP_ALL_REDS(p);
+    BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin);
 }
 
 BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
@@ -1514,7 +1676,7 @@ typedef struct {
     } s;
 } TTBContext;
 
-static void context_destructor(Binary *context_bin) 
+static void ttb_context_destructor(Binary *context_bin)
 {
     TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
     if (context->alive) {
@@ -1567,7 +1729,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
     do {								\
 	if (context_b == NULL) {					\
 	    context_b = erts_create_magic_binary(sizeof(TTBContext),    \
-                                                 context_destructor);   \
+                                                 ttb_context_destructor);   \
 	    context =  ERTS_MAGIC_BIN_DATA(context_b);			\
 	    memcpy(context,&c_buff,sizeof(TTBContext));			\
 	}								\
@@ -2608,15 +2770,34 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
 ** On failure return NULL and (R13B04) *hpp will be unchanged.
 */
 static byte*
-dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
+dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
+         Eterm* objp, B2TContext* ctx)
 {
-    Eterm* hp_saved = *hpp;
+    Eterm* hp_saved;
     int n;
     ErtsAtomEncoding char_enc;
-    register Eterm* hp = *hpp;	/* Please don't take the address of hp */
-    Eterm* next = objp;
-
-    *next = (Eterm) (UWord) NULL;
+    register Eterm* hp;        /* Please don't take the address of hp */
+    Eterm* next;
+    byte* ep_trap_limit;
+
+    if (ctx) {
+        hp_saved = ctx->u.dc.hp_start;
+        ep = ctx->u.dc.ep;
+        ep_trap_limit = ep + ctx->reds;
+        if (ep_trap_limit < ep) {
+            ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK Is there a safe way to create a "largest ptr" */
+        }
+        next = ctx->u.dc.next;
+        hpp = &ctx->u.dc.hp;
+    }
+    else {
+        hp_saved = *hpp;
+        ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK - " - */
+        next = objp;
+        *next = (Eterm) (UWord) NULL;
+    }
+    ASSERT(ep < ep_trap_limit);
+    hp = *hpp;
 
     while (next != NULL) {
 	objp = next;
@@ -3065,7 +3246,7 @@ dec_term_atom_common:
 		    goto error;
 		}
 		*hpp = hp;
-		ep = dec_term(edep, hpp, ep, off_heap, &temp);
+		ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL);
 		hp = *hpp;
 		if (ep == NULL) {
 		    goto error;
@@ -3125,7 +3306,7 @@ dec_term_atom_common:
 		}
 		*hpp = hp;
 		/* Index */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3134,7 +3315,7 @@ dec_term_atom_common:
 		old_index = unsigned_val(temp);
 
 		/* Uniq */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3202,7 +3383,7 @@ dec_term_atom_common:
 		}
 
 		/* Index */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3211,7 +3392,7 @@ dec_term_atom_common:
 		old_index = unsigned_val(temp);
 
 		/* Uniq */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) {
+		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3311,8 +3492,22 @@ dec_term_atom_common:
 	    }
 	    undo_offheap_in_area(off_heap, hp_saved, hp);
 	    *hpp = hp_saved;
-	    return NULL;
+            if (ctx) {
+                ctx->state = B2TDecodeFail;
+            }
+            return NULL;
 	}
+        if (ep > ep_trap_limit) {
+            ASSERT(ctx);
+	    ctx->u.dc.ep = ep;
+	    ctx->u.dc.next = next;
+	    ctx->u.dc.hp = hp;
+	    ctx->reds = 0;
+	    return NULL;
+        }
+    }
+    if (ctx) {
+        ctx->state = B2TDone;
     }
     *hpp = hp;
     return ep;
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index e37d47919e..9afdb52329 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -146,6 +146,7 @@ typedef struct {
 typedef struct {
     byte *extp;
     int exttmp;
+    Uint extsize;
 } ErtsBinary2TermState;
 
 /* -------------------------------------------------------------------------- */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 05bff430e3..11ba554a3d 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -282,16 +282,19 @@ typedef unsigned long UWord;
 typedef long          SWord;
 #define SWORD_CONSTANT(Const) Const##L
 #define UWORD_CONSTANT(Const) Const##UL
+#define ERTS_UWORD_MAX ULONG_MAX
 #elif SIZEOF_VOID_P == SIZEOF_INT
 typedef unsigned int UWord;
 typedef int          SWord;
 #define SWORD_CONSTANT(Const) Const
 #define UWORD_CONSTANT(Const) Const##U
+#define ERTS_UWORD_MAX UINT_MAX
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
 typedef unsigned long long UWord;
 typedef long long          SWord;
 #define SWORD_CONSTANT(Const) Const##LL
 #define UWORD_CONSTANT(Const) Const##ULL
+#define ERTS_UWORD_MAX ULLONG_MAX
 #else
 #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
 #endif
@@ -304,6 +307,7 @@ typedef unsigned long Uint;
 typedef long          Sint;
 #define SWORD_CONSTANT(Const) Const##L
 #define UWORD_CONSTANT(Const) Const##UL
+#define ERTS_UWORD_MAX ULONG_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_INT
@@ -312,6 +316,7 @@ typedef unsigned int Uint;
 typedef int          Sint;
 #define SWORD_CONSTANT(Const) Const
 #define UWORD_CONSTANT(Const) Const##U
+#define ERTS_UWORD_MAX UINT_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
@@ -320,6 +325,7 @@ typedef unsigned long long Uint;
 typedef long long          Sint;
 #define SWORD_CONSTANT(Const) Const##LL
 #define UWORD_CONSTANT(Const) Const##ULL
+#define ERTS_UWORD_MAX ULLONG_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
 #if defined(__WIN32__)
 #define ErtsStrToSint _strtoi64
-- 
cgit v1.2.3


From 78a5dd69c9ed1749f2ad4fd24151e4aa784b1cba Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Oct 2013 12:11:29 +0200
Subject: trapping lists and tuples

---
 erts/emulator/beam/external.c | 153 +++++++++++++++++++++++++++++++++++-------
 erts/emulator/beam/sys.h      |   6 ++
 2 files changed, 133 insertions(+), 26 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index b1cdad1343..48536d610b 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1125,7 +1125,18 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 }
 
 
-enum B2TState { /*B2TUncompress,*/ B2TSize, B2TDecodeInit, B2TDecode, B2TDecodeFail, B2TBadArg, B2TDone };
+enum B2TState {
+    /*B2TUncompress,*/
+    B2TSize,
+    B2TDecodeInit,
+    B2TDecode,
+    B2TDecodeList,
+    B2TDecodeTuple,
+
+    B2TDone,
+    B2TDecodeFail,
+    B2TBadArg
+};
 
 typedef struct {
 } B2TSizeContext;
@@ -1137,6 +1148,10 @@ typedef struct {
     Eterm* hp_start;
     Eterm* hp;
     Eterm* hp_end;
+    int remaining_n;
+#ifdef DEBUG
+    Eterm* container_start;
+#endif
 } B2TDecodeContext;
 
 typedef struct {
@@ -1153,7 +1168,7 @@ typedef struct B2TContext_t {
     byte* aligned_alloc;
     ErtsBinary2TermState b2ts;
     //Uint ext_size;
-    Uint reds;
+    SWord reds;
     Eterm trap_bin;
     enum B2TState state;
     union {
@@ -1309,6 +1324,12 @@ BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
 
 #define B2T_BYTES_PER_REDUCTION 100
 
+static unsigned sverk_rand(void)
+{
+    static unsigned prev = 17;
+    prev = (prev * 214013 + 2531011);
+    return prev;
+}
 
 static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
 {
@@ -1326,7 +1347,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
     } else {
 	ctx = ERTS_MAGIC_BIN_DATA(context_b);
     }
-    ctx->reds = 1; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/
+    ctx->reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/
 
     do {
         switch (ctx->state) {
@@ -1360,6 +1381,10 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
             /*fall through*/
         case B2TDecodeInit:
             if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) {
+                /*SVERK Can we do a better prediction that is still safe
+                    OR is there a way to do HAlloc after result some how...
+                    ... support for mutiple HReleases maybe?
+                    */
                 /* dec_term will probably trap, allocate space for magic bin
                    before result term to make it easy to trim with HRelease.
                  */
@@ -1374,7 +1399,9 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
             ctx->u.dc.hp_end   = ctx->u.dc.hp_start + ctx->heap_size;
             ctx->state = B2TDecode;
             /*fall through*/
-        case B2TDecode:
+	case B2TDecode:
+        case B2TDecodeList:
+        case B2TDecodeTuple:
             dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx);
             break;
 
@@ -1404,7 +1431,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
             return ctx->u.dc.res;
 
         }
-    }while (ctx->reds);
+    }while (ctx->reds || ctx->state >= B2TDone);
 
     if (context_b == NULL) {
         ASSERT(ctx->trap_bin == THE_NON_VALUE);
@@ -1415,7 +1442,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
         sys_memcpy(ctx, &c_buff, sizeof(B2TContext));
 
         if (!magic_space) {
-            ASSERT(ctx->state != B2TDecode);
+            ASSERT(ctx->state < B2TDecode);
             magic_space = HAlloc(p, PROC_BIN_SIZE);
         }
         ctx->trap_bin = erts_mk_magic_binary_term(&magic_space, &MSO(p), context_b);
@@ -2778,28 +2805,81 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
     ErtsAtomEncoding char_enc;
     register Eterm* hp;        /* Please don't take the address of hp */
     Eterm* next;
-    byte* ep_trap_limit;
+    SWord reds;
 
     if (ctx) {
+        reds = ctx->reds;
+        next = ctx->u.dc.next;
+
+        if (ctx->state != B2TDecode) {
+            n = ctx->u.dc.remaining_n;
+            if (reds < n) {
+                ctx->u.dc.remaining_n -= reds;
+                n = reds;
+            }
+            else {
+                ctx->u.dc.remaining_n = 0;
+            }
+            reds -= n;
+
+            switch (ctx->state) {
+            case B2TDecodeList:
+                objp = next - 2;
+                while (n > 0) {
+                    objp[0] = (Eterm) COMPRESS_POINTER(next);
+                    objp[1] = make_list(next);
+                    next = objp;
+                    objp -= 2;
+                    n--;
+                }
+                break;
+
+            case B2TDecodeTuple:
+                objp = next - 1;
+                while (n-- > 0) {
+                    objp[0] = (Eterm) COMPRESS_POINTER(next);
+                    next = objp;
+                    objp--;
+                }
+                break;
+
+            default:
+                ASSERT(!"Unknown state");
+            }
+            if (ctx->u.dc.remaining_n) {
+                ctx->u.dc.next = next;
+                ctx->reds = 0;
+                return NULL;
+            }
+            ASSERT(next == ctx->u.dc.container_start);
+            ctx->state = B2TDecode;
+        }
+
         hp_saved = ctx->u.dc.hp_start;
         ep = ctx->u.dc.ep;
-        ep_trap_limit = ep + ctx->reds;
-        if (ep_trap_limit < ep) {
-            ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK Is there a safe way to create a "largest ptr" */
-        }
-        next = ctx->u.dc.next;
         hpp = &ctx->u.dc.hp;
     }
     else {
         hp_saved = *hpp;
-        ep_trap_limit = (byte*)ERTS_UWORD_MAX; /*SVERK - " - */
+        reds = ERTS_SWORD_MAX;
         next = objp;
         *next = (Eterm) (UWord) NULL;
     }
-    ASSERT(ep < ep_trap_limit);
     hp = *hpp;
 
     while (next != NULL) {
+
+        if (reds <= 0) {
+            if (ctx) {
+                ctx->u.dc.ep = ep;
+                ctx->u.dc.next = next;
+                ctx->u.dc.hp = hp;
+                ctx->reds = 0;
+                return NULL;
+            }
+            reds = ERTS_SWORD_MAX;
+        }
+
 	objp = next;
 	next = (Eterm *) EXPAND_POINTER(*objp);
 
@@ -2918,8 +2998,20 @@ dec_term_atom_common:
 	tuple_loop:
 	    *objp = make_tuple(hp);
 	    *hp++ = make_arityval(n);
+        #ifdef DEBUG
+            if (ctx) ctx->u.dc.container_start = hp;
+        #endif
 	    hp += n;
-	    objp = hp - 1;
+            objp = hp - 1;
+            if (ctx) {
+                if (reds < n) {
+                    ASSERT(reds > 0);
+                    ctx->state = B2TDecodeTuple;
+                    ctx->u.dc.remaining_n = n - reds;
+                    n = reds;
+                }
+		reds -= n;
+	    }
 	    while (n-- > 0) {
 		objp[0] = (Eterm) COMPRESS_POINTER(next);
 		next = objp;
@@ -2937,17 +3029,30 @@ dec_term_atom_common:
 		break;
 	    }
 	    *objp = make_list(hp);
-	    hp += 2*n;
+        #ifdef DEBUG
+            if (ctx) ctx->u.dc.container_start = hp;
+        #endif
+            hp += 2 * n;
 	    objp = hp - 2;
 	    objp[0] = (Eterm) COMPRESS_POINTER((objp+1));
 	    objp[1] = (Eterm) COMPRESS_POINTER(next);
 	    next = objp;
 	    objp -= 2;
-	    while (--n > 0) {
+            n--;
+	    if (ctx) {
+                if (reds < n) {
+		    ctx->state = B2TDecodeList;
+		    ctx->u.dc.remaining_n = n - reds;
+		    n = reds;
+		}
+		reds -= n;
+	    }
+            while (n > 0) {
 		objp[0] = (Eterm) COMPRESS_POINTER(next);
-		objp[1] = make_list(objp + 2);
+		objp[1] = make_list(next);
 		next = objp;
 		objp -= 2;
+                n--;
 	    }
 	    break;
 	case STRING_EXT:
@@ -3494,20 +3599,16 @@ dec_term_atom_common:
 	    *hpp = hp_saved;
             if (ctx) {
                 ctx->state = B2TDecodeFail;
+		ctx->reds = reds;
             }
             return NULL;
 	}
-        if (ep > ep_trap_limit) {
-            ASSERT(ctx);
-	    ctx->u.dc.ep = ep;
-	    ctx->u.dc.next = next;
-	    ctx->u.dc.hp = hp;
-	    ctx->reds = 0;
-	    return NULL;
-        }
+
+        --reds;
     }
     if (ctx) {
         ctx->state = B2TDone;
+	ctx->reds = reds;
     }
     *hpp = hp;
     return ep;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 11ba554a3d..0f4169c81d 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -283,18 +283,21 @@ typedef long          SWord;
 #define SWORD_CONSTANT(Const) Const##L
 #define UWORD_CONSTANT(Const) Const##UL
 #define ERTS_UWORD_MAX ULONG_MAX
+#define ERTS_SWORD_MAX LONG_MAX
 #elif SIZEOF_VOID_P == SIZEOF_INT
 typedef unsigned int UWord;
 typedef int          SWord;
 #define SWORD_CONSTANT(Const) Const
 #define UWORD_CONSTANT(Const) Const##U
 #define ERTS_UWORD_MAX UINT_MAX
+#define ERTS_SWORD_MAX INT_MAX
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
 typedef unsigned long long UWord;
 typedef long long          SWord;
 #define SWORD_CONSTANT(Const) Const##LL
 #define UWORD_CONSTANT(Const) Const##ULL
 #define ERTS_UWORD_MAX ULLONG_MAX
+#define ERTS_SWORD_MAX LLONG_MAX
 #else
 #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
 #endif
@@ -308,6 +311,7 @@ typedef long          Sint;
 #define SWORD_CONSTANT(Const) Const##L
 #define UWORD_CONSTANT(Const) Const##UL
 #define ERTS_UWORD_MAX ULONG_MAX
+#define ERTS_SWORD_MAX LONG_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_INT
@@ -317,6 +321,7 @@ typedef int          Sint;
 #define SWORD_CONSTANT(Const) Const
 #define UWORD_CONSTANT(Const) Const##U
 #define ERTS_UWORD_MAX UINT_MAX
+#define ERTS_SWORD_MAX INT_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
@@ -326,6 +331,7 @@ typedef long long          Sint;
 #define SWORD_CONSTANT(Const) Const##LL
 #define UWORD_CONSTANT(Const) Const##ULL
 #define ERTS_UWORD_MAX ULLONG_MAX
+#define ERTS_SWORD_MAX LLONG_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
 #if defined(__WIN32__)
 #define ErtsStrToSint _strtoi64
-- 
cgit v1.2.3


From d42bd3b37d3b741d546d94fd7611afda81bba419 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Oct 2013 12:11:59 +0200
Subject: trapping STRING_EXT

---
 erts/emulator/beam/external.c | 84 ++++++++++++++++++++++++++++---------------
 1 file changed, 55 insertions(+), 29 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 48536d610b..2116c25cd2 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1125,13 +1125,15 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 }
 
 
-enum B2TState {
+enum B2TState { /* order is somewhat significant */
     /*B2TUncompress,*/
     B2TSize,
     B2TDecodeInit,
+
     B2TDecode,
     B2TDecodeList,
     B2TDecodeTuple,
+    B2TDecodeString,
 
     B2TDone,
     B2TDecodeFail,
@@ -1150,7 +1152,7 @@ typedef struct {
     Eterm* hp_end;
     int remaining_n;
 #ifdef DEBUG
-    Eterm* container_start;
+    Eterm* container_end;
 #endif
 } B2TDecodeContext;
 
@@ -1402,6 +1404,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
 	case B2TDecode:
         case B2TDecodeList:
         case B2TDecodeTuple:
+        case B2TDecodeString:
             dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx);
             break;
 
@@ -2808,8 +2811,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
     SWord reds;
 
     if (ctx) {
-        reds = ctx->reds;
-        next = ctx->u.dc.next;
+        hp_saved = ctx->u.dc.hp_start;
+        reds     = ctx->reds;
+        next     = ctx->u.dc.next;
+        ep       = ctx->u.dc.ep;
+        hpp      = &ctx->u.dc.hp;
 
         if (ctx->state != B2TDecode) {
             n = ctx->u.dc.remaining_n;
@@ -2832,6 +2838,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                     objp -= 2;
                     n--;
                 }
+                ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end);
                 break;
 
             case B2TDecodeTuple:
@@ -2841,23 +2848,35 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                     next = objp;
                     objp--;
                 }
+                ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end);
+                break;
+
+            case B2TDecodeString:
+                hp = *hpp;
+                hp[-1] = make_list(hp);  /* overwrite the premature NIL */
+                while (n-- > 0) {
+                    hp[0] = make_small(*ep++);
+                    hp[1] = make_list(hp+2);
+                    hp += 2;
+                }
+                hp[-1] = NIL;
+                *hpp = hp;
+                ASSERT(ctx->u.dc.remaining_n || hp == ctx->u.dc.container_end);
                 break;
 
             default:
                 ASSERT(!"Unknown state");
             }
-            if (ctx->u.dc.remaining_n) {
+            if (!ctx->u.dc.remaining_n) {
+                ctx->state = B2TDecode;
+            }
+            if (reds <= 0) {
                 ctx->u.dc.next = next;
+                ctx->u.dc.ep = ep;
                 ctx->reds = 0;
                 return NULL;
             }
-            ASSERT(next == ctx->u.dc.container_start);
-            ctx->state = B2TDecode;
         }
-
-        hp_saved = ctx->u.dc.hp_start;
-        ep = ctx->u.dc.ep;
-        hpp = &ctx->u.dc.hp;
     }
     else {
         hp_saved = *hpp;
@@ -2869,17 +2888,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
 
     while (next != NULL) {
 
-        if (reds <= 0) {
-            if (ctx) {
-                ctx->u.dc.ep = ep;
-                ctx->u.dc.next = next;
-                ctx->u.dc.hp = hp;
-                ctx->reds = 0;
-                return NULL;
-            }
-            reds = ERTS_SWORD_MAX;
-        }
-
 	objp = next;
 	next = (Eterm *) EXPAND_POINTER(*objp);
 
@@ -2998,13 +3006,11 @@ dec_term_atom_common:
 	tuple_loop:
 	    *objp = make_tuple(hp);
 	    *hp++ = make_arityval(n);
-        #ifdef DEBUG
-            if (ctx) ctx->u.dc.container_start = hp;
-        #endif
 	    hp += n;
             objp = hp - 1;
             if (ctx) {
                 if (reds < n) {
+                    IF_DEBUG(ctx->u.dc.container_end = hp - n;)
                     ASSERT(reds > 0);
                     ctx->state = B2TDecodeTuple;
                     ctx->u.dc.remaining_n = n - reds;
@@ -3029,9 +3035,6 @@ dec_term_atom_common:
 		break;
 	    }
 	    *objp = make_list(hp);
-        #ifdef DEBUG
-            if (ctx) ctx->u.dc.container_start = hp;
-        #endif
             hp += 2 * n;
 	    objp = hp - 2;
 	    objp[0] = (Eterm) COMPRESS_POINTER((objp+1));
@@ -3041,6 +3044,7 @@ dec_term_atom_common:
             n--;
 	    if (ctx) {
                 if (reds < n) {
+                    IF_DEBUG(ctx->u.dc.container_end = hp - 2*(n+1);)
 		    ctx->state = B2TDecodeList;
 		    ctx->u.dc.remaining_n = n - reds;
 		    n = reds;
@@ -3063,6 +3067,15 @@ dec_term_atom_common:
 		break;
 	    }
 	    *objp = make_list(hp);
+            if (ctx) {
+                if (reds < n) {
+                    IF_DEBUG(ctx->u.dc.container_end = hp + n*2;)
+                    ctx->state = B2TDecodeString;
+                    ctx->u.dc.remaining_n = n - reds;
+                    n = reds;
+		}
+                reds -= n;
+            }
 	    while (n-- > 0) {
 		hp[0] = make_small(*ep++);
 		hp[1] = make_list(hp+2);
@@ -3604,7 +3617,20 @@ dec_term_atom_common:
             return NULL;
 	}
 
-        --reds;
+        if (--reds <= 0) {
+            if (ctx) {
+                if (next || ctx->state != B2TDecode) {
+                    ctx->u.dc.ep = ep;
+                    ctx->u.dc.next = next;
+                    ctx->u.dc.hp = hp;
+                    ctx->reds = 0;
+                    return NULL;
+                }
+            }
+            else {
+                reds = ERTS_SWORD_MAX;
+            }
+        }
     }
     if (ctx) {
         ctx->state = B2TDone;
-- 
cgit v1.2.3


From 74c5c0314d4cf85365d89c1fdf41a5d286a7191a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Oct 2013 19:59:09 +0200
Subject: trapping binary_to_term/2

---
 erts/emulator/beam/external.c | 73 +++++++++++++++----------------------------
 1 file changed, 26 insertions(+), 47 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 2116c25cd2..38280b7a99 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -108,7 +108,7 @@ static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj,
 
 static Export binary_to_term_trap_export;
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
-static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b);
+static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b);
 
 void erts_init_external(void) {
 #if 1 /* In R16 */
@@ -1169,7 +1169,7 @@ typedef struct B2TContext_t {
     Sint heap_size;
     byte* aligned_alloc;
     ErtsBinary2TermState b2ts;
-    //Uint ext_size;
+    Uint32 flags;
     SWord reds;
     Eterm trap_bin;
     enum B2TState state;
@@ -1316,15 +1316,22 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
     Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
 
-    return binary_to_term_int(BIF_P, THE_NON_VALUE, context_bin);
+    return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin);
 }
 
 BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
 {
-    return binary_to_term_int(BIF_P, BIF_ARG_1, NULL);
+/*SVERK  if (++sverk_cnt % 1000 == 0) {
+        erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt);
+    }
+    if (sverk_cnt == 1767) {
+        sverk_break();
+    }
+*/
+    return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
 }
 
-#define B2T_BYTES_PER_REDUCTION 100
+#define B2T_BYTES_PER_REDUCTION 128
 
 static unsigned sverk_rand(void)
 {
@@ -1333,11 +1340,13 @@ static unsigned sverk_rand(void)
     return prev;
 }
 
-static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
+static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b)
 {
+    SWord initial_reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/
     byte* bytes;
     B2TContext c_buff;
     B2TContext *ctx;
+    ErtsDistExternal fakedep;
     Eterm* magic_space = NULL;
 
     if (context_b == NULL) {
@@ -1345,11 +1354,12 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
         ctx = &c_buff;
 	ctx->state = B2TSize;
         ctx->aligned_alloc = NULL;
+        ctx->flags = flags;
         IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
     } else {
 	ctx = ERTS_MAGIC_BIN_DATA(context_b);
     }
-    ctx->reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/
+    ctx->reds = initial_reds;
 
     do {
         switch (ctx->state) {
@@ -1405,7 +1415,8 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
         case B2TDecodeList:
         case B2TDecodeTuple:
         case B2TDecodeString:
-            dec_term(NULL, NULL, NULL, &MSO(p), NULL, ctx);
+            fakedep.flags = ctx->flags;
+            dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx);
             break;
 
         case B2TDecodeFail:
@@ -1416,6 +1427,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
             if (context_b) {
                 erts_set_gc_state(p, 1);
             }
+            BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
             BIF_ERROR(p, BADARG);
 
         case B2TDone:
@@ -1431,6 +1443,7 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
                 erts_set_gc_state(p, 1);
             }
 
+            BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
             return ctx->u.dc.res;
 
         }
@@ -1461,24 +1474,15 @@ static Eterm binary_to_term_int(Process* p, Eterm bin, Binary* context_b)
 
 BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
 {
-    Sint heap_size;
-    Eterm res;
     Eterm opts;
     Eterm opt;
-    Eterm* hp;
-    Eterm* endp;
-    Sint size;
-    byte* bytes;
-    byte* temp_alloc = NULL;
-    ErtsBinary2TermState b2ts;
-    ErtsDistExternal fakedep;
+    Uint32 flags = 0;
 
-    fakedep.flags = 0;
     opts = BIF_ARG_2;
     while (is_list(opts)) {
         opt = CAR(list_val(opts));
         if (opt == am_safe) {
-	    fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE;
+            flags |= ERTS_DIST_EXT_BTT_SAFE;
         }
 	else {
             goto error;
@@ -1489,35 +1493,10 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
     if (is_not_nil(opts))
         goto error;
 
-    if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) {
-    error:
-	erts_free_aligned_binary_bytes(temp_alloc);
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    size = binary_size(BIF_ARG_1);
-
-    heap_size = binary2term_prepare(&b2ts, bytes, size);
-    if (heap_size < 0)
-	goto error;
-
-    hp = HAlloc(BIF_P, heap_size);
-    endp = hp + heap_size;
-
-    res = binary2term_create(&fakedep, &b2ts, &hp, &MSO(BIF_P));
-
-    erts_free_aligned_binary_bytes(temp_alloc);
-
-    if (hp > endp) {
-	erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
-		 __FILE__, __LINE__, hp-endp);
-    }
-
-    HRelease(BIF_P, endp, hp);
-
-    if (res == THE_NON_VALUE)
-	goto error;
+    return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL);
 
-    return res;
+error:
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 Eterm
-- 
cgit v1.2.3


From 38f1140d197e14f6cb4d0040859b0b19876d2a14 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 11 Oct 2013 16:28:59 +0200
Subject: trapping size calculation

---
 erts/emulator/beam/external.c | 175 +++++++++++++++++++++++++++---------------
 1 file changed, 114 insertions(+), 61 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 38280b7a99..d4ffc23536 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -96,7 +96,7 @@ struct B2TContext_t;
 static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
 static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
-static Sint decoded_size(byte *ep, byte* endp, int internal_tags);
+static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*);
 static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1);
 
 static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, 
@@ -889,7 +889,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep)
 	    goto fail;
 	ep = edep->extp+1;
     }
-    res = decoded_size(ep, edep->ext_endp, 0);
+    res = decoded_size(ep, edep->ext_endp, 0, NULL);
     if (res >= 0)
 	return res;
  fail:
@@ -901,12 +901,12 @@ Sint erts_decode_ext_size(byte *ext, Uint size)
 {
     if (size == 0 || *ext != VERSION_MAGIC)
 	return -1;
-    return decoded_size(ext+1, ext+size, 0);
+    return decoded_size(ext+1, ext+size, 0, NULL);
 }
 
 Sint erts_decode_ext_size_ets(byte *ext, Uint size)
 {
-    Sint sz = decoded_size(ext, ext+size, 1);
+    Sint sz = decoded_size(ext, ext+size, 1, NULL);
     ASSERT(sz >= 0);
     return sz;
 }
@@ -1126,7 +1126,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 
 
 enum B2TState { /* order is somewhat significant */
-    /*B2TUncompress,*/
+    B2TPrepare,
     B2TSize,
     B2TDecodeInit,
 
@@ -1141,6 +1141,10 @@ enum B2TState { /* order is somewhat significant */
 };
 
 typedef struct {
+    int heap_size;
+    int terms;
+    byte* ep;
+    int atom_extra_skip;
 } B2TSizeContext;
 
 typedef struct {
@@ -1210,21 +1214,15 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size)
     return err == Z_STREAM_END ? uncomp_size : 0;
 }
 
-static ERTS_INLINE Sint
+static ERTS_INLINE int
 binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 {
-    Sint res;
     byte *bytes = data;
     Sint size = data_size;
 
     state->exttmp = 0;
 
     if (size < 1 || *bytes != VERSION_MAGIC) {
-    error:
-	if (state->exttmp)
-	    erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
-	state->extp = NULL;
-	state->exttmp = 0;
 	return -1;
     }
     bytes++;
@@ -1239,20 +1237,17 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 	if (dest_len > 32*1024*1024
 	    || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) {
 	    if (dest_len != binary2term_uncomp_size(bytes, size)) {
-		goto error;
+                return -1;
 	    }
 	    state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len);
 	}
 	state->exttmp = 1;
 	if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK)
-	    goto error;
+            return -1;
 	size = (Sint) dest_len;
     }
     state->extsize = size;
-    res = decoded_size(state->extp, state->extp + size, 0);
-    if (res < 0)
-	goto error;
-    return res;
+    return 0;
 }
 
 static ERTS_INLINE void
@@ -1280,7 +1275,18 @@ binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **
 Sint
 erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 {
-    return binary2term_prepare(state, data, data_size);
+    Sint res;
+
+    if (binary2term_prepare(state, data, data_size) < 0 ||
+        (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) {
+
+        if (state->exttmp)
+            erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp);
+        state->extp = NULL;
+	state->exttmp = 0;
+	return -1;
+    }
+    return res;
 }
 
 void
@@ -1319,17 +1325,6 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
     return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin);
 }
 
-BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
-{
-/*SVERK  if (++sverk_cnt % 1000 == 0) {
-        erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt);
-    }
-    if (sverk_cnt == 1767) {
-        sverk_break();
-    }
-*/
-    return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
-}
 
 #define B2T_BYTES_PER_REDUCTION 128
 
@@ -1352,18 +1347,19 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
     if (context_b == NULL) {
 	/* Setup enough to get started */
         ctx = &c_buff;
-	ctx->state = B2TSize;
+	ctx->state = B2TPrepare;
         ctx->aligned_alloc = NULL;
         ctx->flags = flags;
         IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
     } else {
 	ctx = ERTS_MAGIC_BIN_DATA(context_b);
+        ASSERT(ctx->state != B2TPrepare);
     }
     ctx->reds = initial_reds;
 
     do {
         switch (ctx->state) {
-        case B2TSize:
+        case B2TPrepare:
             bytes = erts_get_aligned_binary_bytes_extra(bin,
                                                         &ctx->aligned_alloc,
                                                         ERTS_ALC_T_EXT_TERM_DATA,
@@ -1381,16 +1377,19 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
                 break;
             }
 
+            ctx->reds = 0; /*SVERK*/
+            ctx->u.sc.heap_size = 0;
+            ctx->u.sc.terms = 1;
+            ctx->u.sc.ep = ctx->b2ts.extp;
+            ctx->u.sc.atom_extra_skip = 0;
+            ctx->state = B2TSize;
+            break;
 
+        case B2TSize:
+            ctx->heap_size = decoded_size(NULL, ctx->b2ts.extp + ctx->b2ts.extsize,
+                                          0, ctx);
+            break;
 
-            ctx->state = B2TDecodeInit;
-
-            if (ctx->b2ts.extsize >= ctx->reds) {
-                ctx->reds = 0;
-                break;
-            }
-            ctx->reds -= ctx->b2ts.extsize;
-            /*fall through*/
         case B2TDecodeInit:
             if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) {
                 /*SVERK Can we do a better prediction that is still safe
@@ -1472,6 +1471,18 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
     BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin);
 }
 
+BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
+{
+/*SVERK  if (++sverk_cnt % 1000 == 0) {
+        erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt);
+    }
+    if (sverk_cnt == 1767) {
+        sverk_break();
+    }
+*/
+    return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
+}
+
 BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
 {
     Eterm opts;
@@ -3901,18 +3912,32 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj,
 }
 
 static Sint
-decoded_size(byte *ep, byte* endp, int internal_tags)
+decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
 {
     int heap_size = 0;
     int terms;
     int atom_extra_skip = 0;
     Uint n;
+    SWord reds;
 
+    if (ctx) {
+        heap_size = ctx->u.sc.heap_size;
+        terms = ctx->u.sc.terms;
+        ep = ctx->u.sc.ep;
+        atom_extra_skip = ctx->u.sc.atom_extra_skip;
+        reds = ctx->reds;
+    }
+    else {
+        heap_size = 0;
+        terms = 1;
+        atom_extra_skip = 0;
+        reds = ERTS_SWORD_MAX;
+    }
 #define SKIP(sz)				\
     do {					\
 	if ((sz) <= endp-ep) {			\
 	    ep += (sz);				\
-        } else { return -1; };			\
+        } else { goto error; };			\
     } while (0)
 
 #define SKIP2(sz1, sz2)				\
@@ -3920,25 +3945,27 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	Uint sz = (sz1) + (sz2);		\
 	if (sz1 < sz && (sz) <= endp-ep) {	\
 	    ep += (sz);				\
-        } else { return -1; }			\
+        } else { goto error; }			\
     } while (0)
 
 #define CHKSIZE(sz)				\
     do {					\
-	 if ((sz) > endp-ep) { return -1; }	\
+	 if ((sz) > endp-ep) { goto error; }	\
     } while (0)
 
 #define ADDTERMS(n)				\
     do {					\
         int before = terms;		        \
 	terms += (n);                           \
-	if (terms < before) return -1;     	\
+	if (terms < before) goto error;     	\
     } while (0)
 
-
-    for (terms=1; terms > 0; terms--) {
-	int tag;
-
+    ASSERT(terms > 0);
+    do {
+        int tag;
+        /*SVERK
+        erts_fprintf(stderr, "SVERK tag=%d ep=%p terms=%d heap_size=%d endp=%p\n",
+                     ep[0], ep, terms, heap_size, endp); */
 	CHKSIZE(1);
 	tag = ep++[0];
 	switch (tag) {
@@ -3959,7 +3986,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    CHKSIZE(4);
 	    n = get_int32(ep);
 	    if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) {
-		return -1;
+		goto error;
 	    }
 	    SKIP2(n,4+1);		/* skip, size,sign,digits */
 	    heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */
@@ -3968,7 +3995,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    CHKSIZE(2);
 	    n = get_int16(ep);
 	    if (n > MAX_ATOM_CHARACTERS) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+2+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -3978,7 +4005,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    n = get_int16(ep);
 	    ep += 2;
 	    if (n > MAX_ATOM_SZ_LIMIT) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -3987,7 +4014,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    CHKSIZE(1);
 	    n = get_int8(ep);
 	    if (n > MAX_ATOM_CHARACTERS) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+1+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -3997,7 +4024,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	    n = get_int8(ep);
 	    ep++;
 	    if (n > MAX_ATOM_SZ_LIMIT) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(n+atom_extra_skip);
 	    atom_extra_skip = 0;
@@ -4026,7 +4053,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 		id_words = get_int16(ep);
 		    
 		if (id_words > ERTS_MAX_REF_NUMBERS)
-		    return -1;
+		    goto error;
 
 		ep += 2;
 		atom_extra_skip = 1 + 4*id_words;
@@ -4128,7 +4155,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 		num_free = get_int32(ep);
 		ep += 4;
 		if (num_free > MAX_ARG) {
-		    return -1;
+		    goto error;
 		}
 		terms += 4 + num_free;
 		heap_size += ERL_FUN_SIZE + num_free;
@@ -4145,24 +4172,50 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 
 	case BINARY_INTERNAL_REF:
 	    if (!internal_tags) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(sizeof(ProcBin));
 	    heap_size += PROC_BIN_SIZE;
 	    break;
 	case BIT_BINARY_INTERNAL_REF:
 	    if (!internal_tags) {
-		return -1;
+		goto error;
 	    }
 	    SKIP(2+sizeof(ProcBin));
 	    heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
 	    break;
 	default:
-	    return -1;
+	    goto error;
 	}
-    }
+        terms--;
+
+        if (--reds <= 0) {
+            if (ctx && terms > 0) {
+                ctx->u.sc.heap_size = heap_size;
+                ctx->u.sc.terms = terms;
+                ctx->u.sc.ep = ep;
+                ctx->u.sc.atom_extra_skip = atom_extra_skip;
+                ctx->reds = 0;
+                return 0;
+            }
+            reds = ERTS_SWORD_MAX;
+        }
+    }while (terms > 0);
+
     /* 'terms' may be non-zero if it has wrapped around */
-    return terms==0 ? heap_size : -1;
+    if (terms == 0) {
+        if (ctx) {
+            ctx->state = B2TDecodeInit;
+            ctx->reds = reds;
+        }
+        return heap_size;
+    }
+
+error:
+    if (ctx) {
+        ctx->state = B2TBadArg;
+    }
+    return -1;
 #undef SKIP
 #undef SKIP2
 #undef CHKSIZE
-- 
cgit v1.2.3


From f10ea68ce28e9b93ce614b5f829b1ca7f4cc753f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 14 Oct 2013 17:46:28 +0200
Subject: trapping uncompress

---
 erts/emulator/beam/erl_zlib.c |  40 ++++++++++++
 erts/emulator/beam/erl_zlib.h |   6 ++
 erts/emulator/beam/external.c | 146 ++++++++++++++++++++++++++++--------------
 3 files changed, 144 insertions(+), 48 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c
index 47fd92988e..8e33144f96 100644
--- a/erts/emulator/beam/erl_zlib.c
+++ b/erts/emulator/beam/erl_zlib.c
@@ -87,6 +87,46 @@ int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp)
     return deflateEnd(streamp);
 }
 
+int ZEXPORT erl_zlib_inflate_start(z_stream *streamp, const Bytef* source,
+                                   uLong sourceLen)
+{
+    streamp->next_in = (Bytef*)source;
+    streamp->avail_in = (uInt)sourceLen;
+    streamp->total_out = streamp->avail_out = 0;
+    streamp->next_out = NULL;
+    erl_zlib_alloc_init(streamp);
+    return inflateInit(streamp);
+}
+/*
+ * Inflate a chunk, The destination length is the limit.
+ * Returns Z_OK if more to process, Z_STREAM_END if we are done.
+ */
+int ZEXPORT erl_zlib_inflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen)
+{
+    int err;
+    uLongf last_tot = streamp->total_out;
+
+    streamp->next_out = dest;
+    streamp->avail_out = (uInt)*destLen;
+
+    if ((uLong)streamp->avail_out != *destLen) return Z_BUF_ERROR;
+
+    err = inflate(streamp, Z_NO_FLUSH);
+    ASSERT(err != Z_STREAM_ERROR);
+    *destLen = streamp->total_out - last_tot;
+    return err;
+}
+
+/*
+ * When we are done, free up the inflate structure
+ * Retyurns Z_OK or Error
+ */
+int ZEXPORT erl_zlib_inflate_finish(z_stream *streamp)
+{
+    return inflateEnd(streamp);
+}
+
+
 int ZEXPORT erl_zlib_compress2 (Bytef* dest, uLongf* destLen,
 				const Bytef* source, uLong sourceLen,
 				int level)
diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h
index 5ac849d21c..160166c66b 100644
--- a/erts/emulator/beam/erl_zlib.h
+++ b/erts/emulator/beam/erl_zlib.h
@@ -39,6 +39,12 @@ int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source,
 int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen);
 int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp);
 
+int ZEXPORT erl_zlib_inflate_start(z_stream *streamp, const Bytef* source,
+                                   uLong sourceLen);
+int ZEXPORT erl_zlib_inflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen);
+int ZEXPORT erl_zlib_inflate_finish(z_stream *streamp);
+
+
 /* Use instead of compress
 */
 #define erl_zlib_compress(dest,destLen,source,sourceLen) \
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index d4ffc23536..0d2de1b199 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1127,6 +1127,8 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 
 enum B2TState { /* order is somewhat significant */
     B2TPrepare,
+    B2TUncompressChunk,
+    B2TSizeInit,
     B2TSize,
     B2TDecodeInit,
 
@@ -1161,12 +1163,9 @@ typedef struct {
 } B2TDecodeContext;
 
 typedef struct {
-    /*Uint real_size;
-    Uint dest_len;
-    byte *dbytes;
-    Binary *result_bin;
-    Binary *destination_bin;
-    z_stream stream;*/
+    z_stream stream;
+    byte* dbytes;
+    Uint dleft;
 } B2TUncompressContext;
 
 typedef struct B2TContext_t {
@@ -1215,7 +1214,8 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size)
 }
 
 static ERTS_INLINE int
-binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
+binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size,
+		    B2TContext* ctx)
 {
     byte *bytes = data;
     Sint size = data_size;
@@ -1229,6 +1229,8 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
     size--;
     if (size < 5 || *bytes != COMPRESSED) {
 	state->extp = bytes;
+        if (ctx)
+	    ctx->state = B2TSizeInit;
     }
     else  {
 	uLongf dest_len = (Uint32) get_int32(bytes+1);
@@ -1236,14 +1238,33 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 	size -= 5;	
 	if (dest_len > 32*1024*1024
 	    || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) {
+            /*
+             * Try avoid out-of-memory crash due to corrupted 'dest_len'
+             * by checking the actual length of the uncompressed data.
+             * The only way to do that is to uncompress it. Sad but true.
+             */
 	    if (dest_len != binary2term_uncomp_size(bytes, size)) {
                 return -1;
 	    }
 	    state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len);
+            ctx->reds -= dest_len;
 	}
 	state->exttmp = 1;
-	if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK)
-            return -1;
+        if (ctx) {
+	    if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK)
+		return -1;
+
+	    ctx->u.uc.dbytes = state->extp;
+	    ctx->u.uc.dleft = dest_len;
+	    ctx->state = B2TUncompressChunk;
+        }
+	else {
+	    uLongf dlen = dest_len;
+	    if (erl_zlib_uncompress(state->extp, &dlen, bytes, size) != Z_OK
+		|| dlen != dest_len) {
+		return -1;
+	    }
+        }
 	size = (Sint) dest_len;
     }
     state->extsize = size;
@@ -1277,7 +1298,7 @@ erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size
 {
     Sint res;
 
-    if (binary2term_prepare(state, data, data_size) < 0 ||
+    if (binary2term_prepare(state, data, data_size, NULL) < 0 ||
         (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) {
 
         if (state->exttmp)
@@ -1307,6 +1328,9 @@ static void b2t_destroy_context(B2TContext* context)
                                          ERTS_ALC_T_EXT_TERM_DATA);
     context->aligned_alloc = NULL;
     binary2term_abort(&context->b2ts);
+    if (context->state == B2TUncompressChunk) {
+	erl_zlib_inflate_finish(&context->u.uc.stream);
+    }
 }
 
 static void b2t_context_destructor(Binary *context_bin)
@@ -1359,7 +1383,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 
     do {
         switch (ctx->state) {
-        case B2TPrepare:
+        case B2TPrepare: {
+            Uint bin_size;
             bytes = erts_get_aligned_binary_bytes_extra(bin,
                                                         &ctx->aligned_alloc,
                                                         ERTS_ALC_T_EXT_TERM_DATA,
@@ -1369,24 +1394,48 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
                 ctx->state = B2TBadArg;
                 break;
             }
-
-            ctx->heap_size = binary2term_prepare(&ctx->b2ts, bytes,
-                                                 binary_size(bin));
-            if (ctx->heap_size < 0) {
-                ctx->state = B2TBadArg;
-                break;
+            bin_size = binary_size(bin);
+            if (ctx->aligned_alloc) {
+                ctx->reds -= bin_size / 8;
             }
-
-            ctx->reds = 0; /*SVERK*/
-            ctx->u.sc.heap_size = 0;
-            ctx->u.sc.terms = 1;
-            ctx->u.sc.ep = ctx->b2ts.extp;
-            ctx->u.sc.atom_extra_skip = 0;
-            ctx->state = B2TSize;
+            if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) {
+		ctx->state = B2TBadArg;
+	    }
             break;
+        }
+	case B2TUncompressChunk:
+	    {
+		Uint chunk = ctx->reds;
+		int zret;
+                if (chunk > ctx->u.uc.dleft)
+		    chunk = ctx->u.uc.dleft;
+
+		zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream,
+					      ctx->u.uc.dbytes, &chunk);
+		ctx->u.uc.dbytes += chunk;
+		ctx->u.uc.dleft  -= chunk;
+                if (zret == Z_OK && ctx->u.uc.dleft > 0) {
+		    ctx->reds = 0;
+		}
+                else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK
+			 && zret == Z_STREAM_END
+			 && ctx->u.uc.dleft == 0) {
+		    ctx->reds -= chunk;
+		    ctx->state = B2TSizeInit;
+		}
+		else {
+		    ctx->state = B2TBadArg;
+		}
+		break;
+	    }
 
+	case B2TSizeInit:
+	    ctx->u.sc.ep = NULL;
+	    ctx->state = B2TSize;
+	    /*fall through*/
         case B2TSize:
-            ctx->heap_size = decoded_size(NULL, ctx->b2ts.extp + ctx->b2ts.extsize,
+            ctx->heap_size = decoded_size(ctx->b2ts.extp,
+					  ctx->b2ts.extp + ctx->b2ts.extsize,
                                           0, ctx);
             break;
 
@@ -1455,7 +1504,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
                                              b2t_context_destructor);
         ctx = ERTS_MAGIC_BIN_DATA(context_b);
         sys_memcpy(ctx, &c_buff, sizeof(B2TContext));
-
+        if (ctx->state >= B2TDecode && ctx->u.dc.next == &c_buff.u.dc.res) {
+            ctx->u.dc.next = &ctx->u.dc.res;
+        }
         if (!magic_space) {
             ASSERT(ctx->state < B2TDecode);
             magic_space = HAlloc(p, PROC_BIN_SIZE);
@@ -3914,25 +3965,27 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj,
 static Sint
 decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
 {
-    int heap_size = 0;
+    int heap_size;
     int terms;
-    int atom_extra_skip = 0;
+    int atom_extra_skip;
     Uint n;
     SWord reds;
 
     if (ctx) {
-        heap_size = ctx->u.sc.heap_size;
-        terms = ctx->u.sc.terms;
-        ep = ctx->u.sc.ep;
-        atom_extra_skip = ctx->u.sc.atom_extra_skip;
         reds = ctx->reds;
+        if (ctx->u.sc.ep) {
+            heap_size = ctx->u.sc.heap_size;
+            terms = ctx->u.sc.terms;
+            ep = ctx->u.sc.ep;
+            atom_extra_skip = ctx->u.sc.atom_extra_skip;
+            goto init_done;
+        }
     }
-    else {
-        heap_size = 0;
-        terms = 1;
-        atom_extra_skip = 0;
-        reds = ERTS_SWORD_MAX;
-    }
+    heap_size = 0;
+    terms = 1;
+    atom_extra_skip = 0;
+init_done:
+
 #define SKIP(sz)				\
     do {					\
 	if ((sz) <= endp-ep) {			\
@@ -4189,16 +4242,13 @@ decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
 	}
         terms--;
 
-        if (--reds <= 0) {
-            if (ctx && terms > 0) {
-                ctx->u.sc.heap_size = heap_size;
-                ctx->u.sc.terms = terms;
-                ctx->u.sc.ep = ep;
-                ctx->u.sc.atom_extra_skip = atom_extra_skip;
-                ctx->reds = 0;
-                return 0;
-            }
-            reds = ERTS_SWORD_MAX;
+        if (ctx && --reds <= 0 && terms > 0) {
+            ctx->u.sc.heap_size = heap_size;
+            ctx->u.sc.terms = terms;
+            ctx->u.sc.ep = ep;
+            ctx->u.sc.atom_extra_skip = atom_extra_skip;
+            ctx->reds = 0;
+            return 0;
         }
     }while (terms > 0);
 
-- 
cgit v1.2.3


From d5b6c6f0bd96108d788cdfb9be15059125b3d87f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Oct 2013 16:28:21 +0200
Subject: erts: Add erlang wrappers to binary_to_term

to not expose the trapping BIF in the stacktrace
when it throws badarg.
---
 erts/emulator/beam/bif.tab    | 8 ++------
 erts/emulator/beam/external.c | 6 +++---
 2 files changed, 5 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 3b7bb56885..9f3de5f780 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -45,7 +45,6 @@ bif erlang:apply/3
 bif erlang:atom_to_list/1
 bif erlang:binary_to_list/1
 bif erlang:binary_to_list/3
-bif erlang:binary_to_term/1	
 bif erlang:crc32/1
 bif erlang:crc32/2
 bif erlang:crc32_combine/3
@@ -152,6 +151,8 @@ bif erts_internal:port_command/3
 bif erts_internal:port_control/3
 bif erts_internal:port_close/1
 bif erts_internal:port_connect/2
+bif erts_internal:binary_to_term/1
+bif erts_internal:binary_to_term/2
 
 bif erts_internal:request_system_task/3
 bif erts_internal:check_process_code/2
@@ -478,11 +479,6 @@ bif erlang:load_nif/2
 bif erlang:call_on_load_function/1
 bif erlang:finish_after_on_load/2
 
-#
-# New Bifs in R13B4
-#
-bif erlang:binary_to_term/2
-
 #
 # The binary match bifs (New in R14A - EEP9)
 #
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 0d2de1b199..e3e199b198 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1476,7 +1476,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
                 erts_set_gc_state(p, 1);
             }
             BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
-            BIF_ERROR(p, BADARG);
+            BIF_ERROR(p, BADARG & ~EXF_SAVETRACE);
 
         case B2TDone:
             b2t_destroy_context(ctx);
@@ -1522,7 +1522,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
     BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin);
 }
 
-BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
+BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1)
 {
 /*SVERK  if (++sverk_cnt % 1000 == 0) {
         erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt);
@@ -1534,7 +1534,7 @@ BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
     return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
 }
 
-BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
+BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2)
 {
     Eterm opts;
     Eterm opt;
-- 
cgit v1.2.3


From abbc9c2755396d4050db8f15e02c21032a528049 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Oct 2013 16:28:40 +0200
Subject: erts: Cleanup code for trapping binary_to_term

---
 erts/emulator/beam/external.c | 103 ++++++++++++++++++------------------------
 1 file changed, 45 insertions(+), 58 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index e3e199b198..651aac03a2 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1131,7 +1131,6 @@ enum B2TState { /* order is somewhat significant */
     B2TSizeInit,
     B2TSize,
     B2TDecodeInit,
-
     B2TDecode,
     B2TDecodeList,
     B2TDecodeTuple,
@@ -1157,9 +1156,6 @@ typedef struct {
     Eterm* hp;
     Eterm* hp_end;
     int remaining_n;
-#ifdef DEBUG
-    Eterm* container_end;
-#endif
 } B2TDecodeContext;
 
 typedef struct {
@@ -1352,20 +1348,28 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
 
 #define B2T_BYTES_PER_REDUCTION 128
 
-static unsigned sverk_rand(void)
+/* Define for testing */
+/*#define EXTREME_B2T_TRAPPING 1*/
+
+#ifdef EXTREME_B2T_TRAPPING
+static unsigned b2t_rand(void)
 {
     static unsigned prev = 17;
     prev = (prev * 214013 + 2531011);
     return prev;
 }
+#endif
+
 
 static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b)
 {
-    SWord initial_reds = 1 + sverk_rand() % 4; /*(Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);*/
-    byte* bytes;
+#ifdef EXTREME_B2T_TRAPPING
+    SWord initial_reds = 1 + b2t_rand() % 4;
+#else
+    SWord initial_reds = (Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);
+#endif
     B2TContext c_buff;
     B2TContext *ctx;
-    ErtsDistExternal fakedep;
     Eterm* magic_space = NULL;
 
     if (context_b == NULL) {
@@ -1384,6 +1388,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
     do {
         switch (ctx->state) {
         case B2TPrepare: {
+	    byte* bytes;
             Uint bin_size;
             bytes = erts_get_aligned_binary_bytes_extra(bin,
                                                         &ctx->aligned_alloc,
@@ -1403,32 +1408,30 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 	    }
             break;
         }
-	case B2TUncompressChunk:
-	    {
-		Uint chunk = ctx->reds;
-		int zret;
-                if (chunk > ctx->u.uc.dleft)
-		    chunk = ctx->u.uc.dleft;
-
-		zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream,
-					      ctx->u.uc.dbytes, &chunk);
-		ctx->u.uc.dbytes += chunk;
-		ctx->u.uc.dleft  -= chunk;
-                if (zret == Z_OK && ctx->u.uc.dleft > 0) {
-		    ctx->reds = 0;
-		}
-                else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK
-			 && zret == Z_STREAM_END
-			 && ctx->u.uc.dleft == 0) {
-		    ctx->reds -= chunk;
-		    ctx->state = B2TSizeInit;
-		}
-		else {
-		    ctx->state = B2TBadArg;
-		}
-		break;
-	    }
-
+	case B2TUncompressChunk: {
+            Uint chunk = ctx->reds;
+            int zret;
+
+            if (chunk > ctx->u.uc.dleft)
+                chunk = ctx->u.uc.dleft;
+            zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream,
+                                          ctx->u.uc.dbytes, &chunk);
+            ctx->u.uc.dbytes += chunk;
+            ctx->u.uc.dleft  -= chunk;
+            if (zret == Z_OK && ctx->u.uc.dleft > 0) {
+                ctx->reds = 0;
+            }
+            else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK
+                     && zret == Z_STREAM_END
+                     && ctx->u.uc.dleft == 0) {
+                ctx->reds -= chunk;
+                ctx->state = B2TSizeInit;
+            }
+            else {
+                ctx->state = B2TBadArg;
+            }
+            break;
+        }
 	case B2TSizeInit:
 	    ctx->u.sc.ep = NULL;
 	    ctx->state = B2TSize;
@@ -1441,11 +1444,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 
         case B2TDecodeInit:
             if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) {
-                /*SVERK Can we do a better prediction that is still safe
-                    OR is there a way to do HAlloc after result some how...
-                    ... support for mutiple HReleases maybe?
-                    */
-                /* dec_term will probably trap, allocate space for magic bin
+                /* dec_term will maybe trap, allocate space for magic bin
                    before result term to make it easy to trim with HRelease.
                  */
                 magic_space = HAlloc(p, PROC_BIN_SIZE);
@@ -1462,11 +1461,12 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 	case B2TDecode:
         case B2TDecodeList:
         case B2TDecodeTuple:
-        case B2TDecodeString:
+        case B2TDecodeString: {
+	    ErtsDistExternal fakedep;
             fakedep.flags = ctx->flags;
             dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx);
             break;
-
+	}
         case B2TDecodeFail:
             HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start);
             /*fall through*/
@@ -1495,7 +1495,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             return ctx->u.dc.res;
 
         }
-    }while (ctx->reds || ctx->state >= B2TDone);
+    }while (ctx->reds > 0 || ctx->state >= B2TDone);
 
     if (context_b == NULL) {
         ASSERT(ctx->trap_bin == THE_NON_VALUE);
@@ -1524,13 +1524,6 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 
 BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1)
 {
-/*SVERK  if (++sverk_cnt % 1000 == 0) {
-        erts_fprintf(stderr, "Call #%u to binary_to_term_int()\n", sverk_cnt);
-    }
-    if (sverk_cnt == 1767) {
-        sverk_break();
-    }
-*/
     return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
 }
 
@@ -2879,7 +2872,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                     objp -= 2;
                     n--;
                 }
-                ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end);
                 break;
 
             case B2TDecodeTuple:
@@ -2889,7 +2881,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                     next = objp;
                     objp--;
                 }
-                ASSERT(ctx->u.dc.remaining_n || next == ctx->u.dc.container_end);
                 break;
 
             case B2TDecodeString:
@@ -2902,7 +2893,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                 }
                 hp[-1] = NIL;
                 *hpp = hp;
-                ASSERT(ctx->u.dc.remaining_n || hp == ctx->u.dc.container_end);
                 break;
 
             default:
@@ -3051,7 +3041,6 @@ dec_term_atom_common:
             objp = hp - 1;
             if (ctx) {
                 if (reds < n) {
-                    IF_DEBUG(ctx->u.dc.container_end = hp - n;)
                     ASSERT(reds > 0);
                     ctx->state = B2TDecodeTuple;
                     ctx->u.dc.remaining_n = n - reds;
@@ -3085,7 +3074,6 @@ dec_term_atom_common:
             n--;
 	    if (ctx) {
                 if (reds < n) {
-                    IF_DEBUG(ctx->u.dc.container_end = hp - 2*(n+1);)
 		    ctx->state = B2TDecodeList;
 		    ctx->u.dc.remaining_n = n - reds;
 		    n = reds;
@@ -3110,7 +3098,6 @@ dec_term_atom_common:
 	    *objp = make_list(hp);
             if (ctx) {
                 if (reds < n) {
-                    IF_DEBUG(ctx->u.dc.container_end = hp + n*2;)
                     ctx->state = B2TDecodeString;
                     ctx->u.dc.remaining_n = n - reds;
                     n = reds;
@@ -3981,6 +3968,9 @@ decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
             goto init_done;
         }
     }
+    else
+        reds = 0; /* not used but compiler warns anyway */
+
     heap_size = 0;
     terms = 1;
     atom_extra_skip = 0;
@@ -4016,9 +4006,6 @@ init_done:
     ASSERT(terms > 0);
     do {
         int tag;
-        /*SVERK
-        erts_fprintf(stderr, "SVERK tag=%d ep=%p terms=%d heap_size=%d endp=%p\n",
-                     ep[0], ep, terms, heap_size, endp); */
 	CHKSIZE(1);
 	tag = ep++[0];
 	switch (tag) {
-- 
cgit v1.2.3


From bcb227d1f467439fa5f901233e8ad382e75373cb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 4 Nov 2013 17:53:43 +0100
Subject: erts: Trapping memcpy in binary_to_term

---
 erts/emulator/beam/external.c | 73 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 62 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 651aac03a2..a73734d487 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1135,6 +1135,7 @@ enum B2TState { /* order is somewhat significant */
     B2TDecodeList,
     B2TDecodeTuple,
     B2TDecodeString,
+    B2TDecodeBinary,
 
     B2TDone,
     B2TDecodeFail,
@@ -1156,6 +1157,7 @@ typedef struct {
     Eterm* hp;
     Eterm* hp_end;
     int remaining_n;
+    char* remaining_bytes;
 } B2TDecodeContext;
 
 typedef struct {
@@ -1347,6 +1349,7 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
 
 
 #define B2T_BYTES_PER_REDUCTION 128
+#define B2T_MEMCPY_FACTOR 8
 
 /* Define for testing */
 /*#define EXTREME_B2T_TRAPPING 1*/
@@ -1461,7 +1464,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 	case B2TDecode:
         case B2TDecodeList:
         case B2TDecodeTuple:
-        case B2TDecodeString: {
+        case B2TDecodeString:
+        case B2TDecodeBinary: {
 	    ErtsDistExternal fakedep;
             fakedep.flags = ctx->flags;
             dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx);
@@ -1494,6 +1498,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
             return ctx->u.dc.res;
 
+        default:
+            ASSERT(!"Unknown state in binary_to_term");
         }
     }while (ctx->reds > 0 || ctx->state >= B2TDone);
 
@@ -2830,6 +2836,7 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
 #endif /* DEBUG */
 }
 
+
 /* Decode term from external format into *objp.
 ** On failure return NULL and (R13B04) *hpp will be unchanged.
 */
@@ -2852,15 +2859,25 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
         hpp      = &ctx->u.dc.hp;
 
         if (ctx->state != B2TDecode) {
-            n = ctx->u.dc.remaining_n;
-            if (reds < n) {
-                ctx->u.dc.remaining_n -= reds;
-                n = reds;
+            int n_limit = reds;
+
+	    n = ctx->u.dc.remaining_n;
+            if (ctx->state == B2TDecodeBinary) {
+                n_limit *= B2T_MEMCPY_FACTOR;
+                ASSERT(n_limit >= reds);
+		reds -= n / B2T_MEMCPY_FACTOR;
+            }
+	    else
+		reds -= n;
+
+            if (n > n_limit) {
+                ctx->u.dc.remaining_n -= n_limit;
+                n = n_limit;
+                reds = 0;
             }
             else {
                 ctx->u.dc.remaining_n = 0;
             }
-            reds -= n;
 
             switch (ctx->state) {
             case B2TDecodeList:
@@ -2895,6 +2912,12 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                 *hpp = hp;
                 break;
 
+            case B2TDecodeBinary:
+                sys_memcpy(ctx->u.dc.remaining_bytes, ep, n);
+                ctx->u.dc.remaining_bytes += n;
+                ep += n;
+                break;
+
             default:
                 ASSERT(!"Unknown state");
             }
@@ -3311,7 +3334,6 @@ dec_term_atom_common:
 		    dbin->flags = 0;
 		    dbin->orig_size = n;
 		    erts_refc_init(&dbin->refc, 1);
-		    sys_memcpy(dbin->orig_bytes, ep, n);
 		    pb = (ProcBin *) hp;
 		    hp += PROC_BIN_SIZE;
 		    pb->thing_word = HEADER_PROC_BIN;
@@ -3322,7 +3344,20 @@ dec_term_atom_common:
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
 		    *objp = make_binary(pb);
-		}
+                    if (ctx) {
+                        int n_limit = reds * B2T_MEMCPY_FACTOR;
+                        if (n > n_limit) {
+                            ctx->state = B2TDecodeBinary;
+                            ctx->u.dc.remaining_n = n - n_limit;
+                            ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit;
+                            n = n_limit;
+                            reds = 0;
+                        }
+                        else
+                            reds -= n / B2T_MEMCPY_FACTOR;
+                    }
+                    sys_memcpy(dbin->orig_bytes, ep, n);
+                }
 		ep += n;
 		break;
 	    }
@@ -3343,13 +3378,14 @@ dec_term_atom_common:
 		    sys_memcpy(hb->data, ep, n);
 		    bin = make_binary(hb);
 		    hp += heap_bin_size(n);
+                    ep += n;
 		} else {
 		    Binary* dbin = erts_bin_nrml_alloc(n);
 		    ProcBin* pb;
+
 		    dbin->flags = 0;
 		    dbin->orig_size = n;
 		    erts_refc_init(&dbin->refc, 1);
-		    sys_memcpy(dbin->orig_bytes, ep, n);
 		    pb = (ProcBin *) hp;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
@@ -3360,8 +3396,23 @@ dec_term_atom_common:
 		    pb->flags = 0;
 		    bin = make_binary(pb);
 		    hp += PROC_BIN_SIZE;
-		}
-		ep += n;
+                    if (ctx) {
+                        int n_limit = reds * B2T_MEMCPY_FACTOR;
+                        if (n > n_limit) {
+                            ctx->state = B2TDecodeBinary;
+                            ctx->u.dc.remaining_n = n - n_limit;
+                            ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit;
+                            n = n_limit;
+                            reds = 0;
+                        }
+                        else
+                            reds -= n / B2T_MEMCPY_FACTOR;
+                    }
+                    sys_memcpy(dbin->orig_bytes, ep, n);
+                    ep += n;
+                    n = pb->size;
+                }
+
 		if (bitsize == 0) {
 		    *objp = bin;
 		} else {
-- 
cgit v1.2.3


From adcdcb2c7985ad6076a2bedebdfce08d14465b43 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 5 Nov 2013 18:44:39 +0100
Subject: erts: Fix crash when binary_to_term throws badarg

after it has built off_heap data
and then done at least one trap call.

The undo mechanism in dec_term does not work
if we build the magic binary after any other off_heap data.
---
 erts/emulator/beam/external.c | 51 +++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index a73734d487..6a7d15cc88 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1364,6 +1364,21 @@ static unsigned b2t_rand(void)
 #endif
 
 
+static B2TContext* b2t_export_context(Process* p, B2TContext* src)
+{
+    Binary* context_b = erts_create_magic_binary(sizeof(B2TContext),
+                                                 b2t_context_destructor);
+    B2TContext* ctx = ERTS_MAGIC_BIN_DATA(context_b);
+    Eterm* hp;
+    sys_memcpy(ctx, src, sizeof(B2TContext));
+    if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) {
+        ctx->u.dc.next = &ctx->u.dc.res;
+    }
+    hp = HAlloc(p, PROC_BIN_SIZE);
+    ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), context_b);
+    return ctx;
+}
+
 static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b)
 {
 #ifdef EXTREME_B2T_TRAPPING
@@ -1373,16 +1388,18 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 #endif
     B2TContext c_buff;
     B2TContext *ctx;
-    Eterm* magic_space = NULL;
+    int is_first_call;
 
     if (context_b == NULL) {
 	/* Setup enough to get started */
+        is_first_call = 1;
         ctx = &c_buff;
 	ctx->state = B2TPrepare;
         ctx->aligned_alloc = NULL;
         ctx->flags = flags;
         IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
     } else {
+        is_first_call = 0;
 	ctx = ERTS_MAGIC_BIN_DATA(context_b);
         ASSERT(ctx->state != B2TPrepare);
     }
@@ -1446,12 +1463,11 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             break;
 
         case B2TDecodeInit:
-            if (context_b == NULL && ctx->b2ts.extsize > ctx->reds) {
+            if (ctx == &c_buff && ctx->b2ts.extsize > ctx->reds) {
                 /* dec_term will maybe trap, allocate space for magic bin
                    before result term to make it easy to trim with HRelease.
                  */
-                magic_space = HAlloc(p, PROC_BIN_SIZE);
-                *magic_space = make_pos_bignum_header(PROC_BIN_SIZE-1);
+                ctx = b2t_export_context(p, &c_buff);
             }
             ctx->u.dc.ep = ctx->b2ts.extp;
             ctx->u.dc.res = (Eterm) (UWord) NULL;
@@ -1476,7 +1492,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             /*fall through*/
         case B2TBadArg:
             b2t_destroy_context(ctx);
-            if (context_b) {
+            if (!is_first_call) {
                 erts_set_gc_state(p, 1);
             }
             BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
@@ -1491,10 +1507,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             }
             HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp);
 
-            if (context_b) {
+            if (!is_first_call) {
                 erts_set_gc_state(p, 1);
             }
-
             BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
             return ctx->u.dc.res;
 
@@ -1503,27 +1518,15 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
         }
     }while (ctx->reds > 0 || ctx->state >= B2TDone);
 
-    if (context_b == NULL) {
+    if (ctx == &c_buff) {
         ASSERT(ctx->trap_bin == THE_NON_VALUE);
-
-        context_b = erts_create_magic_binary(sizeof(B2TContext),
-                                             b2t_context_destructor);
-        ctx = ERTS_MAGIC_BIN_DATA(context_b);
-        sys_memcpy(ctx, &c_buff, sizeof(B2TContext));
-        if (ctx->state >= B2TDecode && ctx->u.dc.next == &c_buff.u.dc.res) {
-            ctx->u.dc.next = &ctx->u.dc.res;
-        }
-        if (!magic_space) {
-            ASSERT(ctx->state < B2TDecode);
-            magic_space = HAlloc(p, PROC_BIN_SIZE);
-        }
-        ctx->trap_bin = erts_mk_magic_binary_term(&magic_space, &MSO(p), context_b);
-
-        erts_set_gc_state(p, 0);
+        ctx = b2t_export_context(p, &c_buff);
     }
-    ASSERT(context_b);
     ASSERT(ctx->trap_bin != THE_NON_VALUE);
 
+    if (is_first_call) {
+        erts_set_gc_state(p, 0);
+    }
     BUMP_ALL_REDS(p);
     BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin);
 }
-- 
cgit v1.2.3


From 53c26db17f6bf89783e0dd48f7777cfeff18f756 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 5 Nov 2013 21:59:53 +0100
Subject: erts: Fix bug in binary_to_term for compressed on halfword

---
 erts/emulator/beam/external.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 6a7d15cc88..28c6caf5fe 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1429,7 +1429,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             break;
         }
 	case B2TUncompressChunk: {
-            Uint chunk = ctx->reds;
+            uLongf chunk = ctx->reds;
             int zret;
 
             if (chunk > ctx->u.uc.dleft)
-- 
cgit v1.2.3


From 5a00e724a58ee29d4012cca79c8aa33979e74eb6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 20 Nov 2013 21:33:20 +0100
Subject: erts: Fix alignment bug in allocator start code

Bug never released.
---
 erts/emulator/beam/erl_alloc_util.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 1fdee4db2c..00b87aca8b 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -372,6 +372,8 @@ do {										\
 
 #define ERTS_CRR_ALCTR_FLG_IN_POOL	(((erts_aint_t) 1) << 0)
 #define ERTS_CRR_ALCTR_FLG_BUSY		(((erts_aint_t) 1) << 1)
+#define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \
+                                 ERTS_CRR_ALCTR_FLG_BUSY)
 
 #ifdef ERTS_SMP
 #define SBC_HEADER_SIZE	   						\
@@ -1404,14 +1406,14 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 
 	    if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock
 		&& pref_allctr->thread_safe) {
-		used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+		used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK);
 		if (pref_allctr == used_allctr) {
 		    erts_mtx_lock(&pref_allctr->mutex);
 		    locked_pref_allctr = 1;
 		}
 	    }
 
-	    while ((iallctr & ((~FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL))
+	    while ((iallctr & ((~ERTS_CRR_ALCTR_FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL))
 		   == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) {
 		erts_aint_t act;
 
@@ -1426,7 +1428,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 		iallctr = act;
 	    }
 
-	    used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+	    used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK);
 
 	    if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) {
 		if (locked_pref_allctr && used_allctr != pref_allctr) {
@@ -1436,16 +1438,16 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 	    }
 
 	    ERTS_ALC_CPOOL_ASSERT(
-		(((iallctr & ~FLG_MASK) == (erts_aint_t) pref_allctr)
-		 ? (((iallctr & FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL)
-		    || ((iallctr & FLG_MASK) == 0))
+		(((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr)
+		 ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL)
+		    || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0))
 		 : 1));
 
 	    return used_allctr;
 	}
     }
 
-    used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+    used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK);
 
     if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock
 	&& used_allctr == pref_allctr
@@ -1776,7 +1778,7 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
 	    ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr)
 				  != (erts_smp_atomic_read_nob(&crr->allctr)
-				      & ~FLG_MASK));
+				      & ~ERTS_CRR_ALCTR_FLG_MASK));
 
 	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
 
@@ -2919,7 +2921,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 #ifdef ERTS_ALC_CPOOL_DEBUG
 	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
 							   ((erts_aint_t) allctr))
-				  == (((erts_aint_t) allctr) & ~FLG_MASK));
+				  == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK));
 #else
 	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
 #endif
@@ -2961,7 +2963,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 					     (erts_aint_t) allctr,
 					     exp);
 	    if (act == exp) {
-		cpool_delete(allctr, ((Allctr_t *) (act & ~FLG_MASK)), crr);
+		cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
 		return crr;
 	    }
 	}
@@ -3056,7 +3058,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 	ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk));
 	ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr)
 			      == (erts_smp_atomic_read_nob(&crr->allctr)
-				  & ~FLG_MASK));
+				  & ~ERTS_CRR_ALCTR_FLG_MASK));
 
 	if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit))
 	    erts_alloc_notify_delayed_dealloc(orig_allctr->ix);
@@ -5422,6 +5424,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 {
     /* erts_alcu_start assumes that allctr has been zeroed */
 
+    if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) {
+        erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n",
+                 __FILE__, __LINE__);
+    }
+
     if (!initialized)
 	goto error;
 
-- 
cgit v1.2.3


From 522a29666088d5d96956d2752ffb1596d778cffd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 13 Nov 2013 18:07:33 +0100
Subject: erts: Let term_to_binary disable gc while trapping

as an attempt to improve performance
---
 erts/emulator/beam/external.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 28c6caf5fe..d5f3b19b82 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1055,8 +1055,10 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
     Binary *bin = ((ProcBin *) binary_val(bt))->val;
     Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin);
     if (is_tuple(res)) {
+	ASSERT(BIF_P->flags & F_DISABLE_GC);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
+	erts_set_gc_state(BIF_P, 1);
 	BIF_RET(res);
     }
 }
@@ -1065,8 +1067,10 @@ BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
 {
     Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL);
     if (is_tuple(res)) {
+	erts_set_gc_state(BIF_P, 0);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
+	ASSERT(!(BIF_P->flags & F_DISABLE_GC));
 	BIF_RET(res);
     }
 }
@@ -1118,8 +1122,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 
     res = erts_term_to_binary_int(p, Term, level, flags, bin);
     if (is_tuple(res)) {
+	erts_set_gc_state(p, 0);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
+	ASSERT(!(BIF_P->flags & F_DISABLE_GC));
 	BIF_RET(res);
     }
 }
-- 
cgit v1.2.3


From 1f09936f34f5daee534bbfde4f16e5bbb434b6c4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 13 Nov 2013 18:46:17 +0100
Subject: erts: Yield after trapping term_to_binary if gc has been ordered

or if "too much" offheap binaries has been built
---
 erts/emulator/beam/erl_bif_info.c | 5 +++--
 erts/emulator/beam/erl_process.c  | 9 +++------
 erts/emulator/beam/external.c     | 7 +++++--
 3 files changed, 11 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f5893f9291..39bbd6b182 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3603,8 +3603,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    default: BIF_ERROR(BIF_P, BADARG); break;
 	    }
  
-	    res = erts_set_gc_state(BIF_P, enable);
-	    BIF_RET(res ? am_true : am_false);
+            res = (BIF_P->flags & F_DISABLE_GC) ? am_false : am_true;
+	    erts_set_gc_state(BIF_P, enable);
+	    BIF_RET(res);
 	}
 	else if (ERTS_IS_ATOM_STR("send_fake_exit_signal", BIF_ARG_1)) {
 	    /* Used by signal_SUITE (emulator) */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 13b18e9e0e..b88c871ffc 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8271,25 +8271,22 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
 int
 erts_set_gc_state(Process *c_p, int enable)
 {
-    int res;
     ErtsProcSysTaskQs *dgc_tsk_qs;
     ASSERT(c_p == erts_get_current_process());
     ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
 	   & erts_smp_atomic32_read_nob(&c_p->state));
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
 
-    res = !(c_p->flags & F_DISABLE_GC);
-
     if (!enable) {
 	c_p->flags |= F_DISABLE_GC;
-	return res;
+	return 0;
     }
 
     c_p->flags &= ~F_DISABLE_GC;
 
     dgc_tsk_qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p);
     if (!dgc_tsk_qs)
-	return res;
+	return 0;
 
     /* Move delayed gc tasks into sys tasks queues. */
 
@@ -8387,7 +8384,7 @@ erts_set_gc_state(Process *c_p, int enable)
     if (dgc_tsk_qs)
 	proc_sys_task_queues_free(dgc_tsk_qs);
 
-    return res;
+    return 1;
 }
 
 void
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index d5f3b19b82..7dc7ba6f98 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1058,8 +1058,11 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
 	ASSERT(BIF_P->flags & F_DISABLE_GC);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
     } else {
-	erts_set_gc_state(BIF_P, 1);
-	BIF_RET(res);
+        if (erts_set_gc_state(BIF_P, 1)
+            || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P))
+            ERTS_BIF_YIELD_RETURN(BIF_P, res);
+        else
+            BIF_RET(res);
     }
 }
     
-- 
cgit v1.2.3


From 6cff38512b753172a7dfa2bedd60e8987156736d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Oct 2013 20:54:12 +0200
Subject: erts: Adjust term_to_binary reduction factors

Made them powers of 2 for faster calculations.

500 encoded terms per reductions seemed a bit much, lowered to 32.

TERM_TO_BINARY_SIZE_FACTOR was not used in practice as it was only
applied to small binaries.

Lowered from 500kb to 256kb compressed bytes per trap call.
---
 erts/emulator/beam/external.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 7dc7ba6f98..4600d691c7 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1717,12 +1717,10 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
 /* #define EXTREME_TTB_TRAPPING 1 */
 
 #ifndef EXTREME_TTB_TRAPPING
-#define TERM_TO_BINARY_LOOP_FACTOR 500
-#define TERM_TO_BINARY_SIZE_FACTOR 500000
-#define TERM_TO_BINARY_COMPRESS_CHUNK 500000
+#define TERM_TO_BINARY_LOOP_FACTOR 32
+#define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18)
 #else
 #define TERM_TO_BINARY_LOOP_FACTOR 1
-#define TERM_TO_BINARY_SIZE_FACTOR 10
 #define TERM_TO_BINARY_COMPRESS_CHUNK 10
 #endif
 
@@ -1859,7 +1857,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		    /* Finish in one go */
 		    res = erts_term_to_binary_simple(p, Term, size, 
 						     level, flags);
-		    BUMP_REDS(p, size / TERM_TO_BINARY_SIZE_FACTOR);
+		    BUMP_REDS(p, 1);
 		    return res;
 		}
 
-- 
cgit v1.2.3


From e9c04e92199deb274e1bb74e4caecd296443ca5f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Wed, 20 Nov 2013 14:34:06 +0100
Subject: Improve error info when main carrier creation fails

Also fix testcase that failed due to main carrier creation failure.
---
 erts/emulator/beam/erl_alloc_util.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 1fdee4db2c..b59fe0bf15 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -5593,8 +5593,15 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 			     | CFLG_FORCE_SYS_ALLOC
 #endif
 			     | CFLG_MAIN_CARRIER);
-	if (!blk)
-	    goto error;
+	if (!blk) {
+#ifdef USE_THREADS
+	  if (allctr->thread_safe)
+	    erts_mtx_destroy(&allctr->mutex);
+#endif
+	  erl_exit(ERTS_ABORT_EXIT,
+	    "Failed to create main carrier for %salloc\n",
+	    init->name_prefix);
+	}
 
 	(*allctr->link_free_block)(allctr, blk);
 
-- 
cgit v1.2.3


From dc9ff931423b57e64abde0f1de7133f334abb780 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 25 Nov 2013 15:34:03 +0100
Subject: Ensure exit signal due to link precede port BIF return

---
 erts/emulator/beam/bif.c           | 37 ++++++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_bif_port.c  | 34 +++++++++++++++++++++++-----------
 erts/emulator/beam/erl_port_task.h | 11 ++++++++++-
 erts/emulator/beam/io.c            |  2 ++
 4 files changed, 65 insertions(+), 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 755c5e6882..13d31285b2 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -160,7 +160,10 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 
     if (is_internal_port(BIF_ARG_1)) {
 	int send_link_signal = 0;
-	Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+	Port *prt = erts_port_lookup(BIF_ARG_1,
+				     (erts_port_synchronous_ops
+				      ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+				      : ERTS_PORT_SFLGS_INVALID_LOOKUP));
 	if (!prt) {
 	    goto res_no_proc;
 	}
@@ -1363,11 +1366,22 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
       */
 
      if (is_internal_port(BIF_ARG_1)) {
-	 Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+	 Eterm ref, *refp;
+	 Uint32 invalid_flags;
+	 Port *prt;
+
+	 if (erts_port_synchronous_ops) {
+	     refp = &ref;
+	     invalid_flags = ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP;
+	 }
+	 else {
+	     refp = NULL;
+	     invalid_flags = ERTS_PORT_SFLGS_INVALID_LOOKUP;
+	 }
+
+	 prt = erts_port_lookup(BIF_ARG_1, invalid_flags);
 
 	 if (prt) {
-	     Eterm ref;
-	     Eterm *refp = erts_port_synchronous_ops ? &ref : NULL;
 	     ErtsPortOpResult res;
 
 #ifdef DEBUG
@@ -1875,7 +1889,10 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 	if (rp)
 	    goto send_message;
 
-	pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+	pt = erts_port_lookup(id,
+			      (erts_port_synchronous_ops
+			       ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+			       : ERTS_PORT_SFLGS_INVALID_LOOKUP));
 	if (pt) {
 	    portid = id;
 	    goto port_common;
@@ -1905,7 +1922,10 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 	int ret_val;
 	portid = to;
 
-	pt = erts_port_lookup(portid, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+	pt = erts_port_lookup(portid,
+			      (erts_port_synchronous_ops
+			       ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+			       : ERTS_PORT_SFLGS_INVALID_LOOKUP));
 
       port_common:
 	ret_val = 0;
@@ -1994,7 +2014,10 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 	    rp = erts_proc_lookup_raw(id);
 	    if (rp)
 		goto send_message;
-	    pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+	    pt = erts_port_lookup(id,
+				  (erts_port_synchronous_ops
+				   ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+				   : ERTS_PORT_SFLGS_INVALID_LOOKUP));
 	    if (pt) {
 		portid = id;
 		goto port_common;
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 109c54fd7f..3cd53ef65d 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -84,7 +84,7 @@ BIF_RETTYPE open_port_2(BIF_ALIST_2)
 }
 
 static ERTS_INLINE Port *
-lookup_port(Process *c_p, Eterm id_or_name)
+lookup_port(Process *c_p, Eterm id_or_name, Uint32 invalid_flags)
 {
     /* TODO: Implement nicer lookup in register... */
     Eterm id;
@@ -92,7 +92,19 @@ lookup_port(Process *c_p, Eterm id_or_name)
 	id = erts_whereis_name_to_id(c_p, id_or_name);
     else
 	id = id_or_name;
-    return erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+    return erts_port_lookup(id, invalid_flags);
+}
+
+static ERTS_INLINE Port *
+sig_lookup_port(Process *c_p, Eterm id_or_name)
+{
+    return lookup_port(c_p, id_or_name, ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
+}
+
+static ERTS_INLINE Port *
+data_lookup_port(Process *c_p, Eterm id_or_name)
+{
+    return lookup_port(c_p, id_or_name, ERTS_PORT_SFLGS_INVALID_LOOKUP);
 }
 
 /*
@@ -125,7 +137,7 @@ BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3)
 	    BIF_RET(am_badarg);
     }
 
-    prt = lookup_port(BIF_P, BIF_ARG_1);
+    prt = sig_lookup_port(BIF_P, BIF_ARG_1);
     if (!prt)
 	BIF_RET(am_badarg);
 
@@ -185,7 +197,7 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3)
     unsigned int op;
     erts_aint32_t state;
 
-    prt = lookup_port(BIF_P, BIF_ARG_1);
+    prt = sig_lookup_port(BIF_P, BIF_ARG_1);
     if (!prt)
 	BIF_RET(am_badarg);
 
@@ -235,7 +247,7 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
     unsigned int op;
     erts_aint32_t state;
 
-    prt = lookup_port(BIF_P, BIF_ARG_1);
+    prt = sig_lookup_port(BIF_P, BIF_ARG_1);
     if (!prt)
 	BIF_RET(am_badarg);
 
@@ -290,7 +302,7 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1)
     ref = NIL;
 #endif
 
-    prt = lookup_port(BIF_P, BIF_ARG_1);
+    prt = sig_lookup_port(BIF_P, BIF_ARG_1);
     if (!prt)
 	BIF_RET(am_badarg);
 
@@ -320,7 +332,7 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2)
     Eterm ref;
     Port* prt;
 
-    prt = lookup_port(BIF_P, BIF_ARG_1);
+    prt = sig_lookup_port(BIF_P, BIF_ARG_1);
     if (!prt)
 	BIF_RET(am_badarg);
 
@@ -352,7 +364,7 @@ BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1)
     Port* prt;
 
     if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) {
-	prt = lookup_port(BIF_P, BIF_ARG_1);
+	prt = sig_lookup_port(BIF_P, BIF_ARG_1);
 	if (!prt)
 	    BIF_RET(am_undefined);
     }
@@ -391,7 +403,7 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2)
     Port* prt;
 
     if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) {
-	prt = lookup_port(BIF_P, BIF_ARG_1);
+	prt = sig_lookup_port(BIF_P, BIF_ARG_1);
 	if (!prt)
 	    BIF_RET(am_undefined);
     }
@@ -523,7 +535,7 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2)
     erts_aint_t data;
     Port* prt;
 
-    prt = lookup_port(BIF_P, BIF_ARG_1);
+    prt = data_lookup_port(BIF_P, BIF_ARG_1);
     if (!prt)
         BIF_ERROR(BIF_P, BADARG);
 
@@ -564,7 +576,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
     erts_aint_t data;
     Port* prt;
 
-    prt = lookup_port(BIF_P, BIF_ARG_1);
+    prt = data_lookup_port(BIF_P, BIF_ARG_1);
     if (!prt)
         BIF_ERROR(BIF_P, BADARG);
 
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index e4d964146e..123253a057 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -77,6 +77,7 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
 #define ERTS_PTS_FLG_HAVE_NS_TASKS		(((erts_aint32_t) 1) <<  8)
 #define ERTS_PTS_FLG_PARALLELISM		(((erts_aint32_t) 1) <<  9)
 #define ERTS_PTS_FLG_FORCE_SCHED		(((erts_aint32_t) 1) << 10)
+#define ERTS_PTS_FLG_EXITING			(((erts_aint32_t) 1) << 11)
 
 #define ERTS_PTS_FLGS_BUSY \
     (ERTS_PTS_FLG_BUSY_PORT | ERTS_PTS_FLG_BUSY_PORT_Q)
@@ -86,7 +87,8 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
      | ERTS_PTS_FLG_HAVE_BUSY_TASKS		\
      | ERTS_PTS_FLG_HAVE_TASKS			\
      | ERTS_PTS_FLG_EXEC			\
-     | ERTS_PTS_FLG_FORCE_SCHED)
+     | ERTS_PTS_FLG_FORCE_SCHED			\
+     | ERTS_PTS_FLG_EXITING)
 
 #define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH			8192
 #define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW			4096
@@ -135,6 +137,7 @@ ERTS_GLB_INLINE void erts_port_task_fini_sched(ErtsPortTaskSched *ptsp);
 ERTS_GLB_INLINE void erts_port_task_sched_lock(ErtsPortTaskSched *ptsp);
 ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp);
 ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp);
+ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp);
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
 ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
@@ -225,6 +228,12 @@ erts_port_task_fini_sched(ErtsPortTaskSched *ptsp)
 #endif
 }
 
+ERTS_GLB_INLINE void
+erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp)
+{
+    erts_smp_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING);
+}
+
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
 
 ERTS_GLB_INLINE int
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 6911fd8241..9076bbe73c 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3601,6 +3601,8 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
    if (send_closed)
        set_state_flags |= ERTS_PORT_SFLG_SEND_CLOSED;
 
+   erts_port_task_sched_enter_exiting_state(&p->sched);
+   
    state = erts_atomic32_read_bor_mb(&p->state, set_state_flags);
    state |= set_state_flags;
 
-- 
cgit v1.2.3


From a1ce4da11ca0710a166e5048251aa1e128a83695 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 26 Nov 2013 20:25:53 +0100
Subject: erts: Fix invalid read in bitstring comparison

Valgrind complained that erts_cmp_bits sometimes read one byte too much
from bitstrings. The byte was never used and the invalid read is safe
in opt-VM due to padding in binary allocation (CHICKED_PAD).
---
 erts/emulator/beam/erl_bits.c | 79 +++++++++++++++++++++++++++----------------
 1 file changed, 49 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 43eb691338..d5b0e341a1 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1810,6 +1810,11 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz
     Uint rshift;
     int cmp;
     
+    ASSERT(a_offs < 8 && b_offs < 8);
+
+    if (size == 0)
+        return 0;
+
     if (((a_offs | b_offs | size) & 7) == 0) {
 	int byte_size = size >> 3;
 	return sys_memcmp(a_ptr, b_ptr, byte_size);
@@ -1818,13 +1823,15 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz
     /* Compare bit by bit until a_ptr is aligned on byte boundary */
     a = *a_ptr++;
     b = *b_ptr++;
-    while (size > 0) {
+    for (;;) {
 	a_bit = get_bit(a, a_offs);
 	b_bit = get_bit(b, b_offs);
 	if ((cmp = (a_bit-b_bit)) != 0) {
 	    return cmp;
 	}
-	size--;
+        if (--size == 0)
+            return 0;
+
 	b_offs++;
 	if (b_offs == 8) {
 	    b_offs = 0;
@@ -1839,37 +1846,49 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz
     }
 
     /* Compare byte by byte as long as at least 8 bits remain */
-    lshift = b_offs;
-    rshift = 8 - lshift;
-    while (size >= 8) {
-	byte b_cmp = (b << lshift);
-	b = *b_ptr++;
-	b_cmp |= b >> rshift;
-	if ((cmp = (a - b_cmp)) != 0) {
-	    return cmp;
-	}
+    if (size >= 8) {
+        lshift = b_offs;
+        rshift = 8 - lshift;
+        for (;;) {
+            byte b_cmp = (b << lshift);
+            b = *b_ptr++;
+            b_cmp |= b >> rshift;
+            if ((cmp = (a - b_cmp)) != 0) {
+                return cmp;
+            }
+            size -= 8;
+	    if (size < 8)
+		break;
+            a = *a_ptr++;
+        }
+
+	if (size == 0)
+	    return 0;
 	a = *a_ptr++;
-	size -= 8;
     }
 
     /* Compare the remaining bits bit by bit */
-    while (size > 0) {
-	a_bit = get_bit(a, a_offs);
-	b_bit = get_bit(b, b_offs);
-	if ((cmp = (a_bit-b_bit)) != 0) {
-	    return cmp;
-	}
-	a_offs++;
-	if (a_offs == 8) {
-	    a_offs = 0;
-	    a = *a_ptr++;
-	}
-	b_offs++;
-	if (b_offs == 8) {
-	    b_offs = 0;
-	    b = *b_ptr++;
-	}
-	size--;
+    if (size > 0) {
+        for (;;) {
+            a_bit = get_bit(a, a_offs);
+            b_bit = get_bit(b, b_offs);
+            if ((cmp = (a_bit-b_bit)) != 0) {
+                return cmp;
+            }
+            if (--size == 0)
+                return 0;
+
+            a_offs++;
+            if (a_offs == 8) {
+                a_offs = 0;
+                a = *a_ptr++;
+            }
+            b_offs++;
+            if (b_offs == 8) {
+                b_offs = 0;
+                b = *b_ptr++;
+            }
+        }
     }
 
     return 0;
-- 
cgit v1.2.3


From f3589a79de29f1a4e6a2d03518739af214ae9600 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 26 Nov 2013 20:31:43 +0100
Subject: erts: Optimize comparison for bitstrings with byte aligned start

---
 erts/emulator/beam/erl_bits.c | 44 +++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index d5b0e341a1..872ded4be7 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1823,25 +1823,27 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz
     /* Compare bit by bit until a_ptr is aligned on byte boundary */
     a = *a_ptr++;
     b = *b_ptr++;
-    for (;;) {
-	a_bit = get_bit(a, a_offs);
-	b_bit = get_bit(b, b_offs);
-	if ((cmp = (a_bit-b_bit)) != 0) {
-	    return cmp;
-	}
-        if (--size == 0)
-            return 0;
+    if (a_offs) {
+	for (;;) {
+	    a_bit = get_bit(a, a_offs);
+	    b_bit = get_bit(b, b_offs);
+	    if ((cmp = (a_bit-b_bit)) != 0) {
+		return cmp;
+	    }
+	    if (--size == 0)
+		return 0;
 
-	b_offs++;
-	if (b_offs == 8) {
-	    b_offs = 0;
-	    b = *b_ptr++;
-	}
-	a_offs++;
-	if (a_offs == 8) {
-	    a_offs = 0;
-	    a = *a_ptr++;
-	    break;
+	    b_offs++;
+	    if (b_offs == 8) {
+		b_offs = 0;
+		b = *b_ptr++;
+	    }
+	    a_offs++;
+	    if (a_offs == 8) {
+		a_offs = 0;
+		a = *a_ptr++;
+		break;
+	    }
 	}
     }
 
@@ -1879,10 +1881,8 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz
                 return 0;
 
             a_offs++;
-            if (a_offs == 8) {
-                a_offs = 0;
-                a = *a_ptr++;
-            }
+	    ASSERT(a_offs < 8);
+
             b_offs++;
             if (b_offs == 8) {
                 b_offs = 0;
-- 
cgit v1.2.3


From 35b4fd5d91cbeb1b9c8540996624856134e091e9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 27 Nov 2013 18:58:14 +0100
Subject: erts: Fix invalid read when appending binaries during call trace

Found by valgrind. Probably safe on opt-VM due to CHICKEN_PAD.
---
 erts/emulator/beam/erl_bits.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 872ded4be7..73765772c8 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1491,7 +1491,7 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
 	    bptr->flags = 0;
 	    bptr->orig_size = new_size;
 	    erts_refc_init(&bptr->refc, 1);
-	    sys_memcpy(bptr->orig_bytes, binp->orig_bytes, pb->size);
+	    sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size);
 	    pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER;
 	    pb->val = bptr;
 	    pb->bytes = (byte *) bptr->orig_bytes;
-- 
cgit v1.2.3


From ce785835f3fcbab8b6c07e2415b5f636d181c28d Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Mon, 14 Oct 2013 23:26:56 +0200
Subject: Remove uninitialized use of new_crr in erl_alloc_util

When the offending code is reached, new_crr is either uninitalized or
have been set to NULL. This patch removes the following warning:

beam/erl_alloc_util.c:3510:6: warning: variable 'new_crr' is used uninitialized whenever 'if' condition is false [-Wsometimes-uninitialized]
        if (!(flags & CFLG_FORCE_MSEG)) {
            ^~~~~~~~~~~~~~~~~~~~~~~~~~
beam/erl_alloc_util.c:3567:23: note: uninitialized use occurs here
        DEBUG_SAVE_ALIGNMENT(new_crr);
                             ^~~~~~~
beam/erl_alloc_util.c:674:51: note: expanded from macro 'DEBUG_SAVE_ALIGNMENT'
    UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t));\
                                                  ^
beam/erl_alloc_util.c:3510:2: note: remove the 'if' if its condition is always true
        if (!(flags & CFLG_FORCE_MSEG)) {
        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
beam/erl_alloc_util.c:3438:23: note: initialize the variable 'new_crr' to silence this warning
    Carrier_t *new_crr, *old_crr;
                      ^
                       = NULL
---
 erts/emulator/beam/erl_alloc_util.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 3914537d0d..c6cea0185f 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3570,7 +3570,6 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 	    /* Old carrier unchanged; restore... */
 	    STAT_SYS_ALLOC_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz);
 	}
-	DEBUG_SAVE_ALIGNMENT(new_crr);
 	return new_blk;
     }
 #endif
-- 
cgit v1.2.3


From 2567dfefa9426c61cdbea3c984e6c7e7d6dad1ea Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Mon, 14 Oct 2013 23:38:27 +0200
Subject: Properly mark erl_assert_error as non-returning

---
 erts/emulator/beam/sys.h | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d22f125945..31252ed78f 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -149,9 +149,28 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #  define ERTS_EXIT_AFTER_DUMP exit
 #endif
 
+/* In VC++, noreturn is a declspec that has to be before the types,
+ * but in GNUC it is an att ribute to be placed between return type
+ * and function name, hence __decl_noreturn <types> __noreturn <function name>
+ */
+#if __GNUC__
+#  define __decl_noreturn
+#  define __noreturn __attribute__((noreturn))
+#else
+#  if defined(__WIN32__) && defined(_MSC_VER)
+#    define __noreturn
+#    define __decl_noreturn __declspec(noreturn)
+#  else
+#    define __noreturn
+#    define __decl_noreturn
+#  endif
+#endif
+
 #define ERTS_ASSERT(e) \
     ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))
-void erl_assert_error(const char* expr, const char *func, const char* file, int line);
+
+__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func,
+						 const char* file, int line);
 
 #ifdef DEBUG
 #  define ASSERT(e) ERTS_ASSERT(e)
@@ -192,23 +211,6 @@ void erl_assert_error(const char* expr, const char *func, const char* file, int
 #  define erts_align_attribute(SZ)
 #endif
 
-/* In VC++, noreturn is a declspec that has to be before the types,
- * but in GNUC it is an att ribute to be placed between return type 
- * and function name, hence __decl_noreturn <types> __noreturn <function name>
- */
-#if __GNUC__
-#  define __decl_noreturn 
-#  define __noreturn __attribute__((noreturn))
-#else
-#  if defined(__WIN32__) && defined(_MSC_VER)
-#    define __noreturn 
-#    define __decl_noreturn __declspec(noreturn)
-#  else
-#    define __noreturn
-#    define __decl_noreturn 
-#  endif
-#endif
-
 /*
 ** Data types:
 **
-- 
cgit v1.2.3


From 2b8016ebb9d487be590c01390928d1cc83c8225c Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Mon, 14 Oct 2013 23:39:26 +0200
Subject: Fix erts_check_off_heap2 assertion

The test needs to be false to fail, not true. This was noticed with the
following warning, where marking erl_assert_error as non-returning didn't
help:

beam/erl_gc.c:2772:2: warning: variable 'refc' is used uninitialized whenever switch default is taken [-Wsometimes-uninitialized]
        default:
        ^~~~~~~
beam/erl_gc.c:2775:26: note: uninitialized use occurs here
        ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
                                ^~~~
beam/erl_gc.c:2738:11: note: expanded from macro 'ERTS_CHK_OFFHEAP_ASSERT'
    if (!(EXP))                                         \
          ^
beam/erl_gc.c:2759:18: note: initialize the variable 'refc' to silence this warning
        erts_aint_t refc;
                        ^
                         = 0
---
 erts/emulator/beam/erl_gc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8ba94d89e9..e89725c190 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2770,7 +2770,7 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	    refc = erts_refc_read(&u.ext->node->refc, 1);
 	    break;
 	default:
-	    ASSERT(!!"erts_check_off_heap2: Invalid thing_word");
+	    ASSERT(!"erts_check_off_heap2: Invalid thing_word");
 	}
 	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
-- 
cgit v1.2.3


From f8c1256fc624ef79aa4f25574278c0cd2f9f1106 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Tue, 15 Oct 2013 13:06:20 +0200
Subject: Compile in_heapfrag() only in debug mode

---
 erts/emulator/beam/erl_message.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 325d77e911..6a9030fd99 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -46,10 +46,12 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message,
 
 
 
+#ifdef DEBUG
 static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp)
 {
     return ((unsigned)(ptr - bp->mem) < bp->used_size);
 }
+#endif
 
 
 void
-- 
cgit v1.2.3


From 51918f4322b3d30c89ba9b6ca2fbfe5ad6c6b51b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 Dec 2013 09:52:50 +0100
Subject: Update versions of OTP, erts, kernel, and stdlib

Update versions of OTP, erts, kernel, and stdlib to comply with
the new version scheme decided by the OTP technical board.
---
 erts/emulator/beam/erl_bif_info.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 8fa3aa29eb..2b40f9272d 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -65,8 +65,8 @@ static Export *gather_gc_info_res_trap;
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 /* Keep erts_system_version as a global variable for easy access from a core */
-static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
-				     " (erts-" ERLANG_VERSION ")"
+static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
+				     " [erts-" ERLANG_VERSION "]"
 #if !HEAP_ON_C_STACK && !HALFWORD_HEAP
 				     " [no-c-stack-objects]"
 #endif
-- 
cgit v1.2.3


From 289f77ff40833b8b95331efddeaaf29d1af21bae Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Dec 2013 19:43:19 +0100
Subject: erts: Fix valgrind warning for re_SUITE:error_handling

by making sure we at least can read the first four bytes
of a compiled regexp where the 'magic_number' is located.
---
 erts/emulator/beam/erl_bif_re.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 99c31738a5..448c6f6f6d 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1196,8 +1196,8 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 
 	ovsize = 3*(unsigned_val(tp[2])+1);
 	code_size = binary_size(tp[5]);
-	if ((code_tmp = (const pcre *) 
-	     erts_get_aligned_binary_bytes(tp[5], &temp_alloc)) == NULL) {
+	code_tmp = (const pcre *) erts_get_aligned_binary_bytes(tp[5], &temp_alloc);
+	if (code_tmp == NULL || code_size < 4) {
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    BIF_ERROR(p, BADARG);
 	}
-- 
cgit v1.2.3


From d7b9addd8637f67ba670312033acf1c0695f5af6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Dec 2013 16:17:30 +0100
Subject: erts: Refactor remove erts_sys_dll_open2

---
 erts/emulator/beam/erl_bif_ddll.c | 2 +-
 erts/emulator/beam/erl_nif.c      | 4 ++--
 erts/emulator/beam/io.c           | 2 +-
 erts/emulator/beam/sys.h          | 3 +--
 4 files changed, 5 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 1c3e955f47..caee07e491 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1524,7 +1524,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 
     assert_drv_list_rwlocked();
 
-    if ((res =  erts_sys_ddll_open(path, &(dh->handle))) != ERL_DE_NO_ERROR) {
+    if ((res =  erts_sys_ddll_open(path, &(dh->handle), NULL)) != ERL_DE_NO_ERROR) {
 	return res;
     }
     
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e87959f0ab..de626ca4e1 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1407,7 +1407,7 @@ void* enif_dlopen(const char* lib,
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
     void* handle;
     void* init_func;
-    if (erts_sys_ddll_open2(lib, &handle, &errdesc) == ERL_DE_NO_ERROR) {
+    if (erts_sys_ddll_open(lib, &handle, &errdesc) == ERL_DE_NO_ERROR) {
 	if (erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) == ERL_DE_NO_ERROR) {
 	    erts_sys_ddll_call_nif_init(init_func);
 	}
@@ -1626,7 +1626,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 			     "module '%T' not allowed", mod_atom);
     }    
     else if (init_func == NULL &&
-	     (err=erts_sys_ddll_open2(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
+	     (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
 	const char slogan[] = "Failed to load NIF library";
 	if (strstr(errdesc.str, lib_name) != NULL) {
 	    ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index d4623c0450..49af86b36a 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7061,7 +7061,7 @@ void *driver_dl_open(char * path)
     int res;
     int *last_error_p = erts_smp_tsd_get(driver_list_last_error_key);
     int locked = maybe_lock_driver_list();
-    if ((res = erts_sys_ddll_open(path, &ptr)) == 0) {
+    if ((res = erts_sys_ddll_open(path, &ptr, NULL)) == 0) {
 	maybe_unlock_driver_list(locked);
 	return ptr;
     } else {
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 31252ed78f..4a2a87a89a 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -661,8 +661,7 @@ typedef struct {
 #define ERTS_SYS_DDLL_ERROR_INIT {NULL}
 extern void erts_sys_ddll_free_error(ErtsSysDdllError*);
 extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */
-extern int erts_sys_ddll_open2(const char *path, void **handle, ErtsSysDdllError*);
-#define erts_sys_ddll_open(P,H) erts_sys_ddll_open2(P,H,NULL)
+extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*);
 extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*);
 extern int erts_sys_ddll_load_driver_init(void *handle, void **function);
 extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*);
-- 
cgit v1.2.3


From 3c5baf40ff8967d0b3190c5fabe15fb1487f6c52 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 16 Dec 2013 11:55:49 +0100
Subject: erts: Add 'extra' argument to erts_convert_filename_to_encoding

---
 erts/emulator/beam/erl_bif_port.c |  2 +-
 erts/emulator/beam/erl_nif.c      |  3 ++-
 erts/emulator/beam/erl_unicode.c  | 17 ++++++++++-------
 erts/emulator/beam/global.h       |  3 ++-
 4 files changed, 15 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 864349491a..f298422267 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -808,7 +808,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 	    if (encoding == ERL_FILENAME_WIN_WCHAR) {
 		encoding = ERL_FILENAME_UTF8;
 	    }
-	    if ((name_buf = erts_convert_filename_to_encoding(name, NULL, 0, ERTS_ALC_T_TMP,0,1, encoding, NULL))
+	    if ((name_buf = erts_convert_filename_to_encoding(name, NULL, 0, ERTS_ALC_T_TMP,0,1, encoding, NULL, 0))
 		== NULL) {
 		goto badarg;
 	    }
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index de626ca4e1..dc285b3cf7 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1587,7 +1587,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
         encoding = ERL_FILENAME_UTF8;
     }
     lib_name = erts_convert_filename_to_encoding(BIF_ARG_1, NULL, 0,
-                                                 ERTS_ALC_T_TMP, 1, 0, encoding, NULL);
+                                                 ERTS_ALC_T_TMP, 1, 0, encoding,
+						 NULL, 0);
     if (!lib_name) {
 	BIF_ERROR(BIF_P, BADARG);
     }
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 7e3c6681d9..3a968594f3 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1990,12 +1990,14 @@ char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_
 {
     int encoding = erts_get_native_filename_encoding();
     return erts_convert_filename_to_encoding(name, statbuf, statbuf_size, alloc_type,
-					     allow_empty, allow_atom, encoding, used);
+					     allow_empty, allow_atom, encoding,
+					     used, 0);
 }
 
 char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbuf_size,
 					ErtsAlcType_t alloc_type, int allow_empty,
-					int allow_atom, int encoding, Sint *used)
+					int allow_atom, int encoding, Sint *used,
+					Uint extra)
 {
     char* name_buf = NULL;
 
@@ -2008,13 +2010,14 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	}
 	if (encoding == ERL_FILENAME_WIN_WCHAR) {
 	    need += 2;
+	    extra *= 2;
 	} else {
 	    ++need;
 	}
 	if (used) 
 	    *used = (Sint) need;
-	if (need > statbuf_size) {
-	    name_buf = (char *) erts_alloc(alloc_type, need);
+	if (need+extra > statbuf_size) {
+	    name_buf = (char *) erts_alloc(alloc_type, need+extra);
 	} else {
 	    name_buf = statbuf;
 	}
@@ -2035,8 +2038,8 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	    /*Add 0 termination only*/
 	    if (used) 
 		*used = (Sint) size+1;
-	    if (size+1 > statbuf_size) {
-		name_buf = (char *) erts_alloc(alloc_type, size+1);
+	    if (size+1+extra > statbuf_size) {
+		name_buf = (char *) erts_alloc(alloc_type, size+1+extra);
 	    } else {
 		name_buf = statbuf;
 	    }
@@ -2045,7 +2048,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	} else {
             name_buf = erts_convert_filename_to_wchar(bytes, size,
                                                       statbuf, statbuf_size,
-                                                      alloc_type, used, 0);
+                                                      alloc_type, used, extra);
         }
 	erts_free_aligned_binary_bytes(temp_alloc);
     } else {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 94bc1b172a..6e5d352e5b 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -921,7 +921,8 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf,
 					ErtsAlcType_t alloc_type,
 					int allow_empty, int allow_atom,
 					int encoding,
-					Sint *used /* out */);
+					Sint *used /* out */,
+					Uint extra);
 char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
                                      char *statbuf, size_t statbuf_size,
                                      ErtsAlcType_t alloc_type, Sint* used,
-- 
cgit v1.2.3


From 21408235bd6915001cbc5051a1b7c38a3fef9d05 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 16 Dec 2013 19:39:17 +0100
Subject: erts: Support loading of drivers with unicode paths

---
 erts/emulator/beam/erl_bif_ddll.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index caee07e491..1728b200f7 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -182,7 +182,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
     Eterm name_term = BIF_ARG_2;
     Eterm options = BIF_ARG_3;
     char *path = NULL;
-    ErlDrvSizeT path_len;
+    Sint path_len;
     char *name = NULL;
     DE_Handle *dh;
     erts_driver_t *drv;
@@ -198,6 +198,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
     int kill_ports = 0;
     int do_build_load_error = 0;
     int build_this_load_error = 0;
+    int encoding;
 
     for(l = options; is_list(l); l =  CDR(list_val(l))) {
 	Eterm opt = CAR(list_val(l));
@@ -257,18 +258,23 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
 	goto error;
     }
 
-    if (erts_iolist_size(path_term, &path_len)) {
-	goto error;
+    encoding = erts_get_native_filename_encoding();
+    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+        /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */
+        /* since lib_name is used in error messages */
+        encoding = ERL_FILENAME_UTF8;
     }
-    path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1);
-    if (erts_iolist_to_buf(path_term, path, path_len) != 0) {
+    path = erts_convert_filename_to_encoding(path_term, NULL, 0,
+					     ERTS_ALC_T_DDLL_TMP_BUF, 1, 0,
+					     encoding, &path_len,
+					     sys_strlen(name) + 2); /* might need path separator */
+    if (!path) {
 	goto error;
     }
-    while (path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/')) {
-	--path_len;
-    }
+    ASSERT(path_len > 0 && path[path_len-1] == 0);
+    while (--path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/'))
+	;
     path[path_len++] = '/';
-    /*path[path_len] = '\0';*/
     sys_strcpy(path+path_len,name);
 
 #if DDLL_SMP
-- 
cgit v1.2.3


From 71e6fe5bcbd7b2b98dfa159db34ee1fe14823a56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 7 Jan 2014 18:40:43 +0100
Subject: erts: Fix bs_get_integer instruction

The instruction bs_get_integer could unnecessarily trigger a garbage collection
in failure cases which is unwanted or outright dangerous.

Ex:

    <<X:Sz,_/bits>> = <<"some binary">>

Previously, if Sz induced X to a bignum it would reserved memory size this on
the heap via a garbage collection before checking if the size could actually match.

It will now check the binary size before triggering a collection.
---
 erts/emulator/beam/beam_emu.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 78ab6fa30f..592cfe273f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4326,7 +4326,19 @@ void process_main(void)
      flags = Arg(2);
      BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size);
      if (size >= SMALL_BITS) {
-	 Uint wordsneeded = 1+WSIZE(NBYTES((Uint) size));
+	 Uint wordsneeded;
+	 /* check bits size before potential gc.
+	  * We do not want a gc and then realize we don't need
+	  * the allocated space (i.e. if the op fails)
+	  *
+	  * remember to reacquire the matchbuffer after gc.
+	  */
+
+	 mb = ms_matchbuffer(tmp_arg1);
+	 if (mb->size - mb->offset < size) {
+	     ClauseFail();
+	 }
+	 wordsneeded = 1+WSIZE(NBYTES((Uint) size));
 	 TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
      }
      mb = ms_matchbuffer(tmp_arg1);
-- 
cgit v1.2.3


From 99fbb7cffd40562907487278ae5f88b2e76d4923 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 8 Jan 2014 19:19:33 +0100
Subject: erts: Remove overestimation of heap space in binary_to_term

for 32-bit integers (INTEGER_EXT) on 64-bit architectures.
---
 erts/emulator/beam/external.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 2cb44a5b64..bccbedc7bd 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -4074,7 +4074,9 @@ init_done:
 	switch (tag) {
 	case INTEGER_EXT:
 	    SKIP(4);
+#if !defined(ARCH_64) || HALFWORD_HEAP
 	    heap_size += BIG_UINT_HEAP_SIZE;
+#endif
 	    break;
 	case SMALL_INTEGER_EXT:
 	    SKIP(1);
-- 
cgit v1.2.3


From 47979206defa9429458e419b691138ab1b519833 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 9 Jan 2014 12:50:28 +0100
Subject: Fix issues with new versioning

---
 erts/emulator/beam/erl_bif_info.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 414ae2f046..e0b654cb22 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -64,8 +64,10 @@ static Export *gather_gc_info_res_trap;
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
+static char otp_correction_package[] = ERLANG_OTP_CORRECTION_PACKAGE;
 /* Keep erts_system_version as a global variable for easy access from a core */
 static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
+				     "%s"
 				     " [erts-" ERLANG_VERSION "]"
 #if !HEAP_ON_C_STACK && !HALFWORD_HEAP
 				     " [no-c-stack-objects]"
@@ -304,11 +306,28 @@ make_link_list(Process *p, ErtsLink *root, Eterm tail)
 int
 erts_print_system_version(int to, void *arg, Process *c_p)
 {
+    int i, rc = -1;
+    char *rc_str = "";
+    char rc_buf[100];
+    char *ocp = otp_correction_package;
 #ifdef ERTS_SMP
     Uint total, online, active;
     (void) erts_schedulers_state(&total, &online, &active, 0);
 #endif
-    return erts_print(to, arg, erts_system_version
+    for (i = 0; i < sizeof(otp_correction_package)-4; i++) {
+	if (ocp[i] == '-' && ocp[i+1] == 'r' && ocp[i+2] == 'c')
+	    rc = atoi(&ocp[i+3]);
+    }
+    if (rc >= 0) {
+	if (rc == 0)
+	    rc_str = " [DEVELOPMENT]";
+	else {
+	    erts_snprintf(rc_buf, sizeof(rc_buf), " [RELEASE CANDIDATE %d]", rc);
+	    rc_str = rc_buf;
+	}
+    }
+    return erts_print(to, arg, erts_system_version,
+		      rc_str
 #ifdef ERTS_SMP
 		      , total, online
 #endif
@@ -2417,6 +2436,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	    DECL_AM(unknown);
 	    BIF_RET(AM_unknown);
 	}
+    } else if (ERTS_IS_ATOM_STR("otp_correction_package", BIF_ARG_1)) {
+	int n = sizeof(ERLANG_OTP_CORRECTION_PACKAGE)-1;
+	hp = HAlloc(BIF_P, 2*n);
+	BIF_RET(buf_to_intlist(&hp, ERLANG_OTP_CORRECTION_PACKAGE, n, NIL));
     } else if (ERTS_IS_ATOM_STR("otp_release", BIF_ARG_1)) {
 	int n = sizeof(ERLANG_OTP_RELEASE)-1;
 	hp = HAlloc(BIF_P, 2*n);
-- 
cgit v1.2.3


From 79355961135bb107a01b0a7e4668818f58327d99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 9 Jan 2014 18:46:36 +0100
Subject: erts: Increase gc tenure rate

The garbage collector tries to maintain the previous heap block size
during a minor gc, i.e. 'need' is not utilized in determining the size of
the new heap, instead it relies on tenure and garbage to be sufficiently large.

In instances during intense growing with exlusively live data on the heap
coupled with delayed tenure, fullsweeps would be triggered directly after
a minor gc to make room for 'need' since the new heap would be full.

To remedy this, the tenure of terms on the minor heap will always happen
(if it is below the high watermark) instead of every other minor gc.
---
 erts/emulator/beam/erl_gc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c5585d39e8..fd86c658d6 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1157,7 +1157,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 	old_htop = sweep_one_area(OLD_HTOP(p), old_htop, heap, heap_size);
     }
     OLD_HTOP(p) = old_htop;
-    HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop;
+    HIGH_WATER(p) = n_htop;
 
     if (MSO(p).first) {
 	sweep_off_heap(p, 0);
-- 
cgit v1.2.3


From 9bb39214db2de93142976de301427969aa2f9206 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Fri, 10 Jan 2014 15:23:06 +0100
Subject: erts: Replace tab with space for proper alignment

---
 erts/emulator/beam/erl_init.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 8c4fffa75b..1af80dd04b 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -553,8 +553,8 @@ void erts_usage(void)
     erts_fprintf(stderr, "            numbers is %d\n",
 		 ERTS_MAX_NO_OF_SCHEDULERS);
     erts_fprintf(stderr, "-SP p1:p2   specify schedulers (p1) and schedulers online (p2)\n");
-    erts_fprintf(stderr, "	      as percentages of logical processors configured and logical\n");
-    erts_fprintf(stderr, "	      processors available, respectively\n");
+    erts_fprintf(stderr, "            as percentages of logical processors configured and logical\n");
+    erts_fprintf(stderr, "            processors available, respectively\n");
     erts_fprintf(stderr, "-t size     set the maximum number of atoms the "
 			 "emulator can handle\n");
     erts_fprintf(stderr, "            valid range is [%d-%d]\n",
-- 
cgit v1.2.3


From b49efb826e191a0ef18da7ebdb3dd9e56fe654da Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 10 Jan 2014 15:41:48 +0100
Subject: erts: Simplify term_to_binary by removing saved ESTACK from root set

We disabled GC (in 522a29666088d5) during trapping and don't need to
include the saved ESTACK as part of root set.
---
 erts/emulator/beam/erl_alloc.types |   4 +-
 erts/emulator/beam/external.c      | 219 ++++++++++++++++++-------------------
 2 files changed, 106 insertions(+), 117 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 32308fae9b..b4e52770e3 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2003-2013. All Rights Reserved.
+# Copyright Ericsson AB 2003-2014. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -150,7 +150,7 @@ type	LINK_LH		STANDARD	PROCESSES	link_lh
 type	SUSPEND_MON	STANDARD	PROCESSES	suspend_monitor
 type	PEND_SUSPEND	SHORT_LIVED	PROCESSES	pending_suspend
 type	PROC_LIST	SHORT_LIVED	PROCESSES	proc_list
-type	EXTRA_ROOT	SHORT_LIVED	PROCESSES	extra_root
+type	SAVED_ESTACK	SHORT_LIVED	PROCESSES	saved_estack
 type	FUN_ENTRY	LONG_LIVED	CODE		fun_entry
 type	ATOM_TXT	LONG_LIVED	ATOM		atom_text
 type 	BEAM_REGISTER	EHEAP		PROCESSES	beam_register
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 2cb44a5b64..50831e848e 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -87,7 +87,8 @@
 static Export term_to_binary_trap_export;
 
 static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap);
-static int enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
+struct TTBEncodeContext_;
+static int enc_term_int(struct TTBEncodeContext_*,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
 			struct erl_off_heap_header** off_heap, Sint *reds, byte **res);
 static Uint is_external_string(Eterm obj, int* p_is_string);
 static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
@@ -103,7 +104,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 				     Binary *context_b);
 
 static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
-static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, 
+struct TTBSizeContext_;
+static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acmp, Eterm obj,
 				  unsigned dflags, Sint *reds, Uint *res);
 
 static Export binary_to_term_trap_export;
@@ -1086,7 +1088,6 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
     int level = 0;
     Uint flags = TERM_TO_BINARY_DFLAGS;
     Eterm res;
-    Binary *bin = NULL;
 
     while (is_list(Flags)) {
 	Eterm arg = CAR(list_val(Flags));
@@ -1123,7 +1124,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 	goto error;
     }
 
-    res = erts_term_to_binary_int(p, Term, level, flags, bin);
+    res = erts_term_to_binary_int(p, Term, level, flags, NULL);
     if (is_tuple(res)) {
 	erts_set_gc_state(p, 0);
 	BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res);
@@ -1726,14 +1727,22 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
 
 
 typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState;
-typedef struct {
+typedef struct TTBSizeContext_ {
     Uint flags;
     int level;
+    Uint result;
+    Eterm obj;
+    UWord* stack;
+    Uint   stack_sz;
 } TTBSizeContext;
 
-typedef struct {
+typedef struct TTBEncodeContext_ {
     Uint flags;
     int level;
+    byte* ep;
+    Eterm obj;
+    UWord* stack;
+    Uint   stack_sz;
     Binary *result_bin;
 } TTBEncodeContext;
 
@@ -1763,8 +1772,16 @@ static void ttb_context_destructor(Binary *context_bin)
 	context->alive = 0;
 	switch (context->state) {
 	case TTBSize:
+	    if (context->s.sc.stack) {
+		erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.sc.stack);
+		context->s.sc.stack = NULL;
+	    }
 	    break;
 	case TTBEncode:
+	    if (context->s.ec.stack) {
+		erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.ec.stack);
+		context->s.ec.stack = NULL;
+	    }
 	    if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */
 		ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0);
 		erts_bin_free(context->s.ec.result_bin);
@@ -1829,6 +1846,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 	/* Setup enough to get started */
 	context->state = TTBSize;
 	context->alive = 1;
+	context->s.sc.stack = NULL;
 	context->s.sc.flags = flags;
 	context->s.sc.level = level;
     } else {
@@ -1844,7 +1862,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		int level;
 		Uint flags;
 		/* Try for fast path */
-		if (encode_size_struct_int(p, NULL, Term, context->s.sc.flags, &reds, &size) < 0) {
+		if (encode_size_struct_int(&context->s.sc, NULL, Term,
+					   context->s.sc.flags, &reds, &size) < 0) {
 		    EXPORT_CONTEXT();
 		    /* Same state */
 		    RETURN_STATE();
@@ -1870,6 +1889,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		context->state = TTBEncode;
 		context->s.ec.flags = flags;
 		context->s.ec.level = level;
+		context->s.ec.stack = NULL;
 		context->s.ec.result_bin = result_bin;
 		break;
 	    }
@@ -1881,7 +1901,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		Binary *result_bin;
 
 		flags = context->s.ec.flags;
-		if (enc_term_int(p,NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) {
+		if (enc_term_int(&context->s.ec, NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) {
 		    EXPORT_CONTEXT();
 		    RETURN_STATE();
 		}
@@ -2289,27 +2309,6 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 #define ENC_PATCH_FUN_SIZE ((Eterm) 2)
 #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3)
 
-/* Free extra rootset (used when trapping) */
-static void cleanup_ttb_extra_root(ErlExtraRootSet *rs)
-{
-    if (rs->objv != NULL) {
-	erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv);
-    }
-    erts_free(ERTS_ALC_T_EXTRA_ROOT, rs);
-}
-
-/* Same as above, but we have an extra "stack" beyond GC reach, i.e. an array of two extra roots */
-static void cleanup_ttb_extra_root_2(ErlExtraRootSet *rs)
-{
-    if (rs->objv != NULL) {
-	erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv);
-    }
-    if (rs[1].objv != NULL) {
-	erts_free(ERTS_ALC_T_EXTRA_ROOT, rs[1].objv);
-    }
-	
-    erts_free(ERTS_ALC_T_EXTRA_ROOT, rs);
-}
 
 static byte*
 enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
@@ -2321,39 +2320,43 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
 }
 
 static int
-enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
+enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
 	     struct erl_off_heap_header** off_heap, Sint *reds, byte **res)
 {
-    DECLARE_ESTACK(s);
-    DECLARE_WSTACK(com);
+    DECLARE_WSTACK(s);
     Uint n;
     Uint i;
     Uint j;
     Uint* ptr;
     Eterm val;
     FloatDef f;
-    int count_reds = (p != NULL && reds != NULL);
     Sint r = 0;
+#if HALFWORD_HEAP
+    UWord wobj;
+#endif
 
-    if (count_reds) {
-	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT);
-	WSTACK_CHANGE_ALLOCATOR(com, ERTS_ALC_T_EXTRA_ROOT);
+
+    if (ctx) {
+	WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
 	r = *reds;
-    }
 
-    if (p && p->extra_root) { /* restore saved stacks and byte pointer */
-	ESTACK_RESTORE(s,p->extra_root[0].objv, p->extra_root[0].sz);
-	obj = ESTACK_POP(s);
-	WSTACK_RESTORE(com, p->extra_root[1].objv, p->extra_root[1].sz);
-	ep = (byte *) WSTACK_POP(com);
+	if (ctx->stack) { /* restore saved stacks and byte pointer */
+	    WSTACK_RESTORE(s, ctx->stack, ctx->stack_sz);
+	    ep = ctx->ep;
+	    obj = ctx->obj;
+	}
     }
 
     goto L_jump_start;
 
  outer_loop:
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-	switch (val = WSTACK_POP(com)) {
+    while (!WSTACK_ISEMPTY(s)) {
+#if HALFWORD_HEAP
+	obj = (Eterm) (wobj = WSTACK_POP(s));
+#else
+	obj = WSTACK_POP(s);
+#endif
+	switch (val = WSTACK_POP(s)) {
 	case ENC_TERM:
 	    break;
 	case ENC_ONE_CONS:
@@ -2364,55 +2367,52 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla
 
 		obj = CAR(cons);
 		tl = CDR(cons);
-		WSTACK_PUSH(com, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
-		ESTACK_PUSH(s, tl);
+		WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
+		WSTACK_PUSH(s, tl);
 	    }
 	    break;
 	case ENC_PATCH_FUN_SIZE:
-	    /* obj will be discarded, it was NIL */
 	    {
-		byte* size_p = (byte *) WSTACK_POP(com);
+#if HALFWORD_HEAP
+		byte* size_p = (byte *) wobj;
+#else
+		byte* size_p = (byte *) obj;
+#endif
 		put_int32(ep - size_p, size_p);
 	    }
 	    goto outer_loop;
 	case ENC_LAST_ARRAY_ELEMENT:
 	    /* obj is the tuple */
 	    {
-		Eterm* ptr = tuple_val(obj);
-		i = arityval(*ptr);
-		obj = ptr[i];
+#if HALFWORD_HEAP
+		Eterm* ptr = (Eterm *) wobj;
+#else
+		Eterm* ptr = (Eterm *) obj;
+#endif
+		obj = *ptr;
 	    }
 	    break;
 	default:		/* ENC_LAST_ARRAY_ELEMENT+1 and upwards */
 	    {
-		Eterm* ptr = tuple_val(obj);
-		i = arityval(*ptr);
-		ESTACK_PUSH(s, obj); /* put back tuple and next element index */
-		WSTACK_PUSH(com, val-1);
-		obj = ptr[i - (val - ENC_LAST_ARRAY_ELEMENT)]; /* the index is counting down */
+#if HALFWORD_HEAP
+		Eterm* ptr = (Eterm *) wobj;
+#else
+		Eterm* ptr = (Eterm *) obj;
+#endif
+		WSTACK_PUSH(s, val-1);
+		obj = *ptr++;
+		WSTACK_PUSH(s, (UWord)ptr);
 	    }
 	    break;
 	}
 
     L_jump_start:
 
-	if (count_reds && --r == 0) {
+	if (ctx && --r == 0) {
 	    *reds = r;
-	    ESTACK_PUSH(s,obj); /* push back current object, to be popped on restore */
-	    WSTACK_PUSH(com,((UWord) ep));
-	    if (p->extra_root == NULL) {
-		/* NB. Allocate an array of two "extra-roots", of which only the first element
-		   is seen and handled by the GC. Index 1 holds the Wstack. */
-		p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet)*2);
-		p->extra_root->objv = NULL;
-		p->extra_root->sz = 0;
-		p->extra_root->cleanup = cleanup_ttb_extra_root_2;
-		p->extra_root[1].objv = NULL;
-		p->extra_root[1].sz = 0;
-		p->extra_root[1].cleanup = NULL; /* Never used */
-	    }
-	    ESTACK_SAVE(s, p->extra_root[0].objv, p->extra_root[0].sz);
-	    WSTACK_SAVE(com, p->extra_root[1].objv, (p->extra_root[1].sz));
+	    ctx->obj = obj;
+	    ctx->ep = ep;
+	    WSTACK_SAVE(s, ctx->stack, ctx->stack_sz);
 	    return -1;
 	}
 	switch(tag_val_def(obj)) {
@@ -2558,8 +2558,8 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla
 		ep += 4;
 	    }
 	    if (i > 0) {
-		WSTACK_PUSH(com, ENC_LAST_ARRAY_ELEMENT+i-1);
-		ESTACK_PUSH(s, obj);
+		WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1);
+		WSTACK_PUSH(s, (UWord)ptr);
 	    }
 	    break;
 
@@ -2703,9 +2703,8 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla
 		    int ei;
 
 		    *ep++ = NEW_FUN_EXT;
-		    WSTACK_PUSH(com, (UWord) ep); /* Position for patching in size */
-		    WSTACK_PUSH(com, ENC_PATCH_FUN_SIZE);
-		    ESTACK_PUSH(s,NIL); /* Will be thrown away */
+		    WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
+		    WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */
 		    ep += 4;
 		    *ep = funp->arity;
 		    ep += 1;
@@ -2722,8 +2721,8 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla
 
 		fun_env:
 		    for (ei = funp->num_free-1; ei > 0; ei--) {
-			WSTACK_PUSH(com, ENC_TERM);
-			ESTACK_PUSH(s, (UWord) funp->env[ei]);
+			WSTACK_PUSH(s, ENC_TERM);
+			WSTACK_PUSH(s, (UWord) funp->env[ei]);
 		    }
 		    if (funp->num_free != 0) {
 			obj = funp->env[0];
@@ -2766,13 +2765,12 @@ enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dfla
 	    break;
 	}
     }
-    DESTROY_ESTACK(s);
-    DESTROY_WSTACK(com);
-    if (p && p->extra_root) {
-	cleanup_ttb_extra_root_2(p->extra_root);
-	p->extra_root = NULL;
-    }
-    if (count_reds) {
+    DESTROY_WSTACK(s);
+    if (ctx) {
+	if (ctx->stack) {
+	    erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack);
+	    ctx->stack = NULL;
+	}
 	*reds = r;
     }
     *res = ep;
@@ -3742,26 +3740,24 @@ static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dfla
 }
 
 static int
-encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, 
+encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		       unsigned dflags, Sint *reds, Uint *res)
 {
     DECLARE_ESTACK(s);
     Uint m, i, arity;
     Uint result = 0;
-    int count_reds = (p != NULL && reds != 0);
     Sint r = 0;
 
-    if (count_reds) {
-	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT);
+    if (ctx) {
+	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
 	r = *reds;
-    }
-
-    if (p && p->extra_root) { /* restore saved stack */
-	ESTACK_RESTORE(s,p->extra_root->objv, p->extra_root->sz + 1);
-	result = ESTACK_POP(s); /*Untagged, beyond  p->extra_root->sz */
-	obj = ESTACK_POP(s);
 
-    } 
+	if (ctx->stack) { /* restore saved stack */
+	    ESTACK_RESTORE(s, ctx->stack, ctx->stack_sz);
+	    result = ctx->result;
+	    obj = ctx->obj;
+	}
+    }
 
     goto L_jump_start;
 
@@ -3787,18 +3783,11 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj,
 	}
     
     L_jump_start:
-	if (count_reds && --r == 0) {
+	if (ctx && --r == 0) {
 	    *reds = r;
-	    ESTACK_PUSH(s,obj); /* push back current object */
-	    ESTACK_PUSH(s,result); /* Untagged, will be out of GC reach */
-	    if (p->extra_root == NULL) {
-		p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet));
-		p->extra_root->objv = NULL;
-		p->extra_root->sz = 0;
-		p->extra_root->cleanup = cleanup_ttb_extra_root;
-	    }
-	    ESTACK_SAVE(s, p->extra_root->objv, p->extra_root->sz);
-	    --p->extra_root->sz; /* Hide result from GC */
+	    ctx->obj = obj;
+	    ctx->result = result;
+	    ESTACK_SAVE(s, ctx->stack, ctx->stack_sz);
 	    return -1;
 	}
 	switch (tag_val_def(obj)) {
@@ -4001,11 +3990,11 @@ encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj,
     }
 
     DESTROY_ESTACK(s);
-    if (p && p->extra_root) {
-	cleanup_ttb_extra_root(p->extra_root);
-	p->extra_root = NULL;
-    }
-    if (count_reds) {
+    if (ctx) {
+	if (ctx->stack) {
+	    erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack);
+	    ctx->stack = NULL;
+	}
 	*reds = r;
     }
     *res = result;
-- 
cgit v1.2.3


From f7f08e93cc1e976cb2b37d450996cde32be8aae2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 10 Jan 2014 15:47:16 +0100
Subject: erts: Remove the extra_root feature from the process structure

as we don't use it and instead have the feature to disable GC
during trapping BIFs.
---
 erts/emulator/beam/erl_gc.c      | 18 +-----------------
 erts/emulator/beam/erl_process.c |  9 +--------
 erts/emulator/beam/erl_process.h | 11 +----------
 3 files changed, 3 insertions(+), 35 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c5585d39e8..8ff6f9a3b9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1975,17 +1975,6 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
         ++n;
     }
 
-    /*
-     * A trapping BIF can add to rootset by setting the extra_root
-     * in the process_structure.
-     */
-    if (p->extra_root != NULL) {
-	roots[n].v = p->extra_root->objv;
-	roots[n].sz = p->extra_root->sz;
-	++n;
-    }
-
-
     ASSERT((is_nil(p->seq_trace_token) ||
 	    is_tuple(follow_moved(p->seq_trace_token)) ||
 	    is_atom(p->seq_trace_token)));
@@ -2563,11 +2552,6 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 		    p->dictionary->used, 
 		    offs, area, area_size);
     }
-    if (p->extra_root != NULL) {
-	offset_heap_ptr(p->extra_root->objv, 
-			p->extra_root->sz, 
-			offs, area, area_size);
-    }
 
     offset_heap_ptr(&p->fvalue, 1, offs, area, area_size);
     offset_heap_ptr(&p->ftrace, 1, offs, area, area_size);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 21fd8dd50a..9983a26688 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -8755,7 +8755,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->htop = p->heap;
     p->heap_sz = sz;
     p->catches = 0;
-    p->extra_root = NULL;
 
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
@@ -10219,12 +10218,6 @@ erts_continue_exit_process(Process *p)
     if (pbt)
         erts_free(ERTS_ALC_T_BPD, (void *) pbt);
 
-    if (p->extra_root != NULL) {
-	(p->extra_root->cleanup)(p->extra_root); /* Should deallocate 
-						    whole structure */
-	p->extra_root = NULL;
-    }
-
     delete_process(p);
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 043621125c..e35d1c785c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -711,13 +711,6 @@ struct ErtsPendingSuspend_ {
 #endif
 
 
-typedef struct ErlExtraRootSet_ ErlExtraRootSet;
-struct ErlExtraRootSet_ {
-    Eterm *objv;
-    Uint sz;
-    void (*cleanup)(ErlExtraRootSet *);
-};
-
 /* Defines to ease the change of memory architecture */
 #  define HEAP_START(p)     (p)->heap
 #  define HEAP_TOP(p)       (p)->htop
@@ -811,8 +804,6 @@ struct process {
 
     ErlMessageQueue msg;	/* Message queue */
 
-    ErlExtraRootSet *extra_root;   /* Used by trapping BIF's */
-
     union {
 	ErtsBifTimer *bif_timers;	/* Bif timers aiming at this process */
 	void *terminate;
-- 
cgit v1.2.3


From 8ff33cff02d01b2b4f20769cbd77c5ef23b01631 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Wed, 15 Jan 2014 10:13:11 -0500
Subject: remove deprecated driver_async_cancel function

Some time ago the driver_async_cancel function was deprecated and slated
for removal in R17. This commit removes the function along with its
associated tests and documentation, sets the
ERL_DRV_EXTENDED_MAJOR_VERSION to 3 and ERL_DRV_EXTENDED_MINOR_VERSION to
0, and modifies the sys_info_base_drv and sys_info_prev_drv tests in the
driver test suite to check version 3.0 instead of 2.0.
---
 erts/emulator/beam/erl_async.c  | 22 +---------------------
 erts/emulator/beam/erl_driver.h | 10 ++--------
 2 files changed, 3 insertions(+), 29 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index e6d72f569b..f0cec1c53c 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -602,7 +602,7 @@ unsigned int driver_async_port_key(ErlDrvPort port)
 ** return values:
 **  0  completed 
 **  -1 error
-**  N  handle value (used with async_cancel)
+**  N  handle value
 **  arguments:
 **      ix             driver index 
 **      key            pointer to secedule queue (NULL means round robin)
@@ -687,23 +687,3 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 
     return id;
 }
-
-int driver_async_cancel(unsigned int id)
-{
-    /*
-     * Not supported anymore. Always fail (which is backward
-     * compatible).
-     *
-     * This functionality could be implemented again. However,
-     * it is (and always has been) completely useless since
-     * it doesn't give you any guarantees whatsoever. The user
-     * needs to (and always have had to) synchronize in his/her
-     * own code in order to get any guarantees.
-     */
-    return 0;
-}
-
-
-
-
-
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 5cffae92be..2bd3181bdc 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -132,8 +132,8 @@ typedef struct {
 #define DO_WRITE ERL_DRV_WRITE
 
 #define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
-#define ERL_DRV_EXTENDED_MAJOR_VERSION	2
-#define ERL_DRV_EXTENDED_MINOR_VERSION	2
+#define ERL_DRV_EXTENDED_MAJOR_VERSION	3
+#define ERL_DRV_EXTENDED_MINOR_VERSION	0
 
 /*
  * The emulator will refuse to load a driver with different major
@@ -657,12 +657,6 @@ EXTERN long driver_async(ErlDrvPort ix,
 			 void* async_data,
 			 void (*async_free)(void*));
 
-/*
- * driver_async_cancel() is deprecated. It is scheduled for removal
- * in OTP-R16. For more information see the erl_driver(3) documentation.
- */
-EXTERN int driver_async_cancel(unsigned int key) ERL_DRV_DEPRECATED_FUNC;
-
 /* Locks the driver in the machine "forever", there is
    no unlock function. Note that this is almost never useful, as an open
    port towards the driver locks it until the port is closed, why unexpected
-- 
cgit v1.2.3


From ee1886e457302347414787e74ce5017604d2e52a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Jan 2014 21:16:37 +0100
Subject: erts: Fix compiler warnings for NO_JUMP_TABLE

---
 erts/emulator/beam/beam_emu.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 78ab6fa30f..0609e86a39 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -48,7 +48,7 @@
 #  define OpCase(OpCode)    case op_##OpCode
 #  define CountCase(OpCode) case op_count_##OpCode
 #  define OpCode(OpCode)    ((Uint*)op_##OpCode)
-#  define Goto(Rel) {Go = (int)(Rel); goto emulator_loop;}
+#  define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;}
 #  define LabelAddr(Addr) &&##Addr
 #else
 #  define OpCase(OpCode)    lb_##OpCode
@@ -133,7 +133,7 @@ do {                                     \
 
 /* We don't check the range if an ordinary switch is used */
 #ifdef NO_JUMP_TABLE
-#define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10)))
+#define VALID_INSTR(IP) ((UWord)(IP) < (NUMBER_OF_OPCODES*2+10))
 #else
 #define VALID_INSTR(IP) \
    ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
-- 
cgit v1.2.3


From 03493394e8d20591020ce6c8152cb607cb21e967 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Jan 2014 22:31:36 +0100
Subject: erts: Fix benign ESTACK/WSTACK typo

---
 erts/emulator/beam/utils.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 297c4bf439..216952e2ac 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -2846,7 +2846,7 @@ pop_next:
     return 0;
 
 not_equal:
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return j;
 
 #undef CMP_NODES
-- 
cgit v1.2.3


From 66b16c23552418c17a29caf7cecf8e11f587b29d Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Wed, 15 Jan 2014 23:43:03 -0500
Subject: fix system_flag deprecation warnings

Passing cpu_topology or scheduler_bind_type to erlang:system_flag/2 results
in an error report warning that the argument is deprecated and is slated
for removal in erts-5.10/OTP-R16. Since we're already past that version and
no substitute approach has yet been decided for these features, keep the
deprecation warning but bump the removal version it mentions up to Erlang
18.
---
 erts/emulator/beam/bif.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 96666d98ed..61c1abedb5 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4488,7 +4488,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    BIF_P->group_leader,
 	    "A call to erlang:system_flag(cpu_topology, _) was made.\n"
 	    "The cpu_topology argument is deprecated and scheduled\n"
-	    "for removal in erts-5.10/OTP-R16. For more information\n"
+	    "for removal in Erlang/OTP 18. For more information\n"
 	    "see the erlang:system_flag/2 documentation.\n");
 	BIF_TRAP1(set_cpu_topology_trap, BIF_P, BIF_ARG_2);
     } else if (ERTS_IS_ATOM_STR("scheduler_bind_type", BIF_ARG_1)) {
@@ -4496,7 +4496,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    BIF_P->group_leader,
 	    "A call to erlang:system_flag(scheduler_bind_type, _) was\n"
 	    "made. The scheduler_bind_type argument is deprecated and\n"
-	    "scheduled for removal in erts-5.10/OTP-R16. For more\n"
+	    "scheduled for removal in Erlang/OTP 18. For more\n"
 	    "information see the erlang:system_flag/2 documentation.\n");
 	return erts_bind_schedulers(BIF_P, BIF_ARG_2);
     }
-- 
cgit v1.2.3


From 172ebf11dc455e22b87f742c06fa6344995d7db5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 16 Jan 2014 11:10:34 +0100
Subject: erts: Refactor ESTACK & WSTACK to use a struct easy to "export"

This is not a clean refactor. It changes the behaviour slightly
of E/WSTACK_RESTORE. The allocated stack from E/WSTACK_SAVE is used
as-is and not copied into default_stack. This will hopefully fix an
illusive memory leak that valgrind is reporting.
---
 erts/emulator/beam/external.c |  42 ++---
 erts/emulator/beam/global.h   | 346 +++++++++++++++++++++---------------------
 erts/emulator/beam/utils.c    |  44 +++---
 3 files changed, 211 insertions(+), 221 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 50831e848e..94acceaaf1 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1732,8 +1732,7 @@ typedef struct TTBSizeContext_ {
     int level;
     Uint result;
     Eterm obj;
-    UWord* stack;
-    Uint   stack_sz;
+    ErtsEStack estack;
 } TTBSizeContext;
 
 typedef struct TTBEncodeContext_ {
@@ -1741,8 +1740,7 @@ typedef struct TTBEncodeContext_ {
     int level;
     byte* ep;
     Eterm obj;
-    UWord* stack;
-    Uint   stack_sz;
+    ErtsWStack wstack;
     Binary *result_bin;
 } TTBEncodeContext;
 
@@ -1772,16 +1770,10 @@ static void ttb_context_destructor(Binary *context_bin)
 	context->alive = 0;
 	switch (context->state) {
 	case TTBSize:
-	    if (context->s.sc.stack) {
-		erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.sc.stack);
-		context->s.sc.stack = NULL;
-	    }
+	    DESTROY_SAVED_ESTACK(&context->s.sc.estack);
 	    break;
 	case TTBEncode:
-	    if (context->s.ec.stack) {
-		erts_free(ERTS_ALC_T_SAVED_ESTACK, context->s.ec.stack);
-		context->s.ec.stack = NULL;
-	    }
+	    DESTROY_SAVED_WSTACK(&context->s.ec.wstack);
 	    if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */
 		ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0);
 		erts_bin_free(context->s.ec.result_bin);
@@ -1846,7 +1838,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 	/* Setup enough to get started */
 	context->state = TTBSize;
 	context->alive = 1;
-	context->s.sc.stack = NULL;
+	context->s.sc.estack.start = NULL;
 	context->s.sc.flags = flags;
 	context->s.sc.level = level;
     } else {
@@ -1889,7 +1881,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		context->state = TTBEncode;
 		context->s.ec.flags = flags;
 		context->s.ec.level = level;
-		context->s.ec.stack = NULL;
+		context->s.ec.wstack.wstart = NULL;
 		context->s.ec.result_bin = result_bin;
 		break;
 	    }
@@ -2340,8 +2332,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
 	r = *reds;
 
-	if (ctx->stack) { /* restore saved stacks and byte pointer */
-	    WSTACK_RESTORE(s, ctx->stack, ctx->stack_sz);
+	if (ctx->wstack.wstart) { /* restore saved stacks and byte pointer */
+	    WSTACK_RESTORE(s, &ctx->wstack);
 	    ep = ctx->ep;
 	    obj = ctx->obj;
 	}
@@ -2412,7 +2404,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    *reds = r;
 	    ctx->obj = obj;
 	    ctx->ep = ep;
-	    WSTACK_SAVE(s, ctx->stack, ctx->stack_sz);
+	    WSTACK_SAVE(s, &ctx->wstack);
 	    return -1;
 	}
 	switch(tag_val_def(obj)) {
@@ -2767,10 +2759,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
     }
     DESTROY_WSTACK(s);
     if (ctx) {
-	if (ctx->stack) {
-	    erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack);
-	    ctx->stack = NULL;
-	}
+	ASSERT(ctx->wstack.wstart == NULL);
 	*reds = r;
     }
     *res = ep;
@@ -3752,8 +3741,8 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
 	r = *reds;
 
-	if (ctx->stack) { /* restore saved stack */
-	    ESTACK_RESTORE(s, ctx->stack, ctx->stack_sz);
+	if (ctx->estack.start) { /* restore saved stack */
+	    ESTACK_RESTORE(s, &ctx->estack);
 	    result = ctx->result;
 	    obj = ctx->obj;
 	}
@@ -3787,7 +3776,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    *reds = r;
 	    ctx->obj = obj;
 	    ctx->result = result;
-	    ESTACK_SAVE(s, ctx->stack, ctx->stack_sz);
+	    ESTACK_SAVE(s, &ctx->estack);
 	    return -1;
 	}
 	switch (tag_val_def(obj)) {
@@ -3991,10 +3980,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 
     DESTROY_ESTACK(s);
     if (ctx) {
-	if (ctx->stack) {
-	    erts_free(ERTS_ALC_T_SAVED_ESTACK, ctx->stack);
-	    ctx->stack = NULL;
-	}
+	ASSERT(ctx->estack.start == NULL);
 	*reds = r;
     }
     *res = result;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 94bc1b172a..c44f532366 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -370,231 +370,233 @@ extern int stackdump_on_exit;
  * DESTROY_ESTACK(Stack)
  */
 
+typedef struct {
+    UWord* start;
+    UWord* sp;
+    UWord* end;
+    ErtsAlcType_t alloc_type;
+}ErtsEStack;
 
-void erl_grow_stack(ErtsAlcType_t a_type, Eterm** start, Eterm** sp, Eterm** end);
-#define ESTK_CONCAT(a,b) a##b
-#define ESTK_SUBSCRIPT(s,i) *((Eterm *)((byte *)ESTK_CONCAT(s,_start) + (i)))
 #define DEF_ESTACK_SIZE (16)
 
-#define DECLARE_ESTACK(s)						\
-    Eterm ESTK_CONCAT(s,_default_stack)[DEF_ESTACK_SIZE];		\
-    Eterm* ESTK_CONCAT(s,_start) = ESTK_CONCAT(s,_default_stack);	\
-    Eterm* ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start);			\
-    Eterm* ESTK_CONCAT(s,_end) = ESTK_CONCAT(s,_start) + DEF_ESTACK_SIZE;\
-    ErtsAlcType_t ESTK_CONCAT(s,_alloc_type) = ERTS_ALC_T_ESTACK
+void erl_grow_estack(ErtsEStack*, Eterm* def_stack);
+#define ESTK_CONCAT(a,b) a##b
+#define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack)
+
+#define DECLARE_ESTACK(s)				\
+    UWord ESTK_DEF_STACK(s)[DEF_ESTACK_SIZE];		\
+    ErtsEStack s = {					\
+        ESTK_DEF_STACK(s),  /* start */ 		\
+        ESTK_DEF_STACK(s),  /* sp */			\
+        ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */	\
+        ERTS_ALC_T_ESTACK /* alloc_type */		\
+    }
 
 #define ESTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
-    if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) {	\
+    if (s.start != ESTK_DEF_STACK(s)) {					\
 	erl_exit(1, "Internal error - trying to change allocator "	\
 		 "type of active estack\n");				\
     }									\
-    ESTK_CONCAT(s,_alloc_type) = (t);					\
+    s.alloc_type = (t);							\
  } while (0)
 
+#define DESTROY_ESTACK(s)				\
+do {							\
+    if (s.start != ESTK_DEF_STACK(s)) {			\
+	erts_free(s.alloc_type, s.start); 		\
+    }							\
+} while(0)
+
+
 /*
- * Do not free the stack after this, it may have pointers into what 
- * was saved in 'v'. 'v' and 'vsize' are changed by this macro. If
- * 'v' points to anything, it should have been allocated by a previous 
- * call to this macro. Be careful to set a correct allocator prior to 
- * saving.
- * 'v' can be any lvalue pointer, it will point to an array of UWord
- * after calling this macro.
+ * Do not free the stack after this, it may have pointers into what
+ * was saved in 'dst'.
  */
-#define ESTACK_SAVE(s,v,vsize) /* v and vsize are "name parameters" */	\
-do {									\
-    Uint _esz = ESTACK_COUNT(s);					\
-    if (ESTK_CONCAT(s,_start) == ESTK_CONCAT(s,_default_stack)) {	\
-	if ((v) == NULL) {						\
-	    (v) = erts_alloc(ESTK_CONCAT(s,_alloc_type),		\
-			     DEF_ESTACK_SIZE * sizeof(Eterm));		\
-	}								\
-	memcpy((v),ESTK_CONCAT(s,_start),_esz*sizeof(Eterm));		\
-    } else {								\
-	(v) = (void *) ESTK_CONCAT(s,_start);				\
-    }									\
-    (vsize) = _esz;							\
+#define ESTACK_SAVE(s,dst)\
+do {\
+    if (s.start == ESTK_DEF_STACK(s)) {\
+	UWord _wsz = ESTACK_COUNT(s);\
+	(dst)->start = erts_alloc(s.alloc_type,\
+				  DEF_ESTACK_SIZE * sizeof(UWord));\
+	memcpy((dst)->start, s.start,_wsz*sizeof(UWord));\
+	(dst)->sp = (dst)->start + _wsz;\
+	(dst)->end = (dst)->start + DEF_ESTACK_SIZE;\
+	(dst)->alloc_type = s.alloc_type;\
+    } else\
+        *(dst) = s;\
  } while (0)
 
-/* 
- * Use on empty stack, only the allocator can be changed before this 
- * The vector parameter is reset to NULL if the vector is moved to stack,
- * otherwise it's kept for reuse, so a saved and restored vector might 
- * need freeing using the correct allocator parameter.
- * 'v' can be any lvalue pointer, it's cast to an (Eterm *).
+#define DESTROY_SAVED_ESTACK(estack)\
+do {\
+    if ((estack)->start) {\
+	erts_free((estack)->alloc_type, (estack)->start);\
+	(estack)->start = NULL;\
+    }\
+} while(0)
+
+/*
+ * Use on empty stack, only the allocator can be changed before this.
+ * The src stack is reset to NULL.
  */
-#define ESTACK_RESTORE(s, v, vsize) /*v is a "name parameter"*/ \
-do {								\
-    if ((vsize) > DEF_ESTACK_SIZE) {				\
-	Uint _ca = DEF_ESTACK_SIZE;				\
-	while (_ca < (vsize))					\
-	    _ca = _ca * 2;					\
-	ESTK_CONCAT(s,_start) = (Eterm *) (v);			\
-	ESTK_CONCAT(s,_end) = ((Eterm *)(v)) + _ca;		\
-	ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start) + (vsize);	\
-	(v) = NULL;                                             \
-    } else {							\
-	memcpy(ESTK_CONCAT(s,_start),(v),(vsize)*sizeof(Eterm));\
-	ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start) + (vsize);	\
-    }								\
- } while (0)
+#define ESTACK_RESTORE(s, src)			\
+do {						\
+    ASSERT(s.start == ESTK_DEF_STACK(s));	\
+    s = *(src);  /* struct copy */		\
+    (src)->start = NULL;			\
+    ASSERT(s.sp >= s.start);			\
+    ASSERT(s.sp <= s.end);			\
+} while (0)
 
-#define ESTACK_IS_STATIC(s) (ESTK_CONCAT(s,_start) == ESTK_CONCAT(s,_default_stack))
+#define ESTACK_IS_STATIC(s) (s.start == ESTK_DEF_STACK(s)))
 
-#define DESTROY_ESTACK(s)						\
-do {									\
-    if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) {	\
-	erts_free(ESTK_CONCAT(s,_alloc_type), ESTK_CONCAT(s,_start));		\
-    }									\
+#define ESTACK_PUSH(s, x)				\
+do {							\
+    if (s.sp == s.end) {				\
+	erl_grow_estack(&s, ESTK_DEF_STACK(s)); 	\
+    }							\
+    *s.sp++ = (x);					\
 } while(0)
 
-#define ESTACK_PUSH(s, x)						\
-do {									\
-    if (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_end)) {			\
-	erl_grow_stack(ESTK_CONCAT(s,_alloc_type),&ESTK_CONCAT(s,_start), \
-		       &ESTK_CONCAT(s,_sp), &ESTK_CONCAT(s,_end));	\
-    }									\
-    *ESTK_CONCAT(s,_sp)++ = (x);					\
+#define ESTACK_PUSH2(s, x, y)			\
+do {						\
+    if (s.sp > s.end - 2) {			\
+	erl_grow_estack(&s, ESTK_DEF_STACK(s)); \
+    }						\
+    *s.sp++ = (x);				\
+    *s.sp++ = (y);				\
 } while(0)
 
-#define ESTACK_PUSH2(s, x, y)						\
-do {									\
-    if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 2) {			\
-	erl_grow_stack(ESTK_CONCAT(s,_alloc_type),&ESTK_CONCAT(s,_start), \
-		&ESTK_CONCAT(s,_sp), &ESTK_CONCAT(s,_end));	\
-    }									\
-    *ESTK_CONCAT(s,_sp)++ = (x);					\
-    *ESTK_CONCAT(s,_sp)++ = (y);					\
+#define ESTACK_PUSH3(s, x, y, z)		\
+do {						\
+    if (s.sp > s.end - 3) {			\
+	erl_grow_estack(&s, ESTK_DEF_STACK(s)); \
+    }						\
+    *s.sp++ = (x);				\
+    *s.sp++ = (y);				\
+    *s.sp++ = (z);				\
 } while(0)
 
-#define ESTACK_PUSH3(s, x, y, z)					\
-do {									\
-    if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 3) {			\
-	erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp),	\
-		&ESTK_CONCAT(s,_end));					\
-    }									\
-    *ESTK_CONCAT(s,_sp)++ = (x);					\
-    *ESTK_CONCAT(s,_sp)++ = (y);					\
-    *ESTK_CONCAT(s,_sp)++ = (z);					\
-} while(0)
+#define ESTACK_COUNT(s) (s.sp - s.start)
+#define ESTACK_ISEMPTY(s) (s.sp == s.start)
+#define ESTACK_POP(s) (*(--s.sp))
 
-#define ESTACK_COUNT(s) (ESTK_CONCAT(s,_sp) - ESTK_CONCAT(s,_start))
 
-#define ESTACK_ISEMPTY(s) (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_start))
-#define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp)))
+/*
+ * WSTACK: same as ESTACK but with UWord instead of Eterm
+ */
 
+typedef struct {
+    UWord* wstart;
+    UWord* wsp;
+    UWord* wend;
+    ErtsAlcType_t alloc_type;
+}ErtsWStack;
 
-void erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end);
-#define WSTK_CONCAT(a,b) a##b
-#define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i)))
 #define DEF_WSTACK_SIZE (16)
 
-#define DECLARE_WSTACK(s)						\
-    UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE];		\
-    UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack);	\
-    UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start);			\
-    UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE; \
-    ErtsAlcType_t WSTK_CONCAT(s,_alloc_type) = ERTS_ALC_T_ESTACK
+void erl_grow_wstack(ErtsWStack*, Eterm* def_stack);
+#define WSTK_CONCAT(a,b) a##b
+#define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack)
+
+#define DECLARE_WSTACK(s)				\
+    UWord WSTK_DEF_STACK(s)[DEF_WSTACK_SIZE];		\
+    ErtsWStack s = {					\
+        WSTK_DEF_STACK(s),  /* wstart */ 		\
+        WSTK_DEF_STACK(s),  /* wsp */			\
+        WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */	\
+        ERTS_ALC_T_ESTACK /* alloc_type */		\
+    }
 
 #define WSTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
-    if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) {	\
+    if (s.wstart != WSTK_DEF_STACK(s)) {				\
 	erl_exit(1, "Internal error - trying to change allocator "	\
 		 "type of active wstack\n");				\
     }									\
-    WSTK_CONCAT(s,_alloc_type) = (t);					\
+    s.alloc_type = (t);							\
  } while (0)
 
-#define DESTROY_WSTACK(s)						\
-do {									\
-    if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) {	\
-	erts_free(WSTK_CONCAT(s,_alloc_type), WSTK_CONCAT(s,_start));		\
-    }									\
+#define DESTROY_WSTACK(s)				\
+do {							\
+    if (s.wstart != WSTK_DEF_STACK(s)) {		\
+	erts_free(s.alloc_type, s.wstart); 		\
+    }							\
 } while(0)
 
+
 /*
- * Do not free the stack after this, it may have pointers into what 
- * was saved in 'v'. 'v' and 'vsize' are changed by this macro. If
- * 'v' points to anything, it should have been allocated by a previous 
- * call to this macro. Be careful to set a correct allocator prior to 
- * saving.
- * 'v' can be any lvalue pointer, it will point to an array of UWord
- * after calling this macro.
+ * Do not free the stack after this, it may have pointers into what
+ * was saved in 'dst'.
  */
-#define WSTACK_SAVE(s,v,vsize) /* v and vsize are "name parameters" */	\
-do {									\
-    Uint _wsz = WSTACK_COUNT(s);					\
-    if (WSTK_CONCAT(s,_start) == WSTK_CONCAT(s,_default_stack)) {	\
-	if ((v) == NULL) {						\
-	    (v) = erts_alloc(WSTK_CONCAT(s,_alloc_type),		\
-			     DEF_WSTACK_SIZE * sizeof(UWord));		\
-	}								\
-	memcpy((v),WSTK_CONCAT(s,_start),_wsz*sizeof(UWord));		\
-    } else {								\
-	(v) = (void *) WSTK_CONCAT(s,_start);				\
-    }									\
-    (vsize) = _wsz;							\
+#define WSTACK_SAVE(s,dst)\
+do {\
+    if (s.wstart == WSTK_DEF_STACK(s)) {\
+	UWord _wsz = WSTACK_COUNT(s);\
+	(dst)->wstart = erts_alloc(s.alloc_type,\
+				  DEF_WSTACK_SIZE * sizeof(UWord));\
+	memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\
+	(dst)->wsp = (dst)->wstart + _wsz;\
+	(dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\
+	(dst)->alloc_type = s.alloc_type;\
+    } else\
+        *(dst) = s;\
  } while (0)
 
-/* 
- * Use on empty stack, only the allocator can be changed before this 
- * The vector parameter is reset to NULL if the vector is moved to stack,
- * otherwise it's kept for reuse, so a saved and restored vector might 
- * need freeing using the correct allocator parameter.
- * 'v' can be any lvalue pointer, it's cast to an (UWord *).
+#define DESTROY_SAVED_WSTACK(wstack)\
+do {\
+    if ((wstack)->wstart) {\
+	erts_free((wstack)->alloc_type, (wstack)->wstart);\
+	(wstack)->wstart = NULL;\
+    }\
+} while(0)
+
+/*
+ * Use on empty stack, only the allocator can be changed before this.
+ * The src stack is reset to NULL.
  */
-#define WSTACK_RESTORE(s, v, vsize) /*v is a "name parameter"*/ \
-do {								\
-    if ((vsize) > DEF_WSTACK_SIZE) {				\
-	Uint _ca = DEF_WSTACK_SIZE;				\
-	while (_ca < (vsize))					\
-	    _ca = _ca * 2;					\
-	WSTK_CONCAT(s,_start) = (UWord *) (v);			\
-	WSTK_CONCAT(s,_end) = ((UWord *)(v)) + _ca;		\
-	WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start) + (vsize);	\
-	(v) = NULL;                                             \
-    } else {							\
-	memcpy(WSTK_CONCAT(s,_start),(v),(vsize)*sizeof(UWord));\
-	WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start) + (vsize);	\
-    }								\
- } while (0)
+#define WSTACK_RESTORE(s, src)			\
+do {						\
+    ASSERT(s.wstart == WSTK_DEF_STACK(s));	\
+    s = *(src);  /* struct copy */		\
+    (src)->wstart = NULL;			\
+    ASSERT(s.wsp >= s.wstart);			\
+    ASSERT(s.wsp <= s.wend);			\
+} while (0)
 
-#define WSTACK_IS_STATIC(s) (WSTK_CONCAT(s,_start) == WSTK_CONCAT(s,_default_stack))
+#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s)))
 
-#define WSTACK_PUSH(s, x)						\
-do {									\
-    if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) {			\
-	erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \
-			&WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end));	\
-    }									\
-    *WSTK_CONCAT(s,_sp)++ = (x);					\
+#define WSTACK_PUSH(s, x)				\
+do {							\
+    if (s.wsp == s.wend) {				\
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); 	\
+    }							\
+    *s.wsp++ = (x);					\
 } while(0)
 
-#define WSTACK_PUSH2(s, x, y)						\
-do {									\
-    if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) {			\
-	erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \
-		 &WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end));	        \
-    }									\
-    *WSTK_CONCAT(s,_sp)++ = (x);					\
-    *WSTK_CONCAT(s,_sp)++ = (y);					\
+#define WSTACK_PUSH2(s, x, y)			\
+do {						\
+    if (s.wsp > s.wend - 2) {			\
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+    }						\
+    *s.wsp++ = (x);				\
+    *s.wsp++ = (y);				\
 } while(0)
 
-#define WSTACK_PUSH3(s, x, y, z)					\
-do {									\
-    if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) {			\
-	erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \
-		&WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end));		\
-    }									\
-    *WSTK_CONCAT(s,_sp)++ = (x);					\
-    *WSTK_CONCAT(s,_sp)++ = (y);					\
-    *WSTK_CONCAT(s,_sp)++ = (z);					\
+#define WSTACK_PUSH3(s, x, y, z)		\
+do {						\
+    if (s.wsp > s.wend - 3) {	\
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+    }						\
+    *s.wsp++ = (x);				\
+    *s.wsp++ = (y);				\
+    *s.wsp++ = (z);				\
 } while(0)
 
-#define WSTACK_COUNT(s) (WSTK_CONCAT(s,_sp) - WSTK_CONCAT(s,_start))
+#define WSTACK_COUNT(s) (s.wsp - s.wstart)
+#define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
+#define WSTACK_POP(s) (*(--s.wsp))
 
-#define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start))
-#define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp)))
 
 /* binary.c */
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 216952e2ac..7f8bdcb2ca 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -185,39 +185,41 @@ erts_set_hole_marker(Eterm* ptr, Uint sz)
  * Helper function for the ESTACK macros defined in global.h.
  */
 void
-erl_grow_stack(ErtsAlcType_t a_type, Eterm** start, Eterm** sp, Eterm** end)
+erl_grow_estack(ErtsEStack* s, Eterm* default_estack)
 {
-    Uint old_size = (*end - *start);
+    Uint old_size = (s->end - s->start);
     Uint new_size = old_size * 2;
-    Uint sp_offs = *sp - *start;
-    if (new_size > 2 * DEF_ESTACK_SIZE) {
-	*start = erts_realloc(a_type, (void *) *start, new_size*sizeof(Eterm));
+    Uint sp_offs = s->sp - s->start;
+    if (s->start != default_estack) {
+	s->start = erts_realloc(s->alloc_type, s->start,
+				new_size*sizeof(Eterm));
     } else {
-	Eterm* new_ptr = erts_alloc(a_type, new_size*sizeof(Eterm));
-	sys_memcpy(new_ptr, *start, old_size*sizeof(Eterm));
-	*start = new_ptr;
+	Eterm* new_ptr = erts_alloc(s->alloc_type, new_size*sizeof(Eterm));
+	sys_memcpy(new_ptr, s->start, old_size*sizeof(Eterm));
+	s->start = new_ptr;
     }
-    *end = *start + new_size;
-    *sp = *start + sp_offs;
+    s->end = s->start + new_size;
+    s->sp = s->start + sp_offs;
 }
 /*
- * Helper function for the ESTACK macros defined in global.h.
+ * Helper function for the WSTACK macros defined in global.h.
  */
 void
-erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end)
+erl_grow_wstack(ErtsWStack* s, UWord* default_wstack)
 {
-    Uint old_size = (*end - *start);
+    Uint old_size = (s->wend - s->wstart);
     Uint new_size = old_size * 2;
-    Uint sp_offs = *sp - *start;
-    if (new_size > 2 * DEF_ESTACK_SIZE) {
-	*start = erts_realloc(a_type, (void *) *start, new_size*sizeof(UWord));
+    Uint sp_offs = s->wsp - s->wstart;
+    if (s->wstart != default_wstack) {
+	s->wstart = erts_realloc(s->alloc_type, s->wstart,
+				 new_size*sizeof(UWord));
     } else {
-	UWord* new_ptr = erts_alloc(a_type, new_size*sizeof(UWord));
-	sys_memcpy(new_ptr, *start, old_size*sizeof(UWord));
-	*start = new_ptr;
+	UWord* new_ptr = erts_alloc(s->alloc_type, new_size*sizeof(UWord));
+	sys_memcpy(new_ptr, s->wstart, old_size*sizeof(UWord));
+	s->wstart = new_ptr;
     }
-    *end = *start + new_size;
-    *sp = *start + sp_offs;
+    s->wend = s->wstart + new_size;
+    s->wsp = s->wstart + sp_offs;
 }
 
 /* CTYPE macros */
-- 
cgit v1.2.3


From d8565ab63de970f01ec1bf80fcbf0b3c692c7bd0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 22 Jan 2014 11:12:59 +0100
Subject: erts: Fix halfword compile errors in ESTACK

Errors introduced in 172ebf11dc455e2
---
 erts/emulator/beam/global.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c183c519ff..83a8911a36 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -371,9 +371,9 @@ extern int stackdump_on_exit;
  */
 
 typedef struct {
-    UWord* start;
-    UWord* sp;
-    UWord* end;
+    Eterm* start;
+    Eterm* sp;
+    Eterm* end;
     ErtsAlcType_t alloc_type;
 }ErtsEStack;
 
@@ -384,7 +384,7 @@ void erl_grow_estack(ErtsEStack*, Eterm* def_stack);
 #define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack)
 
 #define DECLARE_ESTACK(s)				\
-    UWord ESTK_DEF_STACK(s)[DEF_ESTACK_SIZE];		\
+    Eterm ESTK_DEF_STACK(s)[DEF_ESTACK_SIZE];		\
     ErtsEStack s = {					\
         ESTK_DEF_STACK(s),  /* start */ 		\
         ESTK_DEF_STACK(s),  /* sp */			\
@@ -418,8 +418,8 @@ do {\
     if (s.start == ESTK_DEF_STACK(s)) {\
 	UWord _wsz = ESTACK_COUNT(s);\
 	(dst)->start = erts_alloc(s.alloc_type,\
-				  DEF_ESTACK_SIZE * sizeof(UWord));\
-	memcpy((dst)->start, s.start,_wsz*sizeof(UWord));\
+				  DEF_ESTACK_SIZE * sizeof(Eterm));\
+	memcpy((dst)->start, s.start,_wsz*sizeof(Eterm));\
 	(dst)->sp = (dst)->start + _wsz;\
 	(dst)->end = (dst)->start + DEF_ESTACK_SIZE;\
 	(dst)->alloc_type = s.alloc_type;\
@@ -495,7 +495,7 @@ typedef struct {
 
 #define DEF_WSTACK_SIZE (16)
 
-void erl_grow_wstack(ErtsWStack*, Eterm* def_stack);
+void erl_grow_wstack(ErtsWStack*, UWord* def_stack);
 #define WSTK_CONCAT(a,b) a##b
 #define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack)
 
-- 
cgit v1.2.3


From 1b904fd1fcec000efb33446859e75872dc00ef2b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 22 Jan 2014 16:24:00 +0100
Subject: erts: Refactor big-float compare on HALFWORD to use C-stack

for the temporary conversion from float to big.

Preparation for coming bugfix of 'big_buf' array size.
---
 erts/emulator/beam/big.c         |  6 ++++--
 erts/emulator/beam/erl_process.h |  1 -
 erts/emulator/beam/erl_term.h    |  3 ++-
 erts/emulator/beam/erl_vm.h      |  3 +--
 erts/emulator/beam/utils.c       | 18 ++++++++----------
 5 files changed, 15 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 2b27b111d8..4343c6cb4c 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1603,6 +1603,8 @@ big_to_double(Wterm x, double* resp)
 /*
  * Logic has been copied from erl_bif_guard.c and slightly
  * modified to use a static instead of dynamic heap
+ *
+ * HALFWORD: Return relative term with 'heap' as base.
  */
 Eterm
 double_to_big(double x, Eterm *heap)
@@ -1633,7 +1635,7 @@ double_to_big(double x, Eterm *heap)
     sz = BIG_NEED_SIZE(ds); /* number of words including arity */
 
     hp = heap;
-    res = make_big(hp);
+    res = make_big_rel(hp, heap);
     xp = (ErtsDigit*) (hp + 1);
 
     for (i = ds - 1; i >= 0; i--) {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index e35d1c785c..9c8cce3cb5 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -499,7 +499,6 @@ struct ErtsSchedulerData_ {
     Eterm tmp_heap[TMP_HEAP_SIZE];
     int num_tmp_heap_used;
     Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
-    Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE];
     Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
 #endif
     ErtsSchedulerSleepInfo *ssi;
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 953edf79ea..50d3e63c58 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1126,6 +1126,7 @@ extern unsigned tag_val_def(Wterm);
 #define make_tuple_rel make_boxed_rel
 #define make_external_rel make_boxed_rel
 #define make_internal_ref_rel make_boxed_rel
+#define make_big_rel make_boxed_rel
 
 #define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE))
 #define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE))
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 337422eead..b7de8208ad 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -46,7 +46,6 @@
    heap data on the C stack or if we use the buffers in the scheduler data. */
 #define TMP_HEAP_SIZE 128            /* Number of Eterm in the schedulers
 				        small heap for transient heap data */
-#define CMP_TMP_HEAP_SIZE       32   /* cmp wants its own tmp-heap... */
 #define ERL_ARITH_TMP_HEAP_SIZE 4    /* as does erl_arith... */
 #define BEAM_EMU_TMP_HEAP_SIZE  2    /* and beam_emu... */
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 7f8bdcb2ca..86bb5fd3ad 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2688,11 +2688,7 @@ tailrecur_ne:
     {
 	FloatDef f1, f2;
 	Eterm big;
-#if HEAP_ON_C_STACK
-	Eterm big_buf[CMP_TMP_HEAP_SIZE]; /* If HEAP_ON_C_STACK */
-#else
-	Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap;
-#endif
+	Eterm big_buf[32];
 #if HALFWORD_HEAP
 	Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base);
 	Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base);
@@ -2718,8 +2714,9 @@ tailrecur_ne:
 		/* Float is within the no loss limit */
 		f1.fd = signed_val(aw);
 		j = float_comp(f1.fd, f2.fd);
+	    }
 #if ERTS_SIZEOF_ETERM == 8
-	    } else if (f2.fd > (double) (MAX_SMALL + 1)) {
+	    else if (f2.fd > (double) (MAX_SMALL + 1)) {
 		/* Float is a positive bignum, i.e. bigger */
 		j = -1;
 	    } else if (f2.fd < (double) (MIN_SMALL - 1)) {
@@ -2730,7 +2727,7 @@ tailrecur_ne:
 		j = signed_val(aw) - (Sint) f2.fd;
 	    }
 #else
-	    } else {
+	    else {
 		/* If float is positive it is bigger than small */
 		j = (f2.fd > 0.0) ? -1 : 1;
 	    }
@@ -2765,7 +2762,7 @@ tailrecur_ne:
 		}
 	    } else {
 		big = double_to_big(f2.fd, big_buf);
-		j = big_comp(aw, big);
+		j = big_comp(aw, rterm2wterm(big,big_buf));
 	    }
 	    if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) {
 		j = -j;
@@ -2777,8 +2774,9 @@ tailrecur_ne:
 		/* Float is within the no loss limit */
 		f2.fd = signed_val(bw);
 		j = float_comp(f1.fd, f2.fd);
+	    }
 #if ERTS_SIZEOF_ETERM == 8
-	    } else if (f1.fd > (double) (MAX_SMALL + 1)) {
+	    else if (f1.fd > (double) (MAX_SMALL + 1)) {
 		/* Float is a positive bignum, i.e. bigger */
 		j = 1;
 	    } else if (f1.fd < (double) (MIN_SMALL - 1)) {
@@ -2789,7 +2787,7 @@ tailrecur_ne:
 		j = (Sint) f1.fd - signed_val(bw);
 	    }
 #else
-	    } else {
+	    else {
 		/* If float is positive it is bigger than small */
 		j = (f1.fd > 0.0) ? 1 : -1;
 	    }
-- 
cgit v1.2.3


From af0227d7591bde8927ea95c93cbadee6b812b1d9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 22 Jan 2014 16:24:48 +0100
Subject: erts: Fix crash when comparing very large floats with integers

big_buf was one word too short on 32-bit emulators causing
memory corruption.
Seems like this did not cause a problem before the ESTACK memory layout
was changed in 172ebf11dc455e22b87f.
---
 erts/emulator/beam/big.c   | 3 ++-
 erts/emulator/beam/big.h   | 4 ++--
 erts/emulator/beam/utils.c | 5 +++--
 3 files changed, 7 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 4343c6cb4c..41a041eba6 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1607,7 +1607,7 @@ big_to_double(Wterm x, double* resp)
  * HALFWORD: Return relative term with 'heap' as base.
  */
 Eterm
-double_to_big(double x, Eterm *heap)
+double_to_big(double x, Eterm *heap, Uint hsz)
 {
     int is_negative;
     int ds;
@@ -1638,6 +1638,7 @@ double_to_big(double x, Eterm *heap)
     res = make_big_rel(hp, heap);
     xp = (ErtsDigit*) (hp + 1);
 
+    ASSERT(ds < hsz);
     for (i = ds - 1; i >= 0; i--) {
 	ErtsDigit d;
 
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 1a7b14170f..d80111822e 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -141,7 +141,7 @@ Eterm big_lshift(Eterm, Sint, Eterm*);
 int big_comp (Wterm, Wterm);
 int big_ucomp (Eterm, Eterm);
 int big_to_double(Wterm x, double* resp);
-Eterm double_to_big(double, Eterm*);
+Eterm double_to_big(double, Eterm*, Uint hsz);
 Eterm small_to_big(Sint, Eterm*);
 Eterm uint_to_big(Uint, Eterm*);
 Eterm uword_to_big(UWord, Eterm*);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 86bb5fd3ad..e0776cf67d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2688,7 +2688,6 @@ tailrecur_ne:
     {
 	FloatDef f1, f2;
 	Eterm big;
-	Eterm big_buf[32];
 #if HALFWORD_HEAP
 	Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base);
 	Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base);
@@ -2699,6 +2698,8 @@ tailrecur_ne:
 #define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2))
 #define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1))
 #define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */
+	Eterm big_buf[BIG_NEED_SIZE(BIG_ARITY_FLOAT_MAX)];
+
 	b_tag = tag_val_def(bw);
 
 	switch(_NUMBER_CODE(a_tag, b_tag)) {
@@ -2761,7 +2762,7 @@ tailrecur_ne:
 		    j = float_comp(f1.fd, f2.fd);
 		}
 	    } else {
-		big = double_to_big(f2.fd, big_buf);
+		big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm));
 		j = big_comp(aw, rterm2wterm(big,big_buf));
 	    }
 	    if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) {
-- 
cgit v1.2.3


From 29196a3528e034d3d46ec4589749074f8d766b97 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 23 Jan 2014 12:28:13 +0100
Subject: erts: Fix faulty assert in match spec engine.

---
 erts/emulator/beam/erl_db_util.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index ef3749a2c4..a358ecf326 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1838,7 +1838,7 @@ restart:
 	    ep = termp;
 	    break;
 	case matchArrayBind: /* When the array size is unknown. */
-	    ASSERT(termp);
+	    ASSERT(termp || arity==0);
 	    n = *pc++;
 	    variables[n].term = dpm_array_to_list(psp, termp, arity);
 	    break;
-- 
cgit v1.2.3


From e7ea832a4a3a8ba2f94ce02a47ca34b60277cb0a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 16 Jan 2014 23:41:47 +0100
Subject: Add support for scheduler utilization balancing

For more information see documentation of the new command line argument +sub
---
 erts/emulator/beam/erl_init.c    |  26 +++
 erts/emulator/beam/erl_process.c | 424 +++++++++++++++++++++++++++++++++++----
 erts/emulator/beam/erl_process.h | 104 ++++++++--
 3 files changed, 497 insertions(+), 57 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 1af80dd04b..1d4f617746 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -537,6 +537,12 @@ void erts_usage(void)
     erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sct cput   set cpu topology,\n");
     erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
+    erts_fprintf(stderr, "-sub bool   enable/disable scheduler utilization balancing,\n");
+#else
+    erts_fprintf(stderr, "-sub false  disable scheduler utilization balancing,\n");
+#endif
+    erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sws val    set scheduler wakeup strategy, valid values are:\n");
     erts_fprintf(stderr, "            default|legacy.\n");
     erts_fprintf(stderr, "-swct val   set scheduler wake cleanup threshold, valid values are:\n");
@@ -1512,6 +1518,26 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
+	    else if (has_prefix("ub", sub_param)) {
+		arg = get_arg(sub_param+2, argv[i+1], &i);
+		if (sys_strcmp("true", arg) == 0) {
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
+		    erts_sched_balance_util = 1;
+#else
+		    erts_fprintf(stderr,
+				 "scheduler utilization balancing not "
+				 "supported on this system\n");
+		    erts_usage();
+#endif
+		}
+		else if (sys_strcmp("false", arg) == 0)
+		    erts_sched_balance_util = 0;
+		else {
+		    erts_fprintf(stderr, "bad scheduler utilization balancing "
+				 " value '%s'\n", arg);
+		    erts_usage();
+		}
+	    }
 	    else if (has_prefix("wct", sub_param)) {
 		arg = get_arg(sub_param+3, argv[i+1], &i);
 		if (erts_sched_set_wake_cleanup_threshold(arg) != 0) {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 21fd8dd50a..6143c5fa3a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -144,6 +144,7 @@ extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
 int erts_sched_compact_load;
+int erts_sched_balance_util = 0;
 Uint erts_no_schedulers;
 
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY		(4*1024*1024)
@@ -608,6 +609,7 @@ erts_late_init_process(void)
 static void
 init_sched_wall_time(ErtsSchedWallTime *swtp)
 {
+    swtp->need = erts_sched_balance_util;
     swtp->enabled = 0;
     swtp->start = 0;
     swtp->working.total = 0;
@@ -630,27 +632,253 @@ sched_wall_time_ts(void)
 #endif
 }
 
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+
+#ifdef ARCH_64
+
+static ERTS_INLINE Uint64
+aschedtime_read(ErtsAtomicSchedTime *var)
+{
+    return (Uint64) erts_atomic_read_nob((erts_atomic_t *) var);
+}
+
+static ERTS_INLINE void
+aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val)
+{
+    erts_atomic_set_nob((erts_atomic_t *) var, (erts_aint_t) val);
+}
+
+static ERTS_INLINE void
+aschedtime_init(ErtsAtomicSchedTime *var)
+{
+    erts_atomic_init_nob((erts_atomic_t *) var, (erts_aint_t) 0);
+}
+
+#elif defined(ARCH_32)
+
+static ERTS_INLINE Uint64
+aschedtime_read(ErtsAtomicSchedTime *var)
+{
+    erts_dw_aint_t dw;
+    erts_dw_atomic_read_nob((erts_dw_atomic_t *) var, &dw);
+#ifdef ETHR_SU_DW_NAINT_T__
+    return (Uint64) dw.dw_sint;
+#else
+    {
+	Uint64 res;
+	res = (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_HIGH_WORD]);
+	res <<= 32;
+	res |= (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_LOW_WORD]);
+	return res;
+    }
+#endif    
+}
+
+static ERTS_INLINE void
+aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val)
+{
+    erts_dw_aint_t dw;
+#ifdef ETHR_SU_DW_NAINT_T__
+    dw.dw_sint = (ETHR_SU_DW_NAINT_T__) val;
+#else
+    dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
+    dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
+#endif
+    erts_dw_atomic_set_nob((erts_dw_atomic_t *) var, &dw);
+}
+
+static ERTS_INLINE void
+aschedtime_init(ErtsAtomicSchedTime *var)
+{
+    erts_dw_aint_t dw;
+    dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) 0;
+    dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) 0;
+    erts_dw_atomic_init_nob((erts_dw_atomic_t *) var, &dw);
+}
+
+#else
+#  error :-/
+#endif
+
+#define ERTS_GET_AVG_MAX_UNLOCKED_TRY 50
+#define ERTS_SCHED_AVG_UTIL_WRITE_MARKER (~((Uint64) 0))
+
+/* Intervals in nanoseconds */
+#define ERTS_SCHED_UTIL_SHORT_INTERVAL ((Uint64) 1*1000*1000*1000)
+#define ERTS_SCHED_UTIL_LONG_INTERVAL ((Uint64) 10*1000*1000*1000)
+
+
+#define ERTS_SCHED_UTIL_IGNORE_IMBALANCE_DIFF 5000 /* ppm */
+
+static ERTS_INLINE Uint64
+calc_sched_worktime(int is_working, Uint64 now, Uint64 last,
+		    Uint64 interval, Uint64 old_worktime)
+{
+    Uint64 worktime;
+    Uint64 new;
+
+    if (now <= last)
+	return old_worktime;
+
+    new = now - last;
+
+    if (new >= interval)
+	return is_working ? interval : (Uint64) 0;
+
+
+    /*
+     * Division by 1000 in order to avoid
+     * overflow. If changed update assertions
+     * in init_runq_sched_util().
+     */
+    worktime = old_worktime;
+    worktime *= (interval - new)/1000;
+    worktime /= (interval/1000);
+    if (is_working)
+	worktime += new;
+
+    ASSERT(0 <= worktime && worktime <= interval);
+
+    return worktime;
+}
+
+static ERTS_INLINE void
+update_avg_sched_util(ErtsSchedulerData *esdp, Uint64 now, int is_working)
+{
+    ErtsRunQueue *rq;
+    int worked;
+    Uint64 swt, lwt, last;
+
+    rq = esdp->run_queue;
+    last = aschedtime_read(&rq->sched_util.last);
+
+    if (now <= last) {
+	ASSERT(last == ERTS_SCHED_AVG_UTIL_WRITE_MARKER);
+	return;
+    }
+
+    ASSERT(now >= last);
+
+    worked = rq->sched_util.is_working;
+
+    swt = calc_sched_worktime(worked, now, last, ERTS_SCHED_UTIL_SHORT_INTERVAL,
+			      rq->sched_util.worktime.short_interval);
+    lwt = calc_sched_worktime(worked, now, last, ERTS_SCHED_UTIL_LONG_INTERVAL,
+			      rq->sched_util.worktime.long_interval);
+
+    aschedtime_set(&rq->sched_util.last, ERTS_SCHED_AVG_UTIL_WRITE_MARKER);
+    ERTS_THR_WRITE_MEMORY_BARRIER;
+    rq->sched_util.is_working = is_working;
+    rq->sched_util.worktime.short_interval = swt;
+    rq->sched_util.worktime.long_interval = lwt;
+    ERTS_THR_WRITE_MEMORY_BARRIER;
+    aschedtime_set(&rq->sched_util.last, now);
+}
+
+int
+erts_get_sched_util(ErtsRunQueue *rq, int initially_locked, int short_interval)
+{
+    /* Average scheduler utilization in ppm */
+    int util, is_working, try = 0, locked = initially_locked;
+    Uint64 worktime, old_worktime, now, last, interval, *old_worktimep;
+
+    if (short_interval) {
+	old_worktimep = &rq->sched_util.worktime.short_interval;
+	interval = ERTS_SCHED_UTIL_SHORT_INTERVAL;
+    }
+    else {
+	old_worktimep = &rq->sched_util.worktime.long_interval;
+	interval = ERTS_SCHED_UTIL_LONG_INTERVAL;
+    }
+
+    while (1) {
+	Uint64 chk_last;
+	last = aschedtime_read(&rq->sched_util.last);
+	ERTS_THR_READ_MEMORY_BARRIER;
+	is_working = rq->sched_util.is_working;
+	old_worktime = *old_worktimep;
+	ERTS_THR_READ_MEMORY_BARRIER;
+	chk_last = aschedtime_read(&rq->sched_util.last);
+	if (chk_last == last)
+	    break;
+	if (!locked) {
+	    if (++try >= ERTS_GET_AVG_MAX_UNLOCKED_TRY) {
+		/* Writer will eventually block on runq-lock */
+		erts_smp_runq_lock(rq);
+		locked = 1;
+	    }
+	}
+    }
+
+    if (!initially_locked && locked)
+	erts_smp_runq_unlock(rq);
+
+    now = sched_wall_time_ts();
+    worktime = calc_sched_worktime(is_working, now, last, interval, old_worktime);
+
+    util = (int) ((worktime * 1000000)/interval);
+
+    ASSERT(0 <= util && util <= 1000000);
+
+    return util;
+}
+
+static void
+init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled)
+{
+    aschedtime_init(&rqsu->last);
+    if (!enabled)
+	aschedtime_set(&rqsu->last, ERTS_SCHED_AVG_UTIL_WRITE_MARKER);
+    rqsu->is_working = 0;
+    rqsu->worktime.short_interval = (Uint64) 0;
+    rqsu->worktime.long_interval = (Uint64) 0;
+
+#ifdef DEBUG
+    {
+	Uint64 intrvl;
+	/*
+	 * If one of these asserts fail we may have
+	 * overflow in calc_sched_worktime(). Which
+	 * have to be fixed either by shrinking
+	 * interval size, or fix calculation of
+	 * worktime in calc_sched_worktime().
+	 */
+	intrvl = ERTS_SCHED_UTIL_SHORT_INTERVAL;
+	ASSERT(intrvl*(intrvl/1000) > intrvl);
+	intrvl = ERTS_SCHED_UTIL_LONG_INTERVAL;
+	ASSERT(intrvl*(intrvl/1000) > intrvl);
+    }
+#endif
+}
+
+#endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */
+
 static ERTS_INLINE void
 sched_wall_time_change(ErtsSchedulerData *esdp, int working)
 {
-    if (esdp->sched_wall_time.enabled) {
+    if (esdp->sched_wall_time.need) {
 	Uint64 ts = sched_wall_time_ts();
-	if (working) {
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+	update_avg_sched_util(esdp, ts, working);
+#endif
+	if (esdp->sched_wall_time.enabled) {
+	    if (working) {
 #ifdef DEBUG
-	    ASSERT(!esdp->sched_wall_time.working.currently);
-	    esdp->sched_wall_time.working.currently = 1;
+		ASSERT(!esdp->sched_wall_time.working.currently);
+		esdp->sched_wall_time.working.currently = 1;
 #endif
-	    ts -= esdp->sched_wall_time.start;
-	    esdp->sched_wall_time.working.start = ts;
-	}
-	else {
+		ts -= esdp->sched_wall_time.start;
+		esdp->sched_wall_time.working.start = ts;
+	    }
+	    else {
 #ifdef DEBUG
-	    ASSERT(esdp->sched_wall_time.working.currently);
-	    esdp->sched_wall_time.working.currently = 0;
+		ASSERT(esdp->sched_wall_time.working.currently);
+		esdp->sched_wall_time.working.currently = 0;
 #endif
-	    ts -= esdp->sched_wall_time.start;
-	    ts -= esdp->sched_wall_time.working.start;
-	    esdp->sched_wall_time.working.total += ts;
+		ts -= esdp->sched_wall_time.start;
+		ts -= esdp->sched_wall_time.working.start;
+		esdp->sched_wall_time.working.total += ts;
+	    }
 	}
     }
 }
@@ -705,10 +933,13 @@ reply_sched_wall_time(void *vswtrp)
     ASSERT(esdp);
     
     if (swtrp->set) {
-	if (!swtrp->enable && esdp->sched_wall_time.enabled)
+	if (!swtrp->enable && esdp->sched_wall_time.enabled) {
+	    esdp->sched_wall_time.need = erts_sched_balance_util;
 	    esdp->sched_wall_time.enabled = 0;
+	}
 	else if (swtrp->enable && !esdp->sched_wall_time.enabled) {
 	    Uint64 ts = sched_wall_time_ts();
+	    esdp->sched_wall_time.need = 1;
 	    esdp->sched_wall_time.enabled = 1;
 	    esdp->sched_wall_time.start = ts;
 	    esdp->sched_wall_time.working.total = 0;
@@ -2084,9 +2315,8 @@ ongoing_multi_scheduling_block(void)
 }
 
 static ERTS_INLINE void
-empty_runq(ErtsRunQueue *rq)
+empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags)
 {
-    Uint32 old_flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED);
     if (old_flags & ERTS_RUNQ_FLG_NONEMPTY) {
 #ifdef DEBUG
 	erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
@@ -2106,6 +2336,23 @@ empty_runq(ErtsRunQueue *rq)
     }
 }
 
+static ERTS_INLINE void
+empty_runq(ErtsRunQueue *rq)
+{
+    Uint32 old_flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED);
+    empty_runq_aux(rq, old_flags);
+}
+
+static ERTS_INLINE Uint32
+empty_protected_runq(ErtsRunQueue *rq)
+{
+    Uint32 old_flags = ERTS_RUNQ_FLGS_BSET(rq,
+					   ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED,
+					   ERTS_RUNQ_FLG_PROTECTED);
+    empty_runq_aux(rq, old_flags);
+    return old_flags;
+}
+
 static ERTS_INLINE void
 non_empty_runq(ErtsRunQueue *rq)
 {
@@ -2130,6 +2377,18 @@ non_empty_runq(ErtsRunQueue *rq)
     }
 }
 
+void
+erts_empty_runq(ErtsRunQueue *rq)
+{
+    empty_runq(rq);
+}
+
+void
+erts_non_empty_runq(ErtsRunQueue *rq)
+{
+    non_empty_runq(rq);
+}
+
 static erts_aint32_t
 sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
 {
@@ -2632,7 +2891,7 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
 }
 
 static void
-wake_scheduler(ErtsRunQueue *rq, int incq)
+wake_scheduler(ErtsRunQueue *rq)
 {
     ErtsSchedulerSleepInfo *ssi;
     erts_aint32_t flgs;
@@ -2651,9 +2910,6 @@ wake_scheduler(ErtsRunQueue *rq, int incq)
 
     flgs = ssi_flags_set_wake(ssi);
     erts_sched_finish_poke(ssi, flgs);
-
-    if (incq && (flgs & ERTS_SSI_FLG_WAITING))
-	non_empty_runq(rq);
 }
 
 #define ERTS_NO_USED_RUNQS_SHIFT 16
@@ -2744,7 +3000,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
 	    if (try_inc_no_active_runqs(ix+1))
 		(void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE);
 	}
-	wake_scheduler(wrq, 0);
+	wake_scheduler(wrq);
 	return 1;
     }
     return 0;
@@ -2792,7 +3048,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq)
 {
 #ifdef ERTS_SMP
     if (runq)
-	wake_scheduler(runq, 1);
+	wake_scheduler(runq);
 #endif
 }
 
@@ -2810,7 +3066,7 @@ erts_sched_notify_check_cpu_bind(void)
     for (ix = 0; ix < erts_no_run_queues; ix++) {
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 	(void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND);
-	wake_scheduler(rq, 0);
+	wake_scheduler(rq);
     }
 #else
     erts_sched_check_cpu_bind(erts_get_scheduler_data());
@@ -2938,6 +3194,11 @@ check_immigration_need(ErtsRunQueue *c_rq, ErtsMigrationPath *mp, int prio)
     if (!f_rq)
 	return NULL;
 
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    if (mp->sched_util)
+	return NULL;
+#endif
+
     f_rq_flags = ERTS_RUNQ_FLGS_GET(f_rq);
     if (f_rq_flags & ERTS_RUNQ_FLG_PROTECTED)
 	return NULL;
@@ -3077,7 +3338,7 @@ suspend_run_queue(ErtsRunQueue *rq)
 				   ERTS_SSI_FLG_SUSPENDED);
     (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_SUSPENDED);
 
-    wake_scheduler(rq, 0);
+    wake_scheduler(rq);
 }
 
 static void scheduler_ix_resume_wake(Uint ix);
@@ -3169,6 +3430,9 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	    to_rq->misc.start = start;
 
 	to_rq->misc.end = end;
+
+	non_empty_runq(to_rq);
+
 	erts_smp_runq_unlock(to_rq);
 	smp_notify_inc_runq(to_rq);
 	erts_smp_runq_lock(to_rq);
@@ -3381,7 +3645,7 @@ try_steal_task(ErtsRunQueue *rq)
     Uint32 flags;
 
     /* Protect jobs we steal from getting stolen from us... */
-    flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_PROTECTED);
+    flags = empty_protected_runq(rq);
     if (flags & ERTS_RUNQ_FLG_SUSPENDED)
 	return 0; /* go suspend instead... */
 
@@ -3460,6 +3724,9 @@ typedef struct {
     int full_reds_history_change;
     int oowc;
     int max_len;
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    int sched_util;
+#endif
 } ErtsRunQueueBalance;
 static ErtsRunQueueBalance *run_queue_info;
 
@@ -3623,6 +3890,9 @@ check_balance(ErtsRunQueue *c_rq)
     Sint64 scheds_reds, full_scheds_reds;
     int forced, active, current_active, oowc, half_full_scheds, full_scheds,
 	mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix;
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    int sched_util_balancing;
+#endif
 
     if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) {
 	c_rq->check_balance_reds = INT_MAX;
@@ -3678,6 +3948,10 @@ check_balance(ErtsRunQueue *c_rq)
 	return;
     }
 
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    sched_util_balancing = 0;
+#endif
+
     freds_hist_ix = balance_info.full_reds_history_index;
     balance_info.full_reds_history_index++;
     if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE)
@@ -3708,7 +3982,12 @@ check_balance(ErtsRunQueue *c_rq)
 	run_queue_info[qix].oowc = rq->out_of_work_count;
 	run_queue_info[qix].max_len = rq->max_len;
 	rq->check_balance_reds = INT_MAX;
-	
+
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+	if (erts_sched_balance_util)
+	    run_queue_info[qix].sched_util = erts_get_sched_util(rq, 1, 0);
+#endif
+
 	erts_smp_runq_unlock(rq);
     }
 
@@ -3778,8 +4057,38 @@ check_balance(ErtsRunQueue *c_rq)
 	    mmax_len = run_queue_info[qix].max_len;
     }
 
-    if (!erts_sched_compact_load)
+    if (!erts_sched_compact_load) {
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+	if (erts_sched_balance_util && full_scheds < blnc_no_rqs) {
+	    int avg_util = 0;
+
+	    for (qix = 0; qix < blnc_no_rqs; qix++)
+		avg_util += run_queue_info[qix].sched_util;
+
+	    avg_util /= blnc_no_rqs; /* in ppm */
+
+	    sched_util_balancing = 1;
+	    /*
+	     * In order to avoid renaming a large amount of fields
+	     * we write utilization values instead of lenght values
+	     * in the 'max_len' and 'migration_limit' fields...
+	     */
+	    for (qix = 0; qix < blnc_no_rqs; qix++) {
+		run_queue_info[qix].flags = 0; /* Reset for later use... */
+		for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
+		    run_queue_info[qix].prio[pix].emigrate_to = -1;
+		    run_queue_info[qix].prio[pix].immigrate_from = -1;
+		    run_queue_info[qix].prio[pix].avail = 100;
+		    run_queue_info[qix].prio[pix].max_len = run_queue_info[qix].sched_util;
+		    run_queue_info[qix].prio[pix].migration_limit = avg_util;
+		}
+	    }
+	    active = blnc_no_rqs;
+	    goto setup_migration_paths;
+	}
+#endif
 	goto all_active;
+    }
 
     if (!forced && half_full_scheds != blnc_no_rqs) {
 	int min = 1;
@@ -3896,15 +4205,30 @@ check_balance(ErtsRunQueue *c_rq)
 	    }
 	}
 
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    setup_migration_paths:
+#endif
+
 	/* Setup migration paths for all priorities */
 	for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
 	    int low = 0, high = 0;
 	    for (qix = 0; qix < blnc_no_rqs; qix++) {
 		int len_diff = run_queue_info[qix].prio[pix].max_len;
 		len_diff -= run_queue_info[qix].prio[pix].migration_limit;
+
 #ifdef DBG_PRINT
 if (pix == 2) erts_fprintf(stderr, "%d ", len_diff);
 #endif
+
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+		if (sched_util_balancing
+		    && -ERTS_SCHED_UTIL_IGNORE_IMBALANCE_DIFF <= len_diff
+		    && len_diff <= ERTS_SCHED_UTIL_IGNORE_IMBALANCE_DIFF) {
+		    /* ignore minor imbalance */
+		    len_diff = 0;
+		}
+#endif
+		    
 		run_queue_compare[qix].qix = qix;
 		run_queue_compare[qix].len = len_diff;
 		if (len_diff != 0) {
@@ -4031,6 +4355,9 @@ erts_fprintf(stderr, "--------------------------------\n");
 	Uint32 flags = run_queue_info[qix].flags;
 	ErtsMigrationPath *mp = &new_mpaths->mpath[qix];
 
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+	mp->sched_util = sched_util_balancing;
+#endif
 	mp->flags = flags;
 	mp->misc_evac_runq = NULL;
 
@@ -4628,6 +4955,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     set_wakeup_other_data();
 #endif
 
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    if (erts_sched_balance_util)
+	erts_sched_compact_load = 0;
+#endif
+
     ASSERT(no_schedulers_online <= no_schedulers);
     ASSERT(no_schedulers_online >= 1);
     ASSERT(no_schedulers >= 1);
@@ -4696,6 +5028,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 	rq->ports.info.reds = 0;
 	rq->ports.start = NULL;
 	rq->ports.end = NULL;
+
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+	init_runq_sched_util(&rq->sched_util, erts_sched_balance_util);
+#endif
+
     }
 
 #ifdef ERTS_SMP
@@ -4794,6 +5131,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 	esdp->reductions = 0;
 
 	init_sched_wall_time(&esdp->sched_wall_time);
+
 	erts_port_task_handle_init(&esdp->nosuspend_port_task_handle);
 
     }
@@ -5761,7 +6099,7 @@ erts_set_schedulers_online(Process *p,
 
 		    for (ix = no; ix < online; ix++) {
 			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-			wake_scheduler(rq, 0);
+			wake_scheduler(rq);
 		    }
 		}
 	    }
@@ -5860,7 +6198,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 
 		for (ix = 1; ix < online; ix++) {
 		    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-		    wake_scheduler(rq, 0);
+		    wake_scheduler(rq);
 		}
 
 		if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
@@ -7294,7 +7632,7 @@ Process *schedule(Process *p, int calls)
     continue_check_activities_to_run:
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
     continue_check_activities_to_run_known_flags:
-
+	ASSERT(flags & ERTS_RUNQ_FLG_NONEMPTY);
 
 	if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
 	
@@ -7346,20 +7684,16 @@ Process *schedule(Process *p, int calls)
 	    rq->wakeup_other = 0;
 	    rq->wakeup_other_reds = 0;
 
-	    empty_runq(rq);
-
 	    flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
-	    if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
-		non_empty_runq(rq);
+	    if (flags & ERTS_RUNQ_FLG_SUSPENDED)
 		goto continue_check_activities_to_run_known_flags;
-	    }
-	    else if (!(flags & ERTS_RUNQ_FLG_INACTIVE)) {
-		if (try_steal_task(rq)) {
-		    non_empty_runq(rq);
+	    if (flags & ERTS_RUNQ_FLG_INACTIVE)
+		empty_runq(rq);
+	    else {
+		if (try_steal_task(rq))
 		    goto continue_check_activities_to_run;
-		}
 
-		(void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
+		empty_runq(rq);
 
 		/*
 		 * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done
@@ -7371,7 +7705,6 @@ Process *schedule(Process *p, int calls)
 		    goto continue_check_activities_to_run_known_flags;
 		}
 	    }
-
 #endif
 
 	    scheduler_wait(&fcalls, esdp, rq);
@@ -8486,6 +8819,10 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
 	rq->misc.start = molp;
     rq->misc.end = molp;
 
+#ifdef ERTS_SMP
+    non_empty_runq(rq);
+#endif
+
     erts_smp_runq_unlock(rq);
 
     smp_notify_inc_runq(rq);
@@ -9372,8 +9709,11 @@ save_pending_exiter(Process *p)
 
     erts_proclist_store_last(&rq->procs.pending_exiters, plp);
 
+    non_empty_runq(rq);
+
     erts_smp_runq_unlock(rq);
-    wake_scheduler(rq, 1);
+
+    wake_scheduler(rq);
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 043621125c..c2ef55f2d5 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -70,6 +70,9 @@ typedef struct process Process;
 
 struct ErtsNodesMonitor_;
 
+#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
+#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0
+
 #define ERTS_MAX_NO_OF_SCHEDULERS 1024
 
 #define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)
@@ -98,6 +101,7 @@ struct saved_calls {
 
 extern Export exp_send, exp_receive, exp_timeout;
 extern int erts_sched_compact_load;
+extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
 extern Uint erts_no_run_queues;
 extern int erts_sched_thread_suggested_stack_size;
@@ -198,6 +202,10 @@ extern int erts_sched_thread_suggested_stack_size;
 #define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
     ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
 					      (erts_aint32_t) (FLGS)))
+#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
+    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
+					       (erts_aint32_t) (MSK),	\
+					       (erts_aint32_t) (FLGS)))
 #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
     ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
 					       (erts_aint32_t) ~(FLGS)))
@@ -316,9 +324,40 @@ typedef struct {
     int reds;
 } ErtsRunQueueInfo;
 
+
+#ifdef HAVE_GETHRTIME
+#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
+#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
+#endif
+
 #ifdef ERTS_SMP
 
+#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
+
+#ifdef ARCH_64
+typedef erts_atomic_t ErtsAtomicSchedTime;
+#elif defined(ARCH_32)
+typedef erts_dw_atomic_t ErtsAtomicSchedTime;
+#else
+# error :-/
+#endif
+
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+typedef struct {
+    ErtsAtomicSchedTime last;
+    struct {
+	Uint64 short_interval;
+	Uint64 long_interval;
+    } worktime;
+    int is_working;
+} ErtsRunQueueSchedUtil;
+#endif
+
 typedef struct {
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    int sched_util;
+#endif
     Uint32 flags;
     ErtsRunQueue *misc_evac_runq;
     struct {
@@ -385,6 +424,9 @@ struct ErtsRunQueue_ {
 	Port *start;
 	Port *end;
     } ports;
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+    ErtsRunQueueSchedUtil sched_util;
+#endif
 };
 
 #ifdef ERTS_SMP
@@ -414,6 +456,7 @@ do {								\
 } while (0)
 
 typedef struct {
+    int need; /* "+sbu true" or scheduler_wall_time enabled */
     int enabled;
     Uint64 start;
     struct {
@@ -542,6 +585,12 @@ int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
 
+#ifdef ERTS_SMP
+void erts_empty_runq(ErtsRunQueue *rq);
+void erts_non_empty_runq(ErtsRunQueue *rq);
+#endif
+
+
 /*
  * Run queue locked during modifications. We use atomic ops since
  * other threads peek at values without run queue lock.
@@ -574,6 +623,10 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
 
     erts_smp_atomic32_set_relb(&rqi->len, len);
 
+#ifdef ERTS_SMP
+    if (rq->len == 0)
+	erts_non_empty_runq(rq);
+#endif
     rq->len++;
     if (rq->max_len < rq->len)
 	rq->max_len = len;
@@ -1695,6 +1748,13 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
 
 extern erts_atomic_t erts_migration_paths;
 
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+int erts_get_sched_util(ErtsRunQueue *rq,
+			int initially_locked,
+			int short_interval);
+#endif
+
+
 ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
 ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
 ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
@@ -1746,22 +1806,36 @@ erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
 		return mp->prio[prio].runq;
 	}
 
-
-	if (prio == ERTS_PORT_PRIO_LEVEL)
-	    len = RUNQ_READ_LEN(&c_rq->ports.info.len);
+#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
+	if (mp->sched_util) {
+	    ErtsRunQueue *rq = mp->prio[prio].runq;
+	    /* No migration if other is non-empty */
+	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
+		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
+		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
+		return rq;
+	    }
+	}
 	else
-	    len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);
-
-	if (len > mp->prio[prio].limit.this) {
-	    ErtsRunQueue *n_rq = mp->prio[prio].runq;
-	    if (n_rq) {
-		if (prio == ERTS_PORT_PRIO_LEVEL)
-		    len = RUNQ_READ_LEN(&n_rq->ports.info.len);
-		else
-		    len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);
-
-		if (len < mp->prio[prio].limit.other)
-		    return n_rq;
+#endif
+	{
+
+	    if (prio == ERTS_PORT_PRIO_LEVEL)
+		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
+	    else
+		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);
+
+	    if (len > mp->prio[prio].limit.this) {
+		ErtsRunQueue *n_rq = mp->prio[prio].runq;
+		if (n_rq) {
+		    if (prio == ERTS_PORT_PRIO_LEVEL)
+			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
+		    else
+			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);
+
+		    if (len < mp->prio[prio].limit.other)
+			return n_rq;
+		}
 	    }
 	}
     }
-- 
cgit v1.2.3


From 3ff735cc034dea43e593b5e58b91be95268cbf85 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 24 Jan 2014 12:45:27 +0100
Subject: Disable scheduler utilization balancing if +scl true is passed

---
 erts/emulator/beam/erl_init.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 1d4f617746..19088fd913 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1439,8 +1439,10 @@ erl_start(int argc, char **argv)
 	    }
 	    else if (has_prefix("cl", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-		if (sys_strcmp("true", arg) == 0)
+		if (sys_strcmp("true", arg) == 0) {
 		    erts_sched_compact_load = 1;
+		    erts_sched_balance_util = 0;
+		}
 		else if (sys_strcmp("false", arg) == 0)
 		    erts_sched_compact_load = 0;
 		else {
-- 
cgit v1.2.3


From ddb919031ec719b6ddf6f992cd1af6c27c0be0c5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 27 Jan 2014 16:28:21 +0100
Subject: Fix usage of non-empty run-queue flag

---
 erts/emulator/beam/erl_port_task.c |  5 +++++
 erts/emulator/beam/erl_process.c   | 19 +++++++------------
 2 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 547a42beb2..d4108067d0 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -877,6 +877,11 @@ enqueue_port(ErtsRunQueue *runq, Port *pp)
     ASSERT(runq->ports.start && runq->ports.end);
 
     erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
+
+#ifdef ERTS_SMP
+    if (runq->halt_in_progress)
+	erts_non_empty_runq(runq);
+#endif
 }
 
 static ERTS_INLINE Port *
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6143c5fa3a..f88d44ca03 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7603,19 +7603,13 @@ Process *schedule(Process *p, int calls)
 #ifdef ERTS_SMP
 	ErtsMigrationPaths *mps;
 	ErtsMigrationPath *mp;
-
-#ifdef ERTS_SMP
-	{
-	    ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
-	    if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
-		rq->procs.pending_exiters = NULL;
-		erts_smp_runq_unlock(rq);
-		handle_pending_exiters(pnd_xtrs);
-		erts_smp_runq_lock(rq);
-	    }
-		
+	ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
+	if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
+	    rq->procs.pending_exiters = NULL;
+	    erts_smp_runq_unlock(rq);
+	    handle_pending_exiters(pnd_xtrs);
+	    erts_smp_runq_lock(rq);
 	}
-#endif
 
 	if (rq->check_balance_reds <= 0)
 	    check_balance(rq);
@@ -7702,6 +7696,7 @@ Process *schedule(Process *p, int calls)
 		flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 		if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
 		    non_empty_runq(rq);
+		    flags |= ERTS_RUNQ_FLG_NONEMPTY;
 		    goto continue_check_activities_to_run_known_flags;
 		}
 	    }
-- 
cgit v1.2.3


From c1c03ae4ee50e58b7669ea88ec4d29c6b2b67c7b Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Thu, 9 Jan 2014 21:22:45 -0500
Subject: initial support for dirty schedulers and dirty NIFs

Add initial support for dirty schedulers.

There are two types of dirty schedulers: CPU schedulers and I/O
schedulers. By default, there are as many dirty CPU schedulers as there are
normal schedulers and as many dirty CPU schedulers online as normal
schedulers online. There are 10 dirty I/O schedulers (similar to the choice
of 10 as the default for async threads).

By default, dirty schedulers are disabled and conditionally compiled
out. To enable them, you must pass --enable-dirty-schedulers to the
top-level configure script when building Erlang/OTP.

Current dirty scheduler support requires the emulator to be built with SMP
support. This restriction will be lifted in the future.

You can specify the number of dirty schedulers with the command-line
options +SDcpu (for dirty CPU schedulers) and +SDio (for dirty I/O
schedulers). The +SDcpu option is similar to the +S option in that it takes
two numbers separated by a colon: C1:C2, where C1 specifies the number of
dirty schedulers available and C2 specifies the number of dirty schedulers
online. The +SDPcpu option allows numbers of dirty CPU schedulers available
and dirty CPU schedulers online to be specified as percentages, similar to
the existing +SP option for normal schedulers. The number of dirty CPU
schedulers created and dirty CPU schedulers online may not exceed the
number of normal schedulers created and normal schedulers online,
respectively. The +SDio option takes only a single number specifying the
number of dirty I/O schedulers available and online. There is no support
yet for programmatically changing at run time the number of dirty CPU
schedulers online via erlang:system_flag/2. Also, changing the number of
normal schedulers online via erlang:system_flag(schedulers_online,
NewSchedulersOnline) should ensure that there are no more dirty CPU
schedulers than normal schedulers, but this is not yet implemented. You can
retrieve the number of dirty schedulers by passing dirty_cpu_schedulers,
dirty_cpu_schedulers_online, or dirty_io_schedulers to
erlang:system_info/1.

Currently only NIFs are able to access dirty scheduler
functionality. Neither drivers nor BIFs currently support dirty
schedulers. This restriction will be addressed in the future.

If dirty scheduler support is present in the runtime, the initial status
line Erlang prints before presenting its interactive prompt will include
the indicator "[ds:C1:C2:I]" where "ds" indicates "dirty schedulers", "C1"
indicates the number of dirty CPU schedulers available, "C2" indicates the
number of dirty CPU schedulers online, and "I" indicates the number of
dirty I/O schedulers.

Document The dirty NIF API in the erl_nif man page. The API closely follows
Rickard Green's presentation slides from his talk "Future Extensions to the
Native Interface", presented at the 2011 Erlang Factory held in the San
Francisco Bay Area. Rickard's slides are available online at
http://bit.ly/1m34UHB .

Document the new erl command-line options, the additions to
erlang:system_info/1, and also add the erlang:system_flag/2 dirty scheduler
documentation even though it's not yet implemented.

To determine whether the dirty NIF API is available, native code can check
to see whether the C preprocessor macro ERL_NIF_DIRTY_SCHEDULER_SUPPORT is
defined. To check if dirty schedulers are available at run time, native
code can call the boolean enif_have_dirty_schedulers() function, and Erlang
code can call erlang:system_info(dirty_cpu_schedulers), which raises
badarg if no dirty scheduler support is available.

Add a simple dirty NIF test to the emulator NIF suite.
---
 erts/emulator/beam/beam_emu.c          |   9 +
 erts/emulator/beam/beam_load.h         |   1 +
 erts/emulator/beam/erl_alloc.c         |   3 +
 erts/emulator/beam/erl_bif_info.c      |  35 +-
 erts/emulator/beam/erl_init.c          | 304 +++++++++++++----
 erts/emulator/beam/erl_lock_check.c    |   3 +
 erts/emulator/beam/erl_nif.c           | 151 ++++++++
 erts/emulator/beam/erl_nif.h           |  17 +
 erts/emulator/beam/erl_nif_api_funcs.h |  14 +
 erts/emulator/beam/erl_process.c       | 607 +++++++++++++++++++++++++++------
 erts/emulator/beam/erl_process.h       | 103 +++++-
 11 files changed, 1059 insertions(+), 188 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b413f0e859..7fecdd5c5f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -217,6 +217,7 @@ BeamInstr beam_continue_exit[1];
 
 BeamInstr* em_call_error_handler;
 BeamInstr* em_apply_bif;
+BeamInstr* em_call_nif;
 
 
 /* NOTE These should be the only variables containing trace instructions.
@@ -3323,6 +3324,13 @@ void process_main(void)
 		reg[0] = r(0);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		erts_post_nif(&env);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (is_non_value(nif_bif_result) && c_p->freason == TRAP) {
+		    Export* ep = (Export*) c_p->psd->data[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT];
+		    ep->code[0] = I[-3];
+		    ep->code[1] = I[-2];
+		}
+#endif
 	    }
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -4964,6 +4972,7 @@ void process_main(void)
      
      em_call_error_handler = OpCode(call_error_handler);
      em_apply_bif = OpCode(apply_bif);
+     em_call_nif = OpCode(call_nif);
 
      beam_apply[0]             = (BeamInstr) OpCode(i_apply);
      beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 65a8f26d7c..bd22b0c4de 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -49,6 +49,7 @@ extern void** beam_ops;
 extern BeamInstr beam_debug_apply[];
 extern BeamInstr* em_call_error_handler;
 extern BeamInstr* em_apply_bif;
+extern BeamInstr* em_call_nif;
 
 /*
  * The following variables keep a sorted list of address ranges for
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 8094c6ee2e..c6b324dc15 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1754,6 +1754,9 @@ erts_alloc_register_scheduler(void *vesdp)
     int ix = (int) esdp->no;
     int aix;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
     for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
 	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
 	esdp->alloc_data.deallctr[aix] = NULL;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index e0b654cb22..f25b4dbae5 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -90,6 +90,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     " [smp:%beu:%beu]"
 #endif
 #ifdef USE_THREADS
+#ifdef ERTS_DIRTY_SCHEDULERS
+				     " [ds:%beu:%beu:%beu]"
+#endif
 				     " [async-threads:%d]"
 #endif
 #ifdef HIPE
@@ -312,7 +315,13 @@ erts_print_system_version(int to, void *arg, Process *c_p)
     char *ocp = otp_correction_package;
 #ifdef ERTS_SMP
     Uint total, online, active;
-    (void) erts_schedulers_state(&total, &online, &active, 0);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    Uint dirty_cpu, dirty_cpu_onln, dirty_io;
+
+    (void) erts_schedulers_state(&total, &online, &active, &dirty_cpu, &dirty_cpu_onln, &dirty_io, 0);
+#else
+    (void) erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 0);
+#endif
 #endif
     for (i = 0; i < sizeof(otp_correction_package)-4; i++) {
 	if (ocp[i] == '-' && ocp[i+1] == 'r' && ocp[i+2] == 'c')
@@ -330,6 +339,9 @@ erts_print_system_version(int to, void *arg, Process *c_p)
 		      rc_str
 #ifdef ERTS_SMP
 		      , total, online
+#ifdef ERTS_DIRTY_SCHEDULERS
+		      , dirty_cpu, dirty_cpu_onln, dirty_io
+#endif
 #endif
 #ifdef USE_THREADS
 		      , erts_async_max_threads
@@ -2477,6 +2489,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	switch (erts_schedulers_state(&total,
 				      &online,
 				      &active,
+				      NULL,
+				      NULL,
+				      NULL,
 				      1)) {
 	case ERTS_SCHDLR_SSPND_DONE: {
 	    Eterm *hp = HAlloc(BIF_P, 4);
@@ -2500,7 +2515,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(make_small(1));
 #else
 	Uint total, online, active;
-	switch (erts_schedulers_state(&total, &online, &active, 1)) {
+	switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) {
 	case ERTS_SCHDLR_SSPND_DONE:
 	    BIF_RET(make_small(online));
 	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
@@ -2517,7 +2532,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(make_small(1));
 #else
 	Uint total, online, active;
-	switch (erts_schedulers_state(&total, &online, &active, 1)) {
+	switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) {
 	case ERTS_SCHDLR_SSPND_DONE:
 	    BIF_RET(make_small(active));
 	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
@@ -2528,6 +2543,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	    ASSERT(0);
 	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
 	}
+#endif
+#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
+    } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) {
+	Uint dirty_cpu;
+	erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, 1);
+	BIF_RET(make_small(dirty_cpu));
+    } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) {
+	Uint dirty_cpu_onln;
+	erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, 1);
+	BIF_RET(make_small(dirty_cpu_onln));
+    } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) {
+	Uint dirty_io;
+	erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, &dirty_io, 1);
+	BIF_RET(make_small(dirty_io));
 #endif
     } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
 	res = make_small(erts_no_run_queues);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 19088fd913..c17256f466 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -484,98 +484,107 @@ void erts_usage(void)
 
     /*    erts_fprintf(stderr, "-# number  set the number of items to be used in traces etc\n"); */
 
-    erts_fprintf(stderr, "-a size     suggested stack size in kilo words for threads\n");
-    erts_fprintf(stderr, "            in the async-thread pool, valid range is [%d-%d]\n",
+    erts_fprintf(stderr, "-a size        suggested stack size in kilo words for threads\n");
+    erts_fprintf(stderr, "               in the async-thread pool, valid range is [%d-%d]\n",
 		 ERTS_ASYNC_THREAD_MIN_STACK_SIZE,
 		 ERTS_ASYNC_THREAD_MAX_STACK_SIZE);
-    erts_fprintf(stderr, "-A number   set number of threads in async thread pool,\n");
-    erts_fprintf(stderr, "            valid range is [0-%d]\n",
+    erts_fprintf(stderr, "-A number      set number of threads in async thread pool,\n");
+    erts_fprintf(stderr, "               valid range is [0-%d]\n",
 		 ERTS_MAX_NO_OF_ASYNC_THREADS);
 
-    erts_fprintf(stderr, "-B[c|d|i]   c to have Ctrl-c interrupt the Erlang shell,\n");
-    erts_fprintf(stderr, "            d (or no extra option) to disable the break\n");
-    erts_fprintf(stderr, "            handler, i to ignore break signals\n");
+    erts_fprintf(stderr, "-B[c|d|i]      c to have Ctrl-c interrupt the Erlang shell,\n");
+    erts_fprintf(stderr, "               d (or no extra option) to disable the break\n");
+    erts_fprintf(stderr, "               handler, i to ignore break signals\n");
 
     /*    erts_fprintf(stderr, "-b func    set the boot function (default boot)\n"); */
 
-    erts_fprintf(stderr, "-c          disable continuous date/time correction with\n");
-    erts_fprintf(stderr, "            respect to uptime\n");
+    erts_fprintf(stderr, "-c             disable continuous date/time correction with\n");
+    erts_fprintf(stderr, "               respect to uptime\n");
 
-    erts_fprintf(stderr, "-d          don't write a crash dump for internally detected errors\n");
-    erts_fprintf(stderr, "            (halt(String) will still produce a crash dump)\n");
-    erts_fprintf(stderr, "-fn[u|a|l]  Control how filenames are interpreted\n");
-    erts_fprintf(stderr, "-hms size   set minimum heap size in words (default %d)\n",
+    erts_fprintf(stderr, "-d             don't write a crash dump for internally detected errors\n");
+    erts_fprintf(stderr, "               (halt(String) will still produce a crash dump)\n");
+    erts_fprintf(stderr, "-fn[u|a|l]     Control how filenames are interpreted\n");
+    erts_fprintf(stderr, "-hms size      set minimum heap size in words (default %d)\n",
 	       H_DEFAULT_SIZE);
-    erts_fprintf(stderr, "-hmbs size  set minimum binary virtual heap size in words (default %d)\n",
+    erts_fprintf(stderr, "-hmbs size     set minimum binary virtual heap size in words (default %d)\n",
 	       VH_DEFAULT_SIZE);
 
     /*    erts_fprintf(stderr, "-i module  set the boot module (default init)\n"); */
 
-    erts_fprintf(stderr, "-K boolean  enable or disable kernel poll\n");
-    erts_fprintf(stderr, "-n[s|a|d]   Control behavior of signals to ports\n");
-    erts_fprintf(stderr, "            Note that this flag is deprecated!\n");
-    erts_fprintf(stderr, "-M<X> <Y>   memory allocator switches,\n");
-    erts_fprintf(stderr, "            see the erts_alloc(3) documentation for more info.\n");
-    erts_fprintf(stderr, "-pc <set>   Control what characters are considered printable (default latin1)\n");
-    erts_fprintf(stderr, "-P number   set maximum number of processes on this node,\n");
-    erts_fprintf(stderr, "            valid range is [%d-%d]\n",
+    erts_fprintf(stderr, "-K boolean     enable or disable kernel poll\n");
+    erts_fprintf(stderr, "-n[s|a|d]      Control behavior of signals to ports\n");
+    erts_fprintf(stderr, "               Note that this flag is deprecated!\n");
+    erts_fprintf(stderr, "-M<X> <Y>      memory allocator switches,\n");
+    erts_fprintf(stderr, "               see the erts_alloc(3) documentation for more info.\n");
+    erts_fprintf(stderr, "-pc <set>      Control what characters are considered printable (default latin1)\n");
+    erts_fprintf(stderr, "-P number      set maximum number of processes on this node,\n");
+    erts_fprintf(stderr, "               valid range is [%d-%d]\n",
 		 ERTS_MIN_PROCESSES, ERTS_MAX_PROCESSES);
-    erts_fprintf(stderr, "-Q number   set maximum number of ports on this node,\n");
-    erts_fprintf(stderr, "            valid range is [%d-%d]\n",
+    erts_fprintf(stderr, "-Q number      set maximum number of ports on this node,\n");
+    erts_fprintf(stderr, "               valid range is [%d-%d]\n",
 		 ERTS_MIN_PORTS, ERTS_MAX_PORTS);
-    erts_fprintf(stderr, "-R number   set compatibility release number,\n");
-    erts_fprintf(stderr, "            valid range [%d-%d]\n",
+    erts_fprintf(stderr, "-R number      set compatibility release number,\n");
+    erts_fprintf(stderr, "               valid range [%d-%d]\n",
 		 this_rel-2, this_rel);
 
-    erts_fprintf(stderr, "-r          force ets memory block to be moved on realloc\n");
-    erts_fprintf(stderr, "-rg amount  set reader groups limit\n");
-    erts_fprintf(stderr, "-sbt type   set scheduler bind type, valid types are:\n");
-    erts_fprintf(stderr, "-stbt type  u|ns|ts|ps|s|nnts|nnps|tnnps|db\n");
-    erts_fprintf(stderr, "-sbwt val   set scheduler busy wait threshold, valid values are:\n");
-    erts_fprintf(stderr, "            none|very_short|short|medium|long|very_long.\n");
-    erts_fprintf(stderr, "-scl bool   enable/disable compaction of scheduler load,\n");
-    erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
-    erts_fprintf(stderr, "-sct cput   set cpu topology,\n");
-    erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
+    erts_fprintf(stderr, "-r             force ets memory block to be moved on realloc\n");
+    erts_fprintf(stderr, "-rg amount     set reader groups limit\n");
+    erts_fprintf(stderr, "-sbt type      set scheduler bind type, valid types are:\n");
+    erts_fprintf(stderr, "-stbt type     u|ns|ts|ps|s|nnts|nnps|tnnps|db\n");
+    erts_fprintf(stderr, "-sbwt val      set scheduler busy wait threshold, valid values are:\n");
+    erts_fprintf(stderr, "               none|very_short|short|medium|long|very_long.\n");
+    erts_fprintf(stderr, "-scl bool      enable/disable compaction of scheduler load,\n");
+    erts_fprintf(stderr, "               see the erl(1) documentation for more info.\n");
+    erts_fprintf(stderr, "-sct cput      set cpu topology,\n");
+    erts_fprintf(stderr, "               see the erl(1) documentation for more info.\n");
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
-    erts_fprintf(stderr, "-sub bool   enable/disable scheduler utilization balancing,\n");
+    erts_fprintf(stderr, "-sub bool      enable/disable scheduler utilization balancing,\n");
 #else
-    erts_fprintf(stderr, "-sub false  disable scheduler utilization balancing,\n");
+    erts_fprintf(stderr, "-sub false     disable scheduler utilization balancing,\n");
 #endif
-    erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
-    erts_fprintf(stderr, "-sws val    set scheduler wakeup strategy, valid values are:\n");
-    erts_fprintf(stderr, "            default|legacy.\n");
-    erts_fprintf(stderr, "-swct val   set scheduler wake cleanup threshold, valid values are:\n");
-    erts_fprintf(stderr, "            very_lazy|lazy|medium|eager|very_eager.\n");
-    erts_fprintf(stderr, "-swt val    set scheduler wakeup threshold, valid values are:\n");
-    erts_fprintf(stderr, "            very_low|low|medium|high|very_high.\n");
-    erts_fprintf(stderr, "-sss size   suggested stack size in kilo words for scheduler threads,\n");
-    erts_fprintf(stderr, "            valid range is [%d-%d]\n",
+    erts_fprintf(stderr, "               see the erl(1) documentation for more info.\n");
+    erts_fprintf(stderr, "-sws val       set scheduler wakeup strategy, valid values are:\n");
+    erts_fprintf(stderr, "               default|legacy.\n");
+    erts_fprintf(stderr, "-swct val      set scheduler wake cleanup threshold, valid values are:\n");
+    erts_fprintf(stderr, "               very_lazy|lazy|medium|eager|very_eager.\n");
+    erts_fprintf(stderr, "-swt val       set scheduler wakeup threshold, valid values are:\n");
+    erts_fprintf(stderr, "               very_low|low|medium|high|very_high.\n");
+    erts_fprintf(stderr, "-sss size      suggested stack size in kilo words for scheduler threads,\n");
+    erts_fprintf(stderr, "               valid range is [%d-%d]\n",
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
 		 ERTS_SCHED_THREAD_MAX_STACK_SIZE);
-    erts_fprintf(stderr, "-spp Bool   set port parallelism scheduling hint\n");
-    erts_fprintf(stderr, "-S n1:n2    set number of schedulers (n1), and number of\n");
-    erts_fprintf(stderr, "            schedulers online (n2), maximum for both\n");
-    erts_fprintf(stderr, "            numbers is %d\n",
+    erts_fprintf(stderr, "-spp Bool      set port parallelism scheduling hint\n");
+    erts_fprintf(stderr, "-S n1:n2       set number of schedulers (n1), and number of\n");
+    erts_fprintf(stderr, "               schedulers online (n2), maximum for both\n");
+    erts_fprintf(stderr, "               numbers is %d\n",
 		 ERTS_MAX_NO_OF_SCHEDULERS);
-    erts_fprintf(stderr, "-SP p1:p2   specify schedulers (p1) and schedulers online (p2)\n");
-    erts_fprintf(stderr, "            as percentages of logical processors configured and logical\n");
-    erts_fprintf(stderr, "            processors available, respectively\n");
-    erts_fprintf(stderr, "-t size     set the maximum number of atoms the "
-			 "emulator can handle\n");
-    erts_fprintf(stderr, "            valid range is [%d-%d]\n",
+    erts_fprintf(stderr, "-SP p1:p2      specify schedulers (p1) and schedulers online (p2)\n");
+    erts_fprintf(stderr, "               as percentages of logical processors configured and logical\n");
+    erts_fprintf(stderr, "               processors available, respectively\n");
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_fprintf(stderr, "-SDcpu n1:n2   set number of dirty CPU schedulers (n1), and number of\n");
+    erts_fprintf(stderr, "               dirty CPU schedulers online (n2), valid range for both\n");
+    erts_fprintf(stderr, "               numbers is [1-%d], and n2 must be less than or equal to n1\n",
+		 ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS);
+    erts_fprintf(stderr, "-SDPcpu p1:p2  specify dirty CPU schedulers (p1) and dirty CPU schedulers\n");
+    erts_fprintf(stderr, "               online (p2) as percentages of logical processors configured\n");
+    erts_fprintf(stderr, "               and logical processors available, respectively\n");
+    erts_fprintf(stderr, "-SDio n        set number of dirty I/O schedulers, valid range is [0-%d]\n",
+		 ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS);
+#endif
+    erts_fprintf(stderr, "-t size        set the maximum number of atoms the emulator can handle\n");
+    erts_fprintf(stderr, "               valid range is [%d-%d]\n",
 		 MIN_ATOM_TABLE_SIZE, MAX_ATOM_TABLE_SIZE);
-    erts_fprintf(stderr, "-T number   set modified timing level,\n");
-    erts_fprintf(stderr, "            valid range is [0-%d]\n",
+    erts_fprintf(stderr, "-T number      set modified timing level, valid range is [0-%d]\n",
 		 ERTS_MODIFIED_TIMING_LEVELS-1);
-    erts_fprintf(stderr, "-V          print Erlang version\n");
+    erts_fprintf(stderr, "-V             print Erlang version\n");
 
-    erts_fprintf(stderr, "-v          turn on chatty mode (GCs will be reported etc)\n");
+    erts_fprintf(stderr, "-v             turn on chatty mode (GCs will be reported etc)\n");
 
-    erts_fprintf(stderr, "-W<i|w>     set error logger warnings mapping,\n");
-    erts_fprintf(stderr, "            see error_logger documentation for details\n");
-    erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n");
-    erts_fprintf(stderr, "            valid range is [1-%d]\n", INT_MAX/1024);
+    erts_fprintf(stderr, "-W<i|w>        set error logger warnings mapping,\n");
+    erts_fprintf(stderr, "               see error_logger documentation for details\n");
+    erts_fprintf(stderr, "-zdbbl size    set the distribution buffer busy limit in kilobytes\n");
+    erts_fprintf(stderr, "               valid range is [1-%d]\n", INT_MAX/1024);
     erts_fprintf(stderr, "\n");
     erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n");
     erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n");
@@ -643,6 +652,13 @@ early_init(int *argc, char **argv) /*
     int schdlrs_percentage = 100;
     int schdlrs_onln_percentage = 100;
     int max_main_threads;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    int dirty_cpu_scheds;
+    int dirty_cpu_scheds_online;
+    int dirty_cpu_scheds_pctg = 100;
+    int dirty_cpu_scheds_onln_pctg = 100;
+    int dirty_io_scheds;
+#endif
     int max_reader_groups;
     int reader_groups;
     char envbuf[21]; /* enough for any 64-bit integer */
@@ -718,6 +734,12 @@ early_init(int *argc, char **argv) /*
     schdlrs = no_schedulers;
     schdlrs_onln = no_schedulers_online;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    dirty_cpu_scheds = no_schedulers;
+    dirty_cpu_scheds_online = no_schedulers_online;
+    dirty_io_scheds = 10;
+#endif
+
     envbufsz = sizeof(envbuf);
 
     /* erts_sys_getenv(_raw)() not initialized yet; need erts_sys_getenv__() */
@@ -808,7 +830,121 @@ early_init(int *argc, char **argv) /*
                         VERBOSE(DEBUG_SYSTEM,
                                 ("using %d:%d scheduler percentages\n",
                                  schdlrs_percentage, schdlrs_onln_percentage));
-                    } else {
+                    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    else if (argv[i][2] == 'D') {
+			char *arg;
+			char *type = argv[i]+3;
+			if (strcmp(type, "Pcpu") == 0) {
+			    int ptot, ponln;
+			    arg = get_arg(argv[i]+7, argv[i+1], &i);
+			    switch (sscanf(arg, "%d:%d", &ptot, &ponln)) {
+			    case 0:
+				switch (sscanf(arg, ":%d", &ponln)) {
+				case 1:
+				    if (ponln < 0)
+					goto bad_SDPcpu;
+				    ptot = 100;
+				    goto chk_SDPcpu;
+				default:
+				    goto bad_SDPcpu;
+				}
+			    case 1:
+				if (ptot < 0)
+				    goto bad_SDPcpu;
+				ponln = ptot < 100 ? ptot : 100;
+				goto chk_SDPcpu;
+			    case 2:
+				if (ptot < 0 || ponln < 0)
+				    goto bad_SDPcpu;
+			    chk_SDPcpu:
+				dirty_cpu_scheds_pctg = ptot;
+				dirty_cpu_scheds_onln_pctg = ponln;
+				break;
+			    default:
+			    bad_SDPcpu:
+				erts_fprintf(stderr,
+					     "bad dirty CPU schedulers percentage specifier %s\n",
+					     arg);
+				erts_usage();
+				break;
+			    }
+			    VERBOSE(DEBUG_SYSTEM,
+				    ("using %d:%d dirty CPU scheduler percentages\n",
+				     dirty_cpu_scheds_pctg, dirty_cpu_scheds_onln_pctg));
+			} else if (strcmp(type, "cpu") == 0) {
+			    int tot, onln;
+			    arg = get_arg(argv[i]+6, argv[i+1], &i);
+			    switch (sscanf(arg, "%d:%d", &tot, &onln)) {
+			    case 0:
+				switch (sscanf(arg, ":%d", &onln)) {
+				case 1:
+				    tot = no_schedulers;
+				    goto chk_SDcpu;
+				default:
+				    goto bad_SDcpu;
+				}
+			    case 1:
+				onln = tot < dirty_cpu_scheds_online ?
+				    tot : dirty_cpu_scheds_online;
+			    case 2:
+			    chk_SDcpu:
+				if (tot > 0)
+				    dirty_cpu_scheds = tot;
+				else
+				    dirty_cpu_scheds = no_schedulers + tot;
+				if (onln > 0)
+				    dirty_cpu_scheds_online = onln;
+				else
+				    dirty_cpu_scheds_online = no_schedulers_online + onln;
+				if (dirty_cpu_scheds < 1 ||
+				    ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS < dirty_cpu_scheds) {
+				    erts_fprintf(stderr,
+						 "bad amount of dirty CPU schedulers %d\n",
+						 tot);
+				    erts_usage();
+				}
+				if (dirty_cpu_scheds_online < 1 ||
+				    dirty_cpu_scheds < dirty_cpu_scheds_online) {
+				    erts_fprintf(stderr,
+						 "bad amount of dirty CPU schedulers online %d "
+						 "(total amount of dirty CPU schedulers %d)\n",
+						 dirty_cpu_scheds_online, dirty_cpu_scheds);
+				    erts_usage();
+				}
+				break;
+			    default:
+			    bad_SDcpu:
+				erts_fprintf(stderr,
+					     "bad amount of dirty CPU schedulers %s\n",
+					     arg);
+				erts_usage();
+				break;
+			    }
+			    VERBOSE(DEBUG_SYSTEM,
+				    ("using %d:%d dirty CPU scheduler(s)\n", tot, onln));
+			} else if (strcmp(type, "io") == 0) {
+			    arg = get_arg(argv[i]+5, argv[i+1], &i);
+			    dirty_io_scheds = atoi(arg);
+			    if (dirty_io_scheds < 0 ||
+				dirty_io_scheds > ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS) {
+				erts_fprintf(stderr,
+					     "bad number of dirty I/O schedulers %s\n",
+					     arg);
+				erts_usage();
+			    }
+			    VERBOSE(DEBUG_SYSTEM,
+				    ("using %d dirty I/O scheduler(s)\n", dirty_io_scheds));
+			} else {
+			    erts_fprintf(stderr,
+					 "bad or missing dirty scheduler specifier: %s\n",
+					 argv[i]);
+			    erts_usage();
+			    break;
+			}
+		    }
+#endif
+		    else {
 			int tot, onln;
 			char *arg = get_arg(argv[i]+2, argv[i+1], &i);
 			switch (sscanf(arg, "%d:%d", &tot, &onln)) {
@@ -894,6 +1030,17 @@ early_init(int *argc, char **argv) /*
 	/* Silence gcc warnings */
 	(void)schdlrs_percentage;
 	(void)schdlrs_onln_percentage;
+#endif
+#ifdef ERTS_DIRTY_SCHEDULERS
+	/* apply any dirty scheduler precentages */
+	if (dirty_cpu_scheds_pctg != 100 || dirty_cpu_scheds_onln_pctg != 100) {
+	    dirty_cpu_scheds = dirty_cpu_scheds * dirty_cpu_scheds_pctg / 100;
+	    dirty_cpu_scheds_online = dirty_cpu_scheds_online * dirty_cpu_scheds_onln_pctg / 100;
+	}
+	if (dirty_cpu_scheds > schdlrs)
+	    dirty_cpu_scheds = schdlrs;
+	if (dirty_cpu_scheds_online > schdlrs_onln)
+	    dirty_cpu_scheds_online = schdlrs_onln;
 #endif
     }
 
@@ -906,6 +1053,11 @@ early_init(int *argc, char **argv) /*
     no_schedulers_online = schdlrs_onln;
 
     erts_no_schedulers = (Uint) no_schedulers;
+#endif
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_no_dirty_cpu_schedulers = dirty_cpu_scheds;
+    erts_no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online;
+    erts_no_dirty_io_schedulers = dirty_io_scheds;
 #endif
     erts_early_init_scheduling(no_schedulers);
 
@@ -924,10 +1076,18 @@ early_init(int *argc, char **argv) /*
      *
      * * Unmanaged threads that need to register:
      * ** Async threads (see erl_async.c)
+     * ** Dirty scheduler threads
      */
     erts_thr_progress_init(no_schedulers,
 			   no_schedulers+2,
-			   erts_async_max_threads);
+#ifndef ERTS_DIRTY_SCHEDULERS
+			   erts_async_max_threads
+#else
+			   erts_async_max_threads +
+			   erts_no_dirty_cpu_schedulers +
+			   erts_no_dirty_io_schedulers
+#endif
+			   );
 #endif
     erts_thr_q_init();
     erts_init_utils();
@@ -1392,7 +1552,15 @@ erl_start(int argc, char **argv)
 	    break;
 
 	case 'S' : /* Was handled in early_init() just read past it */
-	    if (argv[i][2] == 'P')
+	    if (argv[i][2] == 'D') {
+		char* type = argv[i]+3;
+		if (strcmp(type, "Pcpu") == 0)
+		    (void) get_arg(argv[i]+7, argv[i+1], &i);
+		if (strcmp(type, "cpu") == 0)
+		    (void) get_arg(argv[i]+6, argv[i+1], &i);
+		else if (strcmp(type, "io") == 0)
+		    (void) get_arg(argv[i]+5, argv[i+1], &i);
+	    } else if (argv[i][2] == 'P')
 		(void) get_arg(argv[i]+3, argv[i+1], &i);
 	    else
 		(void) get_arg(argv[i]+2, argv[i+1], &i);
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 0dd83fa6ed..a8ff94ac89 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -123,6 +123,9 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"schdlr_sspnd",				NULL			},
     {	"migration_info_update",		NULL			},
     {	"run_queue",				"address"		},
+#ifdef ERTS_DIRTY_SCHEDULERS
+    {   "dirty_run_queue_sleep_list",		"address"		},
+#endif
     {	"process_table",			NULL			},
     {	"cpu_info",				NULL			},
     {	"pollset",				"address"		},
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index dc285b3cf7..e1e213c4eb 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -34,6 +34,7 @@
 #include "beam_bp.h"
 #include "erl_thr_progress.h"
 #include "dtrace-wrapper.h"
+#include "erl_process.h"
 #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP))
 #define HAVE_USE_DTRACE 1
 #endif
@@ -1451,6 +1452,156 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent)
     return ERTS_BIF_REDS_LEFT(env->proc) == 0;
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static void
+alloc_proc_psd(Process* proc, Export **ep)
+{
+    int i;
+    if (!*ep) {
+	*ep = erts_alloc(ERTS_ALC_T_PSD, sizeof(Export));
+	sys_memset((void*) *ep, 0, sizeof(Export));
+	for (i=0; i<ERTS_NUM_CODE_IX; i++) {
+	    (*ep)->addressv[i] = &(*ep)->code[3];
+	}
+	(*ep)->code[3] = (BeamInstr) em_call_nif;
+    }
+    (void) ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, *ep);
+}
+
+static ERL_NIF_TERM
+execute_dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    Eterm* reg = ERTS_PROC_GET_SCHDATA(env->proc)->x_reg_array;
+    ERL_NIF_TERM result = (ERL_NIF_TERM) reg[0];
+    typedef ERL_NIF_TERM (*FinalizerFP)(ErlNifEnv*, ERL_NIF_TERM);
+    FinalizerFP fp;
+#if HAVE_INT64 && SIZEOF_LONG != 8
+    ASSERT(sizeof(fp) <= sizeof(ErlNifUInt64));
+    enif_get_uint64(env, reg[1], (ErlNifUInt64 *) &fp);
+#else
+    ASSERT(sizeof(fp) <= sizeof(unsigned long));
+    enif_get_ulong(env, reg[1], (unsigned long *) &fp);
+#endif
+    return (*fp)(env, result);
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+
+ERL_NIF_TERM
+enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
+			ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]),
+			int argc, const ERL_NIF_TERM argv[])
+{
+#ifdef USE_THREADS
+    erts_aint32_t state, n, a;
+    Process* proc = env->proc;
+    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
+    Export* ep = NULL;
+    int i;
+
+    int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND));
+    if (chkflgs != ERL_NIF_DIRTY_JOB_IO_BOUND && chkflgs != ERL_NIF_DIRTY_JOB_CPU_BOUND)
+	return enif_make_badarg(env);
+
+    a = erts_smp_atomic32_read_acqb(&proc->state);
+    while (1) {
+	n = state = a;
+	if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND)
+	    n |= ERTS_PSFLG_DIRTY_CPU_PROC;
+	else
+	    n |= ERTS_PSFLG_DIRTY_IO_PROC;
+	a = erts_smp_atomic32_cmpxchg_mb(&proc->state, n, state);
+	if (a == state)
+	    break;
+    }
+    if (!(ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
+	alloc_proc_psd(proc, &ep);
+    ERTS_VBUMP_ALL_REDS(proc);
+    ep->code[2] = argc;
+    for (i = 0; i < argc; i++) {
+	reg[i] = (Eterm) argv[i];
+    }
+    proc->i = (BeamInstr*) ep->addressv[0];
+    ep->code[4] = (BeamInstr) fp;
+    proc->freason = TRAP;
+
+    return THE_NON_VALUE;
+#else
+    return (*fp)(env, argc, argv);
+#endif
+}
+
+ERL_NIF_TERM
+enif_schedule_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result,
+				  ERL_NIF_TERM (*fp)(ErlNifEnv*, ERL_NIF_TERM))
+{
+#ifdef USE_THREADS
+    erts_aint32_t state, n, a;
+    Process* proc = env->proc;
+    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
+    Export* ep;
+
+    a = erts_smp_atomic32_read_acqb(&proc->state);
+    while (1) {
+	n = state = a;
+	if (!(n & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)))
+	    break;
+	n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC
+	       |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q);
+	a = erts_smp_atomic32_cmpxchg_mb(&proc->state, n, state);
+	if (a == state)
+	    break;
+    }
+    if (!(ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
+	alloc_proc_psd(proc, &ep);
+    ERTS_VBUMP_ALL_REDS(proc);
+    ep->code[2] = 2;
+    reg[0] = (Eterm) result;
+#if HAVE_INT64 && SIZEOF_LONG != 8
+    ASSERT(sizeof(fp) <= sizeof(ErlNifUInt64));
+    reg[1] = (Eterm) enif_make_uint64(env, (ErlNifUInt64) fp);
+#else
+    ASSERT(sizeof(fp) <= sizeof(unsigned long));
+    reg[1] = (Eterm) enif_make_ulong(env, (unsigned long) fp);
+#endif
+    proc->i = (BeamInstr*) ep->addressv[0];
+    ep->code[4] = (BeamInstr) execute_dirty_nif_finalizer;
+    proc->freason = TRAP;
+
+    return THE_NON_VALUE;
+#else
+    return (*fp)(env, result);
+#endif
+}
+
+/* A simple finalizer that just returns its result argument */
+ERL_NIF_TERM
+enif_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result)
+{
+    return result;
+}
+
+int
+enif_is_on_dirty_scheduler(ErlNifEnv* env)
+{
+    return ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data);
+}
+
+int
+enif_have_dirty_schedulers()
+{
+#ifdef USE_THREADS
+    return 1;
+#else
+    return 0;
+#endif
+}
+
+#endif /* ERL_NIF_DIRTY_SCHEDULER_SUPPORT */
+
 /***************************************************************************
  **                              load_nif/2                               **
  ***************************************************************************/
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 5f4dc21d5c..fb3c359ec9 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -23,7 +23,11 @@
 #ifndef __ERL_NIF_H__
 #define __ERL_NIF_H__
 
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 
+#include "erl_native_features_config.h"
 #include "erl_drv_nif.h"
 
 /* Version history:
@@ -34,9 +38,14 @@
 ** 2.2: R14B03 enif_is_exception
 ** 2.3: R15 enif_make_reverse_list, enif_is_number
 ** 2.4: R16 enif_consume_timeslice
+** 2.5: R17 dirty schedulers
 */
 #define ERL_NIF_MAJOR_VERSION 2
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+#define ERL_NIF_MINOR_VERSION 5
+#else
 #define ERL_NIF_MINOR_VERSION 4
+#endif
 
 #include <stdlib.h>
 
@@ -159,6 +168,14 @@ typedef int ErlNifTSDKey;
 
 typedef ErlDrvThreadOpts ErlNifThreadOpts;
 
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+typedef enum
+{
+    ERL_NIF_DIRTY_JOB_CPU_BOUND = 1,
+    ERL_NIF_DIRTY_JOB_IO_BOUND  = 2
+}ErlNifDirtyTaskFlags;
+#endif
+
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
 typedef struct {
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 2f841645e1..f5b27dfdfa 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -141,6 +141,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg));
 ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg));
 ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent));
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_dirty_nif,(ErlNifEnv*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_dirty_nif_finalizer,(ErlNifEnv*,ERL_NIF_TERM,ERL_NIF_TERM (*)(ErlNifEnv*,ERL_NIF_TERM)));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_dirty_nif_finalizer,(ErlNifEnv*,ERL_NIF_TERM));
+ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
+ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
+#endif
 
 /*
 ** Add new entries here to keep compatibility on Windows!!!
@@ -266,6 +273,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent));
 #  define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen)
 #  define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym)
 #  define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice)
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+#  define enif_schedule_dirty_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_dirty_nif)
+#  define enif_schedule_dirty_nif_finalizer ERL_NIF_API_FUNC_MACRO(enif_schedule_dirty_nif_finalizer)
+#  define enif_dirty_nif_finalizer ERL_NIF_API_FUNC_MACRO(enif_dirty_nif_finalizer)
+#  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
+#  define enif_have_dirty_schedulers ERL_NIF_API_FUNC_MACRO(enif_have_dirty_schedulers)
+#endif
 
 /*
 ** Add new entries here
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 74cd84a998..937881212a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -146,6 +146,11 @@ extern BeamInstr beam_continue_exit[];
 int erts_sched_compact_load;
 int erts_sched_balance_util = 0;
 Uint erts_no_schedulers;
+#ifdef ERTS_DIRTY_SCHEDULERS
+Uint erts_no_dirty_cpu_schedulers;
+Uint erts_no_dirty_cpu_schedulers_online;
+Uint erts_no_dirty_io_schedulers;
+#endif
 
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY		(4*1024*1024)
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY			(512*1024)
@@ -259,6 +264,10 @@ ErtsAlignedRunQueue *erts_aligned_run_queues;
 Uint erts_no_run_queues;
 
 ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
+#ifdef ERTS_DIRTY_SCHEDULERS
+ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
+ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
+#endif
 
 typedef union {
     ErtsSchedulerSleepInfo ssi;
@@ -266,6 +275,12 @@ typedef union {
 } ErtsAlignedSchedulerSleepInfo;
 
 static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
+#ifdef ERTS_DIRTY_SCHEDULERS
+#ifdef ERTS_SMP
+static ErtsAlignedSchedulerSleepInfo *aligned_dirty_cpu_sched_sleep_info;
+static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info;
+#endif
+#endif
 
 static Uint last_reductions;
 static Uint last_exact_reductions;
@@ -332,6 +347,16 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
     (ASSERT(-1 <= ((int) (IX))					        \
 		 && ((int) (IX)) < ((int) erts_no_schedulers)),		\
      &aligned_sched_sleep_info[(IX)].ssi)
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX)				\
+    (ASSERT(0 <= ((int) (IX))					        \
+	    && ((int) (IX)) < ((int) erts_no_dirty_cpu_schedulers)),	\
+     &aligned_dirty_cpu_sched_sleep_info[(IX)].ssi)
+#define ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(IX)				\
+    (ASSERT(0 <= ((int) (IX))					        \
+	    && ((int) (IX)) < ((int) erts_no_dirty_io_schedulers)),	\
+     &aligned_dirty_io_sched_sleep_info[(IX)].ssi)
+#endif
 
 #define ERTS_FOREACH_RUNQ(RQVAR, DO)					\
 do {									\
@@ -519,6 +544,13 @@ erts_pre_init_process(void)
      erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks
 	 = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+     erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].get_locks
+	 = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_GET_LOCKS;
+     erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].set_locks
+	 = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_SET_LOCKS;
+#endif
+
      /* Check that we have locks for all entries */
      for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
 	 ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks);
@@ -931,7 +963,9 @@ reply_sched_wall_time(void *vswtrp)
     ErlHeapFragment *bp = NULL;
 
     ASSERT(esdp);
-    
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
     if (swtrp->set) {
 	if (!swtrp->enable && esdp->sched_wall_time.enabled) {
 	    esdp->sched_wall_time.need = erts_sched_balance_util;
@@ -1015,6 +1049,9 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable)
 
     if (!set && !esdp->sched_wall_time.enabled)
 	return THE_NON_VALUE;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
 
     swtrp = swtreq_alloc();
     ref = erts_make_ref(c_p);
@@ -1492,6 +1529,9 @@ erts_schedule_multi_misc_aux_work(int ignore_self,
 
     if (ignore_self) {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+#ifdef ERTS_DIRTY_SCHEDULERS
+	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
 	if (esdp)
 	    self = (int) esdp->no;
     }
@@ -1601,7 +1641,7 @@ void
 erts_alloc_notify_delayed_dealloc(int ix)
 {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    if (esdp)
+    if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp))
 	schedule_aux_work_wakeup(&esdp->aux_work_data,
 				 ix,
 				 ERTS_SSI_AUX_WORK_DD);
@@ -1615,6 +1655,10 @@ erts_alloc_ensure_handle_delayed_dealloc_call(int ix)
 {
 #ifdef DEBUG
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
+	return;
+#endif
     ASSERT(!esdp || ix == (int) esdp->no);
 #endif
     set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1),
@@ -2220,6 +2264,9 @@ static ERTS_INLINE void
 sched_active_sys(Uint no, ErtsRunQueue *rq)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+#endif
     ASSERT(rq->waiting < 0);
     rq->waiting *= -1;
     rq->waiting--;
@@ -2276,6 +2323,9 @@ static ERTS_INLINE void
 sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+#endif
     ASSERT(rq->waiting < 0);
     rq->waiting *= -1;
 }
@@ -2291,7 +2341,7 @@ sched_waiting(Uint no, ErtsRunQueue *rq)
     else
 	rq->waiting++;
     rq->woken = 0;
-    if (erts_system_profile_flags.scheduler)
+    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && erts_system_profile_flags.scheduler)
 	profile_scheduler(make_small(no), am_inactive);
 }
 
@@ -2303,7 +2353,7 @@ sched_active(Uint no, ErtsRunQueue *rq)
 	rq->waiting++;
     else
 	rq->waiting--;
-    if (erts_system_profile_flags.scheduler)
+    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && erts_system_profile_flags.scheduler)
 	profile_scheduler(make_small(no), am_active);
 }
 
@@ -2317,7 +2367,7 @@ ongoing_multi_scheduling_block(void)
 static ERTS_INLINE void
 empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags)
 {
-    if (old_flags & ERTS_RUNQ_FLG_NONEMPTY) {
+    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && old_flags & ERTS_RUNQ_FLG_NONEMPTY) {
 #ifdef DEBUG
 	erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
 	/*
@@ -2357,7 +2407,7 @@ static ERTS_INLINE void
 non_empty_runq(ErtsRunQueue *rq)
 {
     Uint32 old_flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_NONEMPTY);
-    if (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY)) {
+    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY))) {
 #ifdef DEBUG
 	erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
 	/*
@@ -2602,18 +2652,37 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+	erts_smp_spin_lock(&rq->sleepers.lock);
+#endif
     flgs = sched_prep_spin_wait(ssi);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 	/* Go suspend instead... */
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+	    erts_smp_spin_unlock(&rq->sleepers.lock);
+#endif
 	return;
     }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
+	ssi->prev = NULL;
+	ssi->next = rq->sleepers.list;
+	if (rq->sleepers.list)
+	    rq->sleepers.list->prev = ssi;
+	rq->sleepers.list = ssi;
+	erts_smp_spin_unlock(&rq->sleepers.lock);
+    }
+#endif
+
     /*
      * If all schedulers are waiting, one of them *should*
      * be waiting in erl_sys_schedule()
      */
 
-    if (!prepare_for_sys_schedule()) {
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule()) {
 
 	sched_waiting(esdp->no, rq);
 
@@ -2623,12 +2692,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     tse_wait:
 
-	if (thr_prgr_active != working)
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working)
 	    sched_wall_time_change(esdp, thr_prgr_active);
 
 	while (1) {
 
-	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+	    aux_work = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
+		erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
 		if (!thr_prgr_active) {
 		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -2642,11 +2712,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    if (aux_work)
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 	    else {
-		if (thr_prgr_active) {
-		    erts_thr_progress_active(esdp, thr_prgr_active = 0);
-		    sched_wall_time_change(esdp, 0);
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		    if (thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			sched_wall_time_change(esdp, 0);
+		    }
+		    erts_thr_progress_prepare_wait(esdp);
 		}
-		erts_thr_progress_prepare_wait(esdp);
 
 		flgs = sched_spin_wait(ssi, spincount);
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
@@ -2661,7 +2733,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 			} while (res == EINTR);
 		    }
 		}
-		erts_thr_progress_finalize_wait(esdp);
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+		    erts_thr_progress_finalize_wait(esdp);
 	    }
 
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
@@ -2682,7 +2755,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
 	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
 
-	if (!thr_prgr_active) {
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 	    sched_wall_time_change(esdp, 1);
 	}
@@ -2699,6 +2772,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	erts_smp_atomic32_set_relb(&function_calls, 0);
 	*fcalls = 0;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
 	sched_waiting_sys(esdp->no, rq);
 
 
@@ -2912,6 +2988,34 @@ wake_scheduler(ErtsRunQueue *rq)
     erts_sched_finish_poke(ssi, flgs);
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+static void
+wake_dirty_scheduler(ErtsRunQueue *rq)
+{
+    ErtsSchedulerSleepInfo *ssi;
+    ErtsSchedulerSleepList *sl;
+
+    ASSERT(ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+
+    sl = &rq->sleepers;
+    erts_smp_spin_lock(&sl->lock);
+    ssi = sl->list;
+    if (!ssi)
+	erts_smp_spin_unlock(&sl->lock);
+    else {
+	sl->list = NULL;
+	erts_smp_spin_unlock(&sl->lock);
+
+	ERTS_THR_MEMORY_BARRIER;
+	do {
+	    ErtsSchedulerSleepInfo *wake_ssi = ssi;
+	    ssi = ssi->next;
+	    erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi));
+	} while (ssi);
+    }
+}
+#endif
+
 #define ERTS_NO_USED_RUNQS_SHIFT 16
 #define ERTS_NO_RUNQS_MASK 0xffff
 
@@ -3047,8 +3151,14 @@ static ERTS_INLINE void
 smp_notify_inc_runq(ErtsRunQueue *runq)
 {
 #ifdef ERTS_SMP
-    if (runq)
-	wake_scheduler(runq);
+    if (runq) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
+	    wake_dirty_scheduler(runq);
+	else
+#endif
+	    wake_scheduler(runq);
+    }
 #endif
 }
 
@@ -4899,7 +5009,10 @@ erts_sched_set_wake_cleanup_threshold(char *str)
 static void
 init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 {
-    awdp->sched_id = esdp ? (int) esdp->no : 0;
+    if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp))
+	awdp->sched_id = 0;
+    else
+	awdp->sched_id = (int) esdp->no;
     awdp->esdp = esdp;
     awdp->ssi = esdp ? esdp->ssi : NULL;
 #ifdef ERTS_SMP
@@ -4939,14 +5052,71 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 #endif
 }
 
+static void
+init_scheduler_data(ErtsSchedulerData* esdp, int num,
+		    ErtsSchedulerSleepInfo* ssi,
+		    ErtsRunQueue* runq,
+		    char** daww_ptr, size_t daww_sz)
+{
+#ifdef ERTS_SMP
+    erts_bits_init_state(&esdp->erl_bits_state);
+    esdp->match_pseudo_process = NULL;
+    esdp->free_process = NULL;
+#endif
+    esdp->x_reg_array =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
+					   ERTS_X_REGS_ALLOCATED *
+					   sizeof(Eterm));
+    esdp->f_reg_array =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
+					   MAX_REG * sizeof(FloatDef));
+#if !HEAP_ON_C_STACK
+    esdp->num_tmp_heap_used = 0;
+#endif
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
+	esdp->no = 0;
+	esdp->dirty_no = (Uint) num;
+    }
+    else {
+	esdp->no = (Uint) num;
+	esdp->dirty_no = 0;
+    }
+#else
+    esdp->no = (Uint) num;
+#endif
+    esdp->ssi = ssi;
+    esdp->current_process = NULL;
+    esdp->current_port = NULL;
+
+    esdp->virtual_reds = 0;
+    esdp->cpu_id = -1;
+
+    erts_init_atom_cache_map(&esdp->atom_cache_map);
+
+    esdp->run_queue = runq;
+    esdp->run_queue->scheduler = esdp;
+
+    if (daww_ptr) {
+	init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr);
+#ifdef ERTS_SMP
+	*daww_ptr += daww_sz;
+#endif
+    }
+
+    esdp->reductions = 0;
+
+    init_sched_wall_time(&esdp->sched_wall_time);
+    erts_port_task_handle_init(&esdp->nosuspend_port_task_handle);
+}
+
 void
 erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 {
     int ix, n, no_ssi;
     char *daww_ptr;
-#ifdef ERTS_SMP
     size_t daww_sz;
-#endif
+    size_t size_runqs;
 
     init_misc_op_list_alloc();
     init_proc_sys_task_queues_alloc();
@@ -4967,19 +5137,26 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     /* Create and initialize run queues */
 
     n = no_schedulers;
-
-    erts_aligned_run_queues = 
-	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS,
-					   sizeof(ErtsAlignedRunQueue) * n);
+    size_runqs = sizeof(ErtsAlignedRunQueue) * (n + ERTS_NUM_DIRTY_RUNQS);
+    erts_aligned_run_queues =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs);
 #ifdef ERTS_SMP
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_aligned_run_queues += ERTS_NUM_DIRTY_RUNQS;
+#endif
     erts_smp_atomic32_init_nob(&no_empty_run_queues, 0);
 #endif
 
     erts_no_run_queues = n;
 
-    for (ix = 0; ix < n; ix++) {
+    for (ix = -(ERTS_NUM_DIRTY_RUNQS); ix < n; ix++) {
 	int pix, rix;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	ErtsRunQueue *rq = ERTS_RUNQ_IX_IS_DIRTY(ix) ?
+	    ERTS_DIRTY_RUNQ_IX(ix) : ERTS_RUNQ_IX(ix);
+#else
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+#endif
 
 	rq->ix = ix;
 
@@ -4990,6 +5167,15 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 	erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1));
 	erts_smp_cnd_init(&rq->cnd);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#ifdef ERTS_SMP
+	if (ERTS_RUNQ_IX_IS_DIRTY(ix)) {
+	    erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list");
+	    rq->sleepers.list = NULL;
+	}
+#endif
+#endif
+
 	rq->waiting = 0;
 	rq->woken = 0;
 	ERTS_RUNQ_FLGS_INIT(rq, ERTS_RUNQ_FLG_NONEMPTY);
@@ -5076,6 +5262,29 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 
 #ifdef ERTS_SMP
     aligned_sched_sleep_info++;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    aligned_dirty_cpu_sched_sleep_info =
+	erts_alloc_permanent_cache_aligned(
+	    ERTS_ALC_T_SCHDLR_SLP_INFO,
+	    erts_no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+	ErtsSchedulerSleepInfo *ssi = &aligned_dirty_cpu_sched_sleep_info[ix].ssi;
+	erts_smp_atomic32_init_nob(&ssi->flags, 0);
+	ssi->event = NULL; /* initialized in sched_thread_func */
+	erts_atomic32_init_nob(&ssi->aux_work, 0);
+    }
+    aligned_dirty_io_sched_sleep_info =
+	erts_alloc_permanent_cache_aligned(
+	    ERTS_ALC_T_SCHDLR_SLP_INFO,
+	    erts_no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+	ErtsSchedulerSleepInfo *ssi = &aligned_dirty_io_sched_sleep_info[ix].ssi;
+	erts_smp_atomic32_init_nob(&ssi->flags, 0);
+	ssi->event = NULL; /* initialized in sched_thread_func */
+	erts_atomic32_init_nob(&ssi->aux_work, 0);
+    }
+#endif
 #endif
 
     /* Create and initialize scheduler specific data */
@@ -5086,6 +5295,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 						  daww_sz*n);
 #else
+    daww_sz = 0;
     daww_ptr = NULL;
 #endif
 
@@ -5095,46 +5305,32 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 
     for (ix = 0; ix < n; ix++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
-#ifdef ERTS_SMP
-	erts_bits_init_state(&esdp->erl_bits_state);
-	esdp->match_pseudo_process = NULL;
-	esdp->free_process = NULL;
-#endif
-	esdp->x_reg_array =
-	    erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
-					       ERTS_X_REGS_ALLOCATED *
-					       sizeof(Eterm));
-	esdp->f_reg_array =
-	    erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
-					       MAX_REG * sizeof(FloatDef));
-#if !HEAP_ON_C_STACK
-	esdp->num_tmp_heap_used = 0;
-#endif
-	esdp->no = (Uint) ix+1;
-	esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
-	esdp->current_process = NULL;
-	esdp->current_port = NULL;
-
-	esdp->virtual_reds = 0;
-	esdp->cpu_id = -1;
-
-	erts_init_atom_cache_map(&esdp->atom_cache_map);
-
-	esdp->run_queue = ERTS_RUNQ_IX(ix);
-	esdp->run_queue->scheduler = esdp;
+	init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix),
+			    ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz);
+    }
 
-	init_aux_work_data(&esdp->aux_work_data, esdp, daww_ptr);
+#ifdef ERTS_DIRTY_SCHEDULERS
 #ifdef ERTS_SMP
-	daww_ptr += daww_sz;
-#endif
-
-	esdp->reductions = 0;
-
-	init_sched_wall_time(&esdp->sched_wall_time);
-
-	erts_port_task_handle_init(&esdp->nosuspend_port_task_handle);
-
+    erts_aligned_dirty_cpu_scheduler_data =
+	erts_alloc_permanent_cache_aligned(
+					   ERTS_ALC_T_SCHDLR_DATA,
+					   erts_no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerData));
+    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+	ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+	init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix),
+			    ERTS_DIRTY_CPU_RUNQ, NULL, 0);
+    }
+    erts_aligned_dirty_io_scheduler_data =
+	erts_alloc_permanent_cache_aligned(
+					   ERTS_ALC_T_SCHDLR_DATA,
+					   erts_no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerData));
+    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+	ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
+	init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
+			    ERTS_DIRTY_IO_RUNQ, NULL, 0);
     }
+#endif
+#endif
 
     init_misc_aux_work();
 #if !HALFWORD_HEAP
@@ -5198,6 +5394,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 #endif
     }
     erts_no_schedulers = 1;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_no_dirty_cpu_schedulers = 0;
+    erts_no_dirty_io_schedulers = 0;
+#endif
 #endif
 
     erts_smp_atomic32_init_nob(&function_calls, 0);
@@ -5310,28 +5510,39 @@ check_enqueue_in_prio_queue(erts_aint32_t *prq_prio_p,
 
     *prq_prio_p = aprio;
 
-    max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK;
-    max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
-    max_qbit &= -max_qbit;
-    /*
-     * max_qbit now either contain bit set for highest prio queue or a bit
-     * out of range (which will have a value larger than valid range).
-     */
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!(actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))) {
+#endif
+	max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK;
+	max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
+	max_qbit &= -max_qbit;
+	/*
+	 * max_qbit now either contain bit set for highest prio queue or a bit
+	 * out of range (which will have a value larger than valid range).
+	 */
 
-    if (qbit >= max_qbit)
-	return 0; /* Already queued in higher or equal prio */
+	if (qbit >= max_qbit)
+	    return 0; /* Already queued in higher or equal prio */
 
-    /* Need to enqueue (if already enqueued, it is in lower prio) */
-    *newp |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET;
+	/* Need to enqueue (if already enqueued, it is in lower prio) */
+	*newp |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET;
 
-    if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK))
-	!= (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) {
-	/*
-	 * Process struct already enqueued, or actual prio not
-	 * equal to user prio, i.e., enqueue using proxy.
-	 */
-	return -1;
+	if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK))
+	    != (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) {
+	    /*
+	     * Process struct already enqueued, or actual prio not
+	     * equal to user prio, i.e., enqueue using proxy.
+	     */
+	    return -1;
+	}
+#ifdef ERTS_DIRTY_SCHEDULERS
+    } else {
+	if (actual & ERTS_PSFLG_DIRTY_CPU_PROC)
+	    *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q;
+	else
+	    *newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q;
     }
+#endif
 
     /*
      * Enqueue using process struct.
@@ -5387,10 +5598,21 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
     }
     else {
 	Process *sched_p;
-	ErtsRunQueue *runq = erts_get_runq_proc(p);
+	ErtsRunQueue *runq;
 
 	ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS));
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#ifdef ERTS_SMP
+	if (ERTS_PSFLG_DIRTY_CPU_PROC & a)
+	    runq = ERTS_DIRTY_CPU_RUNQ;
+	else if (ERTS_PSFLG_DIRTY_IO_PROC & a)
+	    runq = ERTS_DIRTY_IO_RUNQ;
+	else
+#endif
+#endif
+	    runq = erts_get_runq_proc(p);
+
 	if (enqueue < 0)
 	    sched_p = make_proxy_proc(proxy, p, enq_prio);
 	else {
@@ -5400,7 +5622,11 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 	}
 
 #ifdef ERTS_SMP
-	if (!(ERTS_PSFLG_BOUND & n)) {
+	if (!(ERTS_PSFLG_BOUND & n)
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    && !(n & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))
+#endif
+	    ) {
 	    ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
 	    if (new_runq) {
 		RUNQ_SET_RQ(&sched_p->run_queue, new_runq);
@@ -5827,6 +6053,9 @@ suspend_scheduler(ErtsSchedulerData *esdp)
      * Regardless of why a scheduler is suspended, it ends up here.
      */
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
     ASSERT(no != 1);
 
     evacuate_run_queue(esdp->run_queue, &sbp);
@@ -5994,22 +6223,44 @@ ErtsSchedSuspendResult
 erts_schedulers_state(Uint *total,
 		      Uint *online,
 		      Uint *active,
+		      Uint *dirty_cpu,
+		      Uint *dirty_cpu_online,
+		      Uint *dirty_io,
 		      int yield_allowed)
 {
-    int res;
+    int res = ERTS_SCHDLR_SSPND_EINVAL;
     erts_aint32_t changing;
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-    if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
-	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
-    else {
-	*active = *online = schdlr_sspnd.online;
-	if (ongoing_multi_scheduling_block())
-	    *active = 1;
-	res = ERTS_SCHDLR_SSPND_DONE;
+    if (total) {
+	ASSERT(online);
+	ASSERT(active);
+	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
+	    res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
+	else {
+	    *active = *online = schdlr_sspnd.online;
+	    if (ongoing_multi_scheduling_block())
+		*active = 1;
+	    res = ERTS_SCHDLR_SSPND_DONE;
+	}
+	erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+	*total = erts_no_schedulers;
     }
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-    *total = erts_no_schedulers;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (dirty_cpu)
+	*dirty_cpu = erts_no_dirty_cpu_schedulers;
+    if (dirty_cpu_online)
+	*dirty_cpu_online = erts_no_dirty_cpu_schedulers_online;
+    if (dirty_io)
+	*dirty_io = erts_no_dirty_io_schedulers;
+#else
+    if (dirty_cpu)
+	*dirty_cpu = 0;
+    if (dirty_cpu_online)
+	*dirty_cpu_online = 0;
+    if (dirty_io)
+	*dirty_io = 0;
+#endif
     return res;
 }
 
@@ -6101,6 +6352,10 @@ erts_set_schedulers_online(Process *p,
 			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 			wake_scheduler(rq);
 		    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    wake_dirty_scheduler(ERTS_DIRTY_CPU_RUNQ);
+		    wake_dirty_scheduler(ERTS_DIRTY_IO_RUNQ);
+#endif
 		}
 	    }
 
@@ -6448,6 +6703,92 @@ sched_thread_func(void *vesdp)
     return NULL;
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#ifdef ERTS_SMP
+static void*
+sched_dirty_cpu_thread_func(void *vesdp)
+{
+    ErtsThrPrgrCallbacks callbacks;
+    ErtsSchedulerData *esdp = vesdp;
+    Uint no = esdp->dirty_no;
+    ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch();
+    callbacks.arg = (void *) esdp->ssi;
+    callbacks.wakeup = thr_prgr_wakeup;
+    callbacks.prepare_wait = NULL;
+    callbacks.wait = NULL;
+    callbacks.finalize_wait = NULL;
+
+    erts_thr_progress_register_unmanaged_thread(&callbacks);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    {
+	char buf[31];
+	erts_snprintf(&buf[0], 31, "dirty cpu scheduler %beu", no);
+	erts_lc_set_thread_name(&buf[0]);
+    }
+#endif
+    erts_tsd_set(sched_data_key, vesdp);
+#if ERTS_USE_ASYNC_READY_Q
+    esdp->aux_work_data.async_ready.queue = NULL;
+#endif
+
+    erts_proc_lock_prepare_proc_lock_waiter();
+
+#ifdef HIPE
+    hipe_thread_signal_init();
+#endif
+    erts_thread_init_float();
+
+    process_main();
+    /* No schedulers should *ever* terminate */
+    erl_exit(ERTS_ABORT_EXIT,
+	     "Dirty CPU scheduler thread number %beu terminated\n",
+	     no);
+    return NULL;
+}
+
+static void*
+sched_dirty_io_thread_func(void *vesdp)
+{
+    ErtsThrPrgrCallbacks callbacks;
+    ErtsSchedulerData *esdp = vesdp;
+    Uint no = esdp->dirty_no;
+    ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch();
+    callbacks.arg = (void *) esdp->ssi;
+    callbacks.wakeup = thr_prgr_wakeup;
+    callbacks.prepare_wait = NULL;
+    callbacks.wait = NULL;
+    callbacks.finalize_wait = NULL;
+
+    erts_thr_progress_register_unmanaged_thread(&callbacks);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    {
+	char buf[31];
+	erts_snprintf(&buf[0], 31, "dirty io scheduler %beu", no);
+	erts_lc_set_thread_name(&buf[0]);
+    }
+#endif
+    erts_tsd_set(sched_data_key, vesdp);
+#if ERTS_USE_ASYNC_READY_Q
+    esdp->aux_work_data.async_ready.queue = NULL;
+#endif
+
+    erts_proc_lock_prepare_proc_lock_waiter();
+
+#ifdef HIPE
+    hipe_thread_signal_init();
+#endif
+    erts_thread_init_float();
+
+    process_main();
+    /* No schedulers should *ever* terminate */
+    erl_exit(ERTS_ABORT_EXIT,
+	     "Dirty I/O scheduler thread number %beu terminated\n",
+	     no);
+    return NULL;
+}
+#endif
+#endif
+
 static ethr_tid aux_tid;
 
 void
@@ -6498,6 +6839,26 @@ erts_start_schedulers(void)
     
     erts_no_schedulers = actual;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#ifdef ERTS_SMP
+    {
+	int ix;
+	for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+	    res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts);
+	    if (res != 0)
+		erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix);
+	}
+	for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
+	    res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts);
+	    if (res != 0)
+		erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix);
+	}
+    }
+#endif
+#endif
+
     ERTS_THR_MEMORY_BARRIER;
 
     res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
@@ -7486,7 +7847,8 @@ Process *schedule(Process *p, int calls)
 	input_reductions = INPUT_REDUCTIONS;
     }
 
-    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+    ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())
+		       || !erts_thr_progress_is_blocking());
 
     /*
      * Clean up after the process being scheduled out.
@@ -7597,7 +7959,8 @@ Process *schedule(Process *p, int calls)
 
     }
 
-    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+    ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)
+		       || !erts_thr_progress_is_blocking());
 
  check_activities_to_run: {
 #ifdef ERTS_SMP
@@ -7611,22 +7974,25 @@ Process *schedule(Process *p, int calls)
 	    erts_smp_runq_lock(rq);
 	}
 
-	if (rq->check_balance_reds <= 0)
-	    check_balance(rq);
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	    if (rq->check_balance_reds <= 0)
+		check_balance(rq);
 
-	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
-	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+	    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
 
-	mps = erts_get_migration_paths_managed();
-	mp = &mps->mpath[rq->ix];
+	    mps = erts_get_migration_paths_managed();
+	    mp = &mps->mpath[rq->ix];
 
-	if (mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK)
-	    immigrate(rq, mp);
+	    if (mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK)
+		immigrate(rq, mp);
+	}
 
+	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
     continue_check_activities_to_run:
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
     continue_check_activities_to_run_known_flags:
-	ASSERT(flags & ERTS_RUNQ_FLG_NONEMPTY);
+	ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)
+	       || flags & ERTS_RUNQ_FLG_NONEMPTY);
 
 	if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
 	
@@ -7641,7 +8007,7 @@ Process *schedule(Process *p, int calls)
 	    }
 	}
 
-	{
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    erts_aint32_t aux_work;
 	    int leader_update = erts_thr_progress_update(esdp);
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
@@ -7653,9 +8019,9 @@ Process *schedule(Process *p, int calls)
 		    handle_aux_work(&esdp->aux_work_data, aux_work, 0);
 		erts_smp_runq_lock(rq);
 	    }
-	}
 
-	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+	    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+	}
 	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
 #else /* ERTS_SMP */
@@ -7684,7 +8050,7 @@ Process *schedule(Process *p, int calls)
 	    if (flags & ERTS_RUNQ_FLG_INACTIVE)
 		empty_runq(rq);
 	    else {
-		if (try_steal_task(rq))
+		if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && try_steal_task(rq))
 		    goto continue_check_activities_to_run;
 
 		empty_runq(rq);
@@ -7710,7 +8076,8 @@ Process *schedule(Process *p, int calls)
 
 	    goto check_activities_to_run;
 	}
-	else if (fcalls > input_reductions && prepare_for_sys_schedule()) {
+	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
+		 (fcalls > input_reductions && prepare_for_sys_schedule())) {
 	    /*
 	     * Schedule system-level activities.
 	     */
@@ -7814,6 +8181,14 @@ Process *schedule(Process *p, int calls)
 	    psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state)
 							+ ERTS_PSFLGS_IN_PRQ_MASK_OFFSET));
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    /* if a non-dirty scheduler picks up a process marked as already being
+	       in a dirty run queue, just drop it and go get another process */
+	    if (state & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) &&
+		!ERTS_SCHEDULER_IS_DIRTY(esdp))
+		goto pick_next_process;
+#endif
+
 	    if (!(state & ERTS_PSFLG_PROXY))
 		psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ;
 	    else {
@@ -7896,6 +8271,10 @@ Process *schedule(Process *p, int calls)
 					  (UWord) esdp->no);
 	    int migrated = old && old != esdp->no;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
+
 	    prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state);
 
 	    erts_smp_spin_lock(&erts_sched_stat.lock);
@@ -9706,8 +10085,12 @@ save_pending_exiter(Process *p)
     non_empty_runq(rq);
 
     erts_smp_runq_unlock(rq);
-
-    wake_scheduler(rq);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+	wake_dirty_scheduler(rq);
+    else
+#endif
+	wake_scheduler(rq);
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 6155f99b85..a0519dc0d9 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -74,6 +74,10 @@ struct ErtsNodesMonitor_;
 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0
 
 #define ERTS_MAX_NO_OF_SCHEDULERS 1024
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
+#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
+#endif
 
 #define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)
 
@@ -103,6 +107,11 @@ extern Export exp_send, exp_receive, exp_timeout;
 extern int erts_sched_compact_load;
 extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
+#ifdef ERTS_DIRTY_SCHEDULERS
+extern Uint erts_no_dirty_cpu_schedulers;
+extern Uint erts_no_dirty_cpu_schedulers_online;
+extern Uint erts_no_dirty_io_schedulers;
+#endif
 extern Uint erts_no_run_queues;
 extern int erts_sched_thread_suggested_stack_size;
 #define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
@@ -275,6 +284,13 @@ typedef enum {
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+typedef struct {
+    erts_smp_spinlock_t lock;
+    ErtsSchedulerSleepInfo *list;
+} ErtsSchedulerSleepList;
+#endif
+
 struct ErtsSchedulerSleepInfo_ {
 #ifdef ERTS_SMP
     ErtsSchedulerSleepInfo *next;
@@ -387,6 +403,12 @@ struct ErtsRunQueue_ {
     erts_smp_mtx_t mtx;
     erts_smp_cnd_t cnd;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#ifdef ERTS_SMP
+    ErtsSchedulerSleepList sleepers;
+#endif
+#endif
+
     ErtsSchedulerData *scheduler;
     int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
     int woken;
@@ -547,7 +569,10 @@ struct ErtsSchedulerData_ {
 #endif
     ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
-    Uint no;			/* Scheduler number */
+    Uint no;			/* Scheduler number for normal schedulers */
+#ifdef ERTS_DIRTY_SCHEDULERS
+    Uint dirty_no;		/* Scheduler number for dirty schedulers */
+#endif
     Port *current_port;
     ErtsRunQueue *run_queue;
     int virtual_reds;
@@ -574,6 +599,10 @@ typedef union {
 } ErtsAlignedSchedulerData;
 
 extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
+#ifdef ERTS_DIRTY_SCHEDULERS
+extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
+extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
+#endif
 
 #ifndef ERTS_SMP
 extern ErtsSchedulerData *erts_scheduler_data;
@@ -685,8 +714,13 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_DIST_ENTRY			3
 #define ERTS_PSD_CALL_TIME_BP			4
 #define ERTS_PSD_DELAYED_GC_TASK_QS		5
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT	6
 
+#define ERTS_PSD_SIZE				7
+#else
 #define ERTS_PSD_SIZE				6
+#endif
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -713,6 +747,11 @@ typedef struct {
 #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
+#endif
+
 typedef struct {
     ErtsProcLocks get_locks;
     ErtsProcLocks set_locks;
@@ -1026,6 +1065,12 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
+#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
+#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
+#endif
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
 					 | ERTS_PSFLG_IN_PRQ_HIGH	\
@@ -1231,12 +1276,46 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 	    (p)->flags &= ~F_TIMO; \
     } while (0)
 
+#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
+#define ERTS_NUM_DIRTY_RUNQS 2
+#else
+#define ERTS_NUM_DIRTY_RUNQS 0
+#endif
+
 #define ERTS_RUNQ_IX(IX)						\
   (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
    &erts_aligned_run_queues[(IX)].runq)
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
+  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
+#define ERTS_DIRTY_RUNQ_IX(IX)						\
+  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
+   &erts_aligned_run_queues[(IX)].runq)
+#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
+#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
+#else
+#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
+#endif
 #define ERTS_SCHEDULER_IX(IX)						\
   (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
    &erts_aligned_scheduler_data[(IX)].esd)
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
+  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
+   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
+#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
+  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
+   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
+#ifdef ERTS_SMP
+#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
+  ((ESDP)->dirty_no != 0)
+#else
+#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
+#endif
+#else
+#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
+#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
+#endif
 
 void erts_pre_init_process(void);
 void erts_late_init_process(void);
@@ -1439,9 +1518,11 @@ int erts_dbg_check_halloc_lock(Process *p);
 void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
 #endif
 int erts_get_max_no_executing_schedulers(void);
-#ifdef ERTS_SMP
+#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
 ErtsSchedSuspendResult
-erts_schedulers_state(Uint *, Uint *, Uint *, int);
+erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
+#endif
+#ifdef ERTS_SMP
 ErtsSchedSuspendResult
 erts_set_schedulers_online(Process *p,
 			   ErtsProcLocks plocks,
@@ -1559,7 +1640,7 @@ do {										\
     ErtsSchedulerData *esdp__ = ((P)						\
 				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
 				 : erts_get_scheduler_data());			\
-    if (esdp__)									\
+    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
 	esdp__->verify_unused_temp_alloc(					\
 	    esdp__->verify_unused_temp_alloc_data);				\
 } while (0)
@@ -1694,6 +1775,13 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
     ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(P) \
+    ((Export *) erts_psd_get((P), ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT))
+#define ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(P, L, DSTE) \
+    ((Export *) erts_psd_set((P), (L), ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT, (void *) (DSTE)))
+#endif
+
 
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
@@ -1887,7 +1975,12 @@ Uint erts_get_scheduler_id(void)
 {
 #ifdef ERTS_SMP
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    return esdp ? esdp->no : (Uint) 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
+	return 0;
+    else
+#endif
+	return esdp ? esdp->no : (Uint) 0;
 #else
     return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
 #endif
-- 
cgit v1.2.3


From 331bb6cab54e6697e12cc9c5a4ca0ae618c37dd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 23 May 2012 16:08:59 +0200
Subject: BEAM loader: Support preservation of extra operand in transforms

It was not possible to preserve extra arguments in transformations.
The following (hypothetical) example will now work:

some_op Lit=c SizeArg Rest=* => move Lit x | some_op x SizeArg Rest
---
 erts/emulator/beam/beam_load.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 938fd8f2c9..58207ec75b 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4376,6 +4376,7 @@ transform_engine(LoaderState* st)
     Uint* restart;		/* Where to restart if current match fails. */
     GenOpArg def_vars[TE_MAX_VARS]; /* Default buffer for variables. */
     GenOpArg* var = def_vars;
+    int num_vars = 0;
     int i;			/* General index. */
     Uint mask;
     GenOp* instr;
@@ -4578,9 +4579,9 @@ transform_engine(LoaderState* st)
 	    {
 		int n = *pc++;
 		int formal_arity = gen_opc[instr->op].arity;
-		int num_vars = n + (instr->arity - formal_arity);
 		int j = formal_arity;
 
+		num_vars = n + (instr->arity - formal_arity);
 		var = erts_alloc(ERTS_ALC_T_LOADER_TMP,
 				 num_vars * sizeof(GenOpArg));
 		for (i = 0; i < n; i++) {
@@ -4592,7 +4593,6 @@ transform_engine(LoaderState* st)
 	    }
 	    break;
 #endif
-
 	case TOP_next_arg:
 	    ap++;
 	    break;
@@ -4680,6 +4680,20 @@ transform_engine(LoaderState* st)
 	    instr->a[ap].val = var[i].val;
 	    ap++;
 	    break;
+#if defined(TOP_store_rest_args)
+	case TOP_store_rest_args:
+	    {
+		int n = *pc++;
+		int num_extra = num_vars - n;
+
+		ASSERT(n <= num_vars);
+		GENOP_ARITY(instr, instr->arity+num_extra);
+		memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg));
+		memcpy(instr->a+ap, var+n, num_extra*sizeof(GenOpArg));
+		ap += num_extra;
+	    }
+	    break;
+#endif
 	case TOP_try_me_else:
 	    restart = pc + 1;
 	    restart += *pc++;
-- 
cgit v1.2.3


From 64ee859fc4c17259ab95192abf7493fed8f2b0ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 16 May 2012 10:50:30 +0200
Subject: Implement support for maps in the compiler

To make it possible to build the entire OTP system, also define
dummys for the instructions in ops.tab.
---
 erts/emulator/beam/ops.tab | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index c29f3f9b1b..b52ecfaefc 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1465,6 +1465,15 @@ fclearerror
 apply I
 apply_last I P
 
+#
+# Map instructions in R16.
+#
+
+put_map Fail Src Dst Live Size Rest=* => jump Fail
+is_map Fail Src => jump Fail
+has_map_field Fail Src Key => jump Fail
+get_map_element Fail Src Key Dst => jump Fail
+
 #
 # Optimize addition and subtraction of small literals using
 # the i_increment/4 instruction (in bodies, not in guards).
-- 
cgit v1.2.3


From 345bf9d4634f81ad0343e8c60442bdd293e41835 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 21 May 2013 17:00:11 +0200
Subject: erts: Maps beam-instruction definitions

---
 erts/emulator/beam/ops.tab | 67 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 61 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b52ecfaefc..b89bdb2e3a 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1466,14 +1466,69 @@ apply I
 apply_last I P
 
 #
-# Map instructions in R16.
-#
-
-put_map Fail Src Dst Live Size Rest=* => jump Fail
-is_map Fail Src => jump Fail
-has_map_field Fail Src Key => jump Fail
+# Map instructions in R17.
+#
+
+# put_map Fail Src Dst Live Size Rest=* => jump Fail
+# is_map Fail Src => jump Fail
+# has_map_field Fail Src Key => jump Fail
+# get_map_element Fail Src Key Dst => jump Fail
+
+put_map F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
+put_map F Src Dst Live Size Rest=* => update_map F Src Dst Live Size Rest
+
+new_map j d I I
+update_map j d d I I
+
+is_map Fail cq => jump Fail
+
+%macro: is_map IsMap -fail_action
+is_map f r
+is_map f x
+is_map f y
+
+has_map_field Fail Src=rxy Key=arxy => i_has_map_field Fail Src Key
+has_map_field Fail Src Key => move Key x | i_has_map_field Fail Src x
+
+%macro: i_has_map_field HasMapField -fail_action
+i_has_map_field f r a
+i_has_map_field f x a
+i_has_map_field f y a
+i_has_map_field f r r
+i_has_map_field f x r
+i_has_map_field f y r
+i_has_map_field f r x
+i_has_map_field f x x
+i_has_map_field f y x
+i_has_map_field f r y
+i_has_map_field f x y
+i_has_map_field f y y
+
+get_map_element Fail Src=rxy Key=ax Dst => i_get_map_element Fail Src Key Dst
+get_map_element Fail Src=rxy Key=rycq Dst => \
+    move Key x | i_get_map_element Fail Src x Dst
 get_map_element Fail Src Key Dst => jump Fail
 
+%macro: i_get_map_element GetMapElement -fail_action
+i_get_map_element f r a r
+i_get_map_element f x a r
+i_get_map_element f y a r
+i_get_map_element f r a x
+i_get_map_element f x a x
+i_get_map_element f y a x
+i_get_map_element f r a y
+i_get_map_element f x a y
+i_get_map_element f y a y
+i_get_map_element f r x r
+i_get_map_element f x x r
+i_get_map_element f y x r
+i_get_map_element f r x x
+i_get_map_element f x x x
+i_get_map_element f y x x
+i_get_map_element f r x y
+i_get_map_element f x x y
+i_get_map_element f y x y
+
 #
 # Optimize addition and subtraction of small literals using
 # the i_increment/4 instruction (in bodies, not in guards).
-- 
cgit v1.2.3


From 03d5bb0d1bd5d7c4e09ee793a680f2d538fe0122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 21 May 2013 18:09:55 +0200
Subject: erts: Initial Map instructions, type and structure

---
 erts/emulator/beam/beam_emu.c        | 341 ++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/bif.tab           |   6 +
 erts/emulator/beam/copy.c            |  32 ++++
 erts/emulator/beam/erl_bif_op.c      |  12 +-
 erts/emulator/beam/erl_db_util.c     |   1 +
 erts/emulator/beam/erl_debug.c       |   1 +
 erts/emulator/beam/erl_gc.c          |   1 +
 erts/emulator/beam/erl_gc.h          |  37 ++--
 erts/emulator/beam/erl_map.c         | 225 +++++++++++++++++++++++
 erts/emulator/beam/erl_map.h         |  64 +++++++
 erts/emulator/beam/erl_printf_term.c |  52 +++++-
 erts/emulator/beam/erl_term.c        |   6 +-
 erts/emulator/beam/erl_term.h        |  44 +++--
 erts/emulator/beam/utils.c           |  41 ++++-
 14 files changed, 808 insertions(+), 55 deletions(-)
 create mode 100644 erts/emulator/beam/erl_map.c
 create mode 100644 erts/emulator/beam/erl_map.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 7fecdd5c5f..5c955b9566 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -31,6 +31,7 @@
 #include "big.h"
 #include "beam_load.h"
 #include "erl_binary.h"
+#include "erl_map.h"
 #include "erl_bits.h"
 #include "dist.h"
 #include "beam_bp.h"
@@ -701,6 +702,19 @@ extern int count_instructions;
         Fail; 								  \
     }
 
+#define IsMap(Src, Fail) if (is_not_map(Src)) { Fail; }
+
+#define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; }
+
+#define GetMapElement(Src, Key, Dst, Fail)	\
+  do {						\
+     Eterm _res = get_map_element(Src, Key);	\
+     if (is_non_value(_res)) {			\
+        Fail;					\
+     }						\
+     Dst = _res;				\
+  } while (0)
+
 #define IsFunction(X, Action)			\
   do {						\
      if ( !(is_any_fun(X)) ) {			\
@@ -944,7 +958,11 @@ static BeamInstr* apply_fun(Process* p, Eterm fun,
 			    Eterm args, Eterm* reg) NOINLINE;
 static Eterm new_fun(Process* p, Eterm* reg,
 		     ErlFunEntry* fe, int num_free) NOINLINE;
-
+static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE;
+static Eterm update_map(Process* p, Eterm* reg,
+			Eterm map, BeamInstr* I) NOINLINE;
+static int has_not_map_field(Eterm map, Eterm key);
+static Eterm get_map_element(Eterm map, Eterm key);
 
 /*
  * Functions not directly called by process_main(). OK to inline.
@@ -2323,6 +2341,37 @@ void process_main(void)
      Goto(*I);
  }
 
+ OpCase(new_map_jdII): {
+     Eterm res;
+
+     x(0) = r(0);
+     SWAPOUT;
+     res = new_map(c_p, reg, I);
+     SWAPIN;
+     r(0) = x(0);
+     StoreResult(res, Arg(1));
+     Next(4+Arg(3));
+ }
+
+ OpCase(update_map_jddII): {
+     Eterm res;
+     Eterm map;
+
+     GetArg1(1, map);
+     x(0) = r(0);
+     SWAPOUT;
+     res = update_map(c_p, reg, map, I);
+     SWAPIN;
+     if (res) {
+	 r(0) = x(0);
+	 StoreResult(res, Arg(2));
+	 Next(5+Arg(4));
+     } else {
+	 goto lb_Cl_error;
+     }
+ }
+
+
     /*
      * All guards with zero arguments have special instructions:
      * 	self/0
@@ -6227,6 +6276,296 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     return make_fun(funp);
 }
 
+static int has_not_map_field(Eterm map, Eterm key)
+{
+    map_t* mp;
+    Eterm* keys;
+    Uint i;
+    Uint n;
+
+    mp   = (map_t *)map_val(map);
+    keys = map_get_keys(mp);
+    n    = map_get_size(mp);
+    if (is_immed(key)) {
+	for (i = 0; i < n; i++) {
+	    if (keys[i] == key) {
+		return 0;
+	    }
+	}
+    } else {
+	for (i = 0; i <  n; i++) {
+	    if (eq(keys[i], key)) {
+		return 0;
+	    }
+	}
+    }
+    return 1;
+}
+
+static Eterm get_map_element(Eterm map, Eterm key)
+{
+    map_t *mp;
+    Eterm* ks, *vs;
+    Uint i;
+    Uint n;
+
+    mp = (map_t *)map_val(map);
+    ks = map_get_keys(mp);
+    vs = map_get_values(mp);
+    n  = map_get_size(mp);
+    if (is_immed(key)) {
+	for (i = 0; i < n; i++) {
+	    if (ks[i] == key) {
+		return vs[i];
+	    }
+	}
+    } else {
+	for (i = 0; i < n; i++) {
+	    if (eq(ks[i], key)) {
+		return vs[i];
+	    }
+	}
+    }
+    return THE_NON_VALUE;
+}
+
+#define GET_TERM(term, dest)					\
+do {								\
+    Eterm src = term;						\
+    switch (src & _TAG_IMMED1_MASK) {				\
+    case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
+	dest = x(0);						\
+	break;							\
+    case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
+	dest = x(src >> _TAG_IMMED1_SIZE);			\
+	break;							\
+    case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
+	dest = y(src >> _TAG_IMMED1_SIZE);			\
+	break;							\
+    default:							\
+	dest = src;						\
+	break;							\
+    }								\
+} while(0)
+
+
+static Eterm
+new_map(Process* p, Eterm* reg, BeamInstr* I)
+{
+    Uint n = Arg(3);
+    Uint i;
+    Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */;
+    Eterm keys;
+    Eterm *mhp,*thp;
+    Eterm *E;
+    Eterm *tp;
+    map_t *mp;
+
+    if (HeapWordsLeft(p) < need) {
+	erts_garbage_collect(p, need, reg, Arg(2));
+    }
+
+    thp    = p->htop;
+    mhp    = thp + 1 + n/2;
+    E      = p->stop;
+    tp     = &Arg(4);
+    keys   = make_tuple(thp);
+    *thp++ = make_arityval(n/2);
+
+    mp = (map_t *)mhp; mhp += 3;
+    mp->thing_word = MAP_HEADER;
+    mp->size = n/2;
+    mp->keys = keys;
+
+    for (i = 0; i < n/2; i++) {
+	GET_TERM(*tp++, *thp++);
+	GET_TERM(*tp++, *mhp++);
+    }
+    p->htop = mhp;
+    return make_map(mp);
+}
+
+
+/* This entire instruction will be split into two.
+ * 1) update_map_exact (literals) <- this can be much more optimized
+ * 2) update_map_assoc (literals)
+ * Also update_map is pretty bad code as it stands now.
+ */
+
+static Eterm
+update_map(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
+{
+    Uint n;
+    Uint i;
+    Uint num_old;
+    Uint num_updates;
+    Uint need;
+    map_t *old_mp, *mp;
+    Eterm res;
+    Eterm* hp;
+    Eterm* E;
+    Eterm* old_keys;
+    Eterm* old_vals;
+    Eterm* new_p;
+    Eterm new_key;
+    Eterm* kp;
+
+    if (is_not_map(map)) {
+	return 0;
+    }
+
+    old_mp = (map_t *) map_val(map);
+    num_old = map_get_size(old_mp);
+
+    /*
+     * If the old map is empty, create a new map.
+     */
+
+    if (num_old == 0) {
+	return new_map(p, reg, I+1);
+    }
+
+    /*
+     * Allocate heap space for the worst case (i.e. all keys are new).
+     */
+
+    num_updates = Arg(4) / 2;
+    need = 2*(num_old+num_updates) + 4;
+    if (HeapWordsLeft(p) < need) {
+	Uint live = Arg(3);
+	reg[live] = map;
+	erts_garbage_collect(p, need, reg, live+1);
+	map      = reg[live];
+	old_mp   = (map_t *)map_val(map);
+    }
+
+    /*
+     * Update map, optimistically assuming that there are no
+     * new keys, allowing us to keep the old key tuple.
+     */
+
+    hp = p->htop;
+    E  = p->stop;
+
+    old_vals = map_get_values(old_mp);
+    old_keys = map_get_keys(old_mp);
+
+    res = make_map(hp);
+    mp = (map_t *)hp; hp += 3;
+    mp->thing_word = MAP_HEADER;
+    mp->size = num_old;
+    mp->keys = old_mp->keys;
+
+    ASSERT(num_updates > 0);
+
+    /* Get array of key/value pairs to be updated */
+    new_p = &Arg(5);
+    GET_TERM(*new_p, new_key);
+    
+    n = num_updates;
+
+    for (i = 0; i < num_old; i++) {
+	if (new_key == THE_NON_VALUE || !eq(*old_keys, new_key)) {
+	    /* not same keys */
+	    *hp++ = *old_vals;
+	} else {
+	    GET_TERM(new_p[1], *hp);
+	    hp++;
+	    n--;
+	    if (n == 0) {
+		new_key = THE_NON_VALUE;
+	    } else {
+		new_p += 2;
+		GET_TERM(*new_p, new_key);
+	    }
+	}
+	old_vals++, old_keys++;
+    }
+
+    /*
+     * If we have exhausted the update list we are done.
+     */
+
+    if (n == 0) {
+	p->htop = hp;
+	return res;
+    }
+
+    /*
+     * There were some new keys. We'll have to start over and rebuild
+     * the key tuple too.
+     */
+
+    kp    = p->htop;
+    *kp++ = make_arityval(0);
+
+    res = make_map(hp);
+    mp  = (map_t *)hp; hp += 3;
+    mp->keys = make_tuple(kp-1);
+
+    old_vals = map_get_values(old_mp);
+    old_keys = map_get_keys(old_mp);
+
+    new_p = &Arg(5);
+    GET_TERM(*new_p, new_key);
+    n = num_updates;
+
+    for (;;) {
+	Eterm key;
+	Sint c;
+
+	ASSERT(kp < (Eterm *)mp);
+	key = *old_keys;
+	if ((c = cmp(key, new_key)) < 0) {
+	    *kp++ = key;
+	    *hp++ = *old_vals;
+	    old_keys++, old_vals++, num_old--;
+	} else {		/* Replace or insert new */
+	    *kp++ = new_key;
+	    GET_TERM(new_p[1], *hp++);
+	    if (c == 0) {	/* If replacement */
+		old_keys++, old_vals++, num_old--;
+	    }
+	    n--;
+	    if (n == 0) {
+		break;
+	    } else {
+		new_p += 2;
+		GET_TERM(*new_p, new_key);
+	    }
+	}
+	if (num_old == 0) {
+	    break;
+	}
+    }
+
+    while (n-- > 0) {
+	GET_TERM(new_p[0], *kp++);
+	GET_TERM(new_p[1], *hp++);
+	new_p += 2;
+    }
+
+    /*
+     * All updates done. Now copy the remaining part of the frame's
+     * keys and values.
+     */
+
+    while (num_old-- > 0) {
+	ASSERT(kp < (Eterm *)mp);
+	*kp++ = *old_keys++;
+	*hp++ = *old_vals++;
+    }
+    if ((n = (Eterm *)mp - kp) > 0) {
+	*kp = make_pos_bignum_header(n-1);
+    }
+    n = kp - p->htop - 1;	/* Actual number of keys/values */
+    *p->htop = make_arityval(n);
+    mp->thing_word = MAP_HEADER;
+    mp->size = n;
+    p->htop = hp;
+    return res;
+}
+#undef GET_TERM
 
 int catchlevel(Process *p)
 {
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 3ec534f0bc..42515d0494 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -578,6 +578,12 @@ bif os:unsetenv/1
 
 bif re:inspect/2
 
+ubif erlang:is_map/1
+bif map:to_list/1
+bif map:new/0
+bif map:get/2
+bif map:put/3
+
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 23c0fca6aa..3a987e213b 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -27,6 +27,7 @@
 #include "erl_process.h"
 #include "erl_gc.h"
 #include "big.h"
+#include "erl_map.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "dtrace-wrapper.h"
@@ -150,6 +151,24 @@ Uint size_object(Eterm obj)
 			goto pop_next;
 		    }
 		    break;
+		case MAP_SUBTAG:
+		    {
+			Uint n;
+			map_t *mp;
+			mp  = (map_t*)map_val_rel(obj,base);
+			ptr = (Eterm *)mp;
+			n   = map_get_size(mp) + 1;
+			sum += n + 2;
+			ptr += 2; /* hdr + size words */
+			while (n--) {
+			    obj = *ptr++;
+			    if (!IS_CONST(obj)) {
+				ESTACK_PUSH(s, obj);
+			    }
+			}
+			goto pop_next;
+		    }
+		    break;
 		case BIN_MATCHSTATE_SUBTAG:
 		    erl_exit(ERTS_ABORT_EXIT,
 			     "size_object: matchstate term not allowed");
@@ -318,6 +337,15 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    }
 		}
 		break;
+	    case MAP_SUBTAG:
+		{
+		    i = map_get_size(objp) + 3;
+		    *argp = make_map_rel(htop, dst_base);
+		    while (i--) {
+			*htop++ = *objp++;
+		    }
+		}
+		break;
 	    case REFC_BINARY_SUBTAG:
 		{
 		    ProcBin* pb;
@@ -537,6 +565,10 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		}
 		goto off_heap_common;
 
+	    case MAP_SUBTAG:
+		*hp++ = *tp++;
+		sz--;
+		break;
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index adac0052d6..37dd6457db 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -36,6 +36,7 @@
 #include "dist.h"
 #include "erl_version.h"
 #include "erl_binary.h"
+#include "erl_map.h"
 
 BIF_RETTYPE and_2(BIF_ALIST_2)
 {
@@ -321,7 +322,10 @@ BIF_RETTYPE is_record_3(BIF_ALIST_3)
     BIF_RET(am_false);
 }
 	
-
-    
-
-
+BIF_RETTYPE is_map_1(BIF_ALIST_1)
+{
+    if (is_map(BIF_ARG_1)) {
+	BIF_RET(am_true);
+    }
+    BIF_RET(am_false);
+}
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index a358ecf326..d903053aa4 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -35,6 +35,7 @@
 #include "bif.h"
 #include "big.h"
 #include "erl_binary.h"
+#include "erl_map.h"
 #include "erl_thr_progress.h"
 
 #include "erl_db_util.h"
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 873a9860da..50bdc79506 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -29,6 +29,7 @@
 #include "bif.h"
 #include "beam_catches.h"
 #include "erl_debug.h"
+#include "erl_map.h"
 
 #define WITHIN(ptr, x, y) ((x) <= (ptr) && (ptr) < (y))
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index ab8448e8a1..2022f70cbb 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -28,6 +28,7 @@
 #include "beam_catches.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
+#include "erl_map.h"
 #include "error.h"
 #include "big.h"
 #include "erl_gc.h"
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 1801df359a..5203dda263 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -20,6 +20,8 @@
 #ifndef __ERL_GC_H__
 #define __ERL_GC_H__
 
+#include "erl_map.h"
+
 /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */
 
 #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -42,23 +44,24 @@ do {									\
     HTOP += 2;			/* update tospace htop */		\
 } while(0)
 
-#define MOVE_BOXED(PTR,HDR,HTOP,ORIG)					\
-do {									\
-    Eterm gval;								\
-    Sint nelts;								\
-									\
-    ASSERT(is_header(HDR));						\
-    gval = make_boxed(HTOP);						\
-    *ORIG = gval;							\
-    *HTOP++ = HDR;							\
-    *PTR++ = gval;							\
-    nelts = header_arity(HDR);						\
-    switch ((HDR) & _HEADER_SUBTAG_MASK) {				\
-    case SUB_BINARY_SUBTAG: nelts++; break;				\
-    case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR-1))->num_free+1; break;	\
-    }									\
-    while (nelts--)							\
-	*HTOP++ = *PTR++;						\
+#define MOVE_BOXED(PTR,HDR,HTOP,ORIG)                                   \
+do {                                                                    \
+    Eterm gval;                                                         \
+    Sint nelts;                                                         \
+                                                                        \
+    ASSERT(is_header(HDR));                                             \
+    nelts = header_arity(HDR);                                          \
+    switch ((HDR) & _HEADER_SUBTAG_MASK) {                              \
+    case SUB_BINARY_SUBTAG: nelts++; break;                             \
+    case MAP_SUBTAG: nelts+=map_get_size(PTR) + 1; break;               \
+    case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break;   \
+    }                                                                   \
+    gval    = make_boxed(HTOP);                                         \
+    *ORIG   = gval;                                                     \
+    *HTOP++ = HDR;                                                      \
+    *PTR++  = gval;                                                     \
+    while (nelts--) *HTOP++ = *PTR++;                                   \
+                                                                        \
 } while(0)
 
 #define in_area(ptr,start,nbytes) \
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
new file mode 100644
index 0000000000..1fd7943c30
--- /dev/null
+++ b/erts/emulator/beam/erl_map.c
@@ -0,0 +1,225 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2013. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ *
+ * Author: Björn-Egil Dahlberg
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "erl_process.h"
+#include "error.h"
+#include "bif.h"
+
+
+#include "erl_map.h"
+
+
+BIF_RETTYPE map_to_list_1(BIF_ALIST_1) {
+    if (is_map(BIF_ARG_1)) {
+	Uint n;
+	Eterm* hp;
+	Eterm *ks,*vs, res, tup;
+	map_t *mp = (map_t*)map_val(BIF_ARG_1);
+
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
+	n   = map_get_size(mp);
+	hp  = HAlloc(BIF_P, (2 + 3) * n);
+	res = NIL;
+
+	while(n--) {
+	    tup = TUPLE2(hp, ks[n], vs[n]); hp += 3;
+	    res = CONS(hp, tup, res); hp += 2;
+	}
+
+	BIF_RET(res);
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+BIF_RETTYPE map_new_0(BIF_ALIST_0) {
+    Eterm* hp;
+    Eterm tup;
+    map_t *mp;
+
+    hp    = HAlloc(BIF_P, (3 + 1));
+    tup   = make_tuple(hp);
+    *hp++ = make_arityval(0);
+
+    mp    = (map_t*)hp;
+    mp->thing_word = MAP_HEADER;
+    mp->size = 0;
+    mp->keys = tup;
+
+    BIF_RET(make_map(mp));
+}
+
+BIF_RETTYPE map_get_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+	Eterm *hp, *ks,*vs, key, error;
+	map_t *mp;
+	Uint n,i;
+	char *s_error;
+
+	mp  = (map_t*)map_val(BIF_ARG_2);
+	key = BIF_ARG_1;
+	n   = map_get_size(mp);
+	
+	if (n == 0)
+	    goto error;
+
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
+
+	if (is_immed(key)) {
+	    for( i = 0; i < n; i++) {
+		if (ks[i] == key) {
+		    BIF_RET(vs[i]);
+		}
+	    }
+	}
+
+	for( i = 0; i < n; i++) {
+	    if (eq(ks[i], key)) {
+		BIF_RET(vs[i]);
+	    }
+	}
+error:
+
+	s_error = "bad_key";
+	error = am_atom_put(s_error, sys_strlen(s_error));
+
+	hp = HAlloc(BIF_P, 3);
+	BIF_P->fvalue = TUPLE2(hp, error, key);
+	BIF_ERROR(BIF_P, EXC_ERROR_2);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+BIF_RETTYPE map_put_3(BIF_ALIST_3) {
+    if (is_map(BIF_ARG_3)) {
+	Sint n,i;
+	Sint c = 0;
+	Eterm* hp, *shp;
+	Eterm *ks,*vs, res, key, tup;
+	map_t *mp = (map_t*)map_val(BIF_ARG_3);
+
+	key = BIF_ARG_1;
+	n   = map_get_size(mp);
+
+	if (n == 0) {
+	    hp    = HAlloc(BIF_P, 4 + 2);
+	    tup   = make_tuple(hp);
+	    *hp++ = make_arityval(1);
+	    *hp++ = key;
+	    res   = make_map(hp);
+	    *hp++ = MAP_HEADER;
+	    *hp++ = 1;
+	    *hp++ = tup;
+	    *hp++ = BIF_ARG_2;
+
+	    BIF_RET(res);
+	}
+
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
+	/* only allocate for values,
+	 * assume key-tuple will be intact
+	 */
+
+	hp  = HAlloc(BIF_P, 3 + n);
+	shp = hp; /* save hp, used if optimistic update fails */
+	res = make_map(hp);
+	*hp++ = MAP_HEADER;
+	*hp++ = n;
+	*hp++ = mp->keys;
+
+	if (is_immed(key)) {
+	    for( i = 0; i < n; i ++) {
+		if (ks[i] == key) {
+		    *hp++ = BIF_ARG_2;
+		    vs++;
+		    c = 1;
+		} else {
+		    *hp++ = *vs++;
+		}
+	    }
+	} else {
+	    for( i = 0; i < n; i ++) {
+		if (eq(ks[i], key)) {
+		    *hp++ = BIF_ARG_2;
+		    vs++;
+		    c = 1;
+		} else {
+		    *hp++ = *vs++;
+		}
+	    }
+	}
+
+	if (c)
+	    BIF_RET(res);
+
+	/* need to make a new tuple,
+	 * use old hp since it needs to be recreated anyway.
+	 */
+	tup    = make_tuple(shp);
+	*shp++ = make_arityval(n+1);
+
+	hp    = HAlloc(BIF_P, 3 + n + 1);
+	res   = make_map(hp);
+	*hp++ = MAP_HEADER;
+	*hp++ = n + 1;
+	*hp++ = tup;
+
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
+
+	ASSERT(n >= 0);
+
+	/* copy map in order */
+	while (n && ((c = CMP(*ks, key)) < 0)) {
+	    *shp++ = *ks++;
+	    *hp++  = *vs++;
+	    n--;
+	}
+
+	*shp++ = key;
+	*hp++  = BIF_ARG_2;
+
+	ASSERT(n >= 0);
+
+	while(n--) {
+	    *shp++ = *ks++;
+	    *hp++  = *vs++;
+	}
+	/* we have one word remaining
+	 * this will work out fine once we get the size word
+	 * in the header.
+	 */
+	*shp = make_pos_bignum_header(0);
+	BIF_RET(res);
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
new file mode 100644
index 0000000000..ca8ce0f974
--- /dev/null
+++ b/erts/emulator/beam/erl_map.h
@@ -0,0 +1,64 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2013. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+
+#ifndef __ERL_MAP_H__
+#define __ERL_MAP_H__
+
+#include "sys.h"
+/* MAP */
+
+typedef struct map_s {
+    Eterm thing_word;
+    Uint  size;
+    Eterm keys;      /* tuple */
+} map_t;
+/* map node
+ *
+ * -----------
+ * Eterm   THING	
+ * Eterm   Keys -> {K1, K2, K3, ..., Kn} where n = arity
+ * ----
+ * Eterm   V1
+ * ...
+ * Eterm   Vn, where n = arity
+ * -----------
+ */
+
+
+
+/* erl_term.h stuff */
+#define make_map(x)		make_boxed((Eterm*)(x))
+#define make_map_rel(x, BASE)   make_boxed_rel((Eterm*)(x),(BASE))
+#define is_map(x)		(is_boxed((x)) && is_map_header(*boxed_val((x))))
+#define is_map_rel(RTERM,BASE)  is_map(rterm2wterm(RTERM,BASE))
+#define is_not_map(x)           (!is_map((x)))
+#define is_map_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
+#define header_is_map(x)        ((((x) & (_HEADER_SUBTAG_MASK)) == MAP_SUBTAG))
+#define map_val(x)		(_unchecked_boxed_val((x)))
+#define map_val_rel(RTERM, BASE) map_val(rterm2wterm(RTERM, BASE))
+
+#define map_get_values(x)      (((Eterm *)(x)) + 3)
+#define map_get_keys(x)        (((Eterm *)tuple_val(((map_t *)(x))->keys)) + 1)
+#define map_get_size(x)        (((map_t*)(x))->size)
+
+#define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
+
+#endif
+
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 436147749e..d18760dc43 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -24,6 +24,7 @@
 #include "erl_printf_term.h"
 #include "sys.h"
 #include "big.h"
+#include "erl_map.h"
 
 #define PRINT_CHAR(CNT, FN, ARG, C)					\
 do {									\
@@ -216,14 +217,15 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount)
 }
 
 
-#define PRT_BAR ((Eterm) 0)
-#define PRT_COMMA ((Eterm) 1)
-#define PRT_CLOSE_LIST ((Eterm) 2)
-#define PRT_CLOSE_TUPLE ((Eterm) 3)
-#define PRT_TERM ((Eterm) 4)
-#define PRT_ONE_CONS ((Eterm) 5)
-#define PRT_PATCH_FUN_SIZE ((Eterm) 6)
-#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */
+#define PRT_BAR                ((Eterm) 0)
+#define PRT_COMMA              ((Eterm) 1)
+#define PRT_CLOSE_LIST         ((Eterm) 2)
+#define PRT_CLOSE_TUPLE        ((Eterm) 3)
+#define PRT_ASSOC              ((Eterm) 4)
+#define PRT_TERM               ((Eterm) 5)
+#define PRT_ONE_CONS           ((Eterm) 6)
+#define PRT_PATCH_FUN_SIZE     ((Eterm) 7)
+#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 8) /* Note! Must be last... */
 
 static int
 print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
@@ -260,6 +262,9 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	case PRT_CLOSE_TUPLE:
 	    PRINT_CHAR(res, fn, arg, '}');
 	    goto L_outer_loop;
+	case PRT_ASSOC:
+	    PRINT_STRING(res, fn, arg, "=>");
+	    goto L_outer_loop;
 	default:
 	    popped.word = WSTACK_POP(s);
 
@@ -483,6 +488,37 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		PRINT_CHAR(res, fn, arg, '>');
 	    }
 	    break;
+	case MAP_DEF:
+	    {
+		Uint n;
+		Eterm *ks, *vs;
+		map_t *mp = (map_t *)map_val(wobj);
+		n  = map_get_size(mp);
+		ks = map_get_keys(mp);
+		vs = map_get_values(mp);
+
+		PRINT_CHAR(res, fn, arg, '#');
+		PRINT_CHAR(res, fn, arg, '{');
+		WSTACK_PUSH(s, PRT_CLOSE_TUPLE);
+		if (n > 0) {
+		    n--;
+		    WSTACK_PUSH(s, vs[n]);
+		    WSTACK_PUSH(s, PRT_TERM);
+		    WSTACK_PUSH(s, PRT_ASSOC);
+		    WSTACK_PUSH(s, ks[n]);
+		    WSTACK_PUSH(s, PRT_TERM);
+
+		    while (n--) {
+			WSTACK_PUSH(s, PRT_COMMA);
+			WSTACK_PUSH(s, vs[n]);
+			WSTACK_PUSH(s, PRT_TERM);
+			WSTACK_PUSH(s, PRT_ASSOC);
+			WSTACK_PUSH(s, ks[n]);
+			WSTACK_PUSH(s, PRT_TERM);
+		    }
+		}
+	    }
+	    break;
 	default:
 	    PRINT_STRING(res, fn, arg, "<unknown:");
 	    PRINT_POINTER(res, fn, arg, wobj);
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 2f206ffbec..28cbe7004f 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -23,6 +23,7 @@
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
+#include "erl_map.h"
 #include <stdlib.h>
 #include <stdio.h>
 
@@ -85,7 +86,10 @@ unsigned tag_val_def(Wterm x)
 	    case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PID_DEF;
 	    case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PORT_DEF;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):	return EXTERNAL_REF_DEF;
-	    default:						return BINARY_DEF;
+	    case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
+	    case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
+	    case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
+	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):	return MAP_DEF;
 	  }
 	  break;
       }
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 50d3e63c58..f10a3a9d38 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -135,11 +135,12 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define REF_SUBTAG		(0x4 << _TAG_PRIMARY_SIZE) /* REF */
 #define FUN_SUBTAG		(0x5 << _TAG_PRIMARY_SIZE) /* FUN */
 #define FLOAT_SUBTAG		(0x6 << _TAG_PRIMARY_SIZE) /* FLOAT */
-#define EXPORT_SUBTAG	(0x7 << _TAG_PRIMARY_SIZE) /* FLOAT */
+#define EXPORT_SUBTAG		(0x7 << _TAG_PRIMARY_SIZE) /* FLOAT */
 #define _BINARY_XXX_MASK	(0x3 << _TAG_PRIMARY_SIZE)
 #define REFC_BINARY_SUBTAG	(0x8 << _TAG_PRIMARY_SIZE) /* BINARY */
 #define HEAP_BINARY_SUBTAG	(0x9 << _TAG_PRIMARY_SIZE) /* BINARY */
 #define SUB_BINARY_SUBTAG	(0xA << _TAG_PRIMARY_SIZE) /* BINARY */
+#define MAP_SUBTAG		(0xB << _TAG_PRIMARY_SIZE) /* MAP */
 #define EXTERNAL_PID_SUBTAG	(0xC << _TAG_PRIMARY_SIZE) /* EXTERNAL_PID */
 #define EXTERNAL_PORT_SUBTAG	(0xD << _TAG_PRIMARY_SIZE) /* EXTERNAL_PORT */
 #define EXTERNAL_REF_SUBTAG	(0xE << _TAG_PRIMARY_SIZE) /* EXTERNAL_REF */
@@ -155,6 +156,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define _TAG_HEADER_REFC_BIN	(TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG)
 #define _TAG_HEADER_HEAP_BIN	(TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG)
 #define _TAG_HEADER_SUB_BIN	(TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG)
+#define _TAG_HEADER_MAP	 	(TAG_PRIMARY_HEADER|MAP_SUBTAG)
 #define _TAG_HEADER_EXTERNAL_PID  (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG)
 #define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG)
 #define _TAG_HEADER_EXTERNAL_REF  (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG)
@@ -354,7 +356,10 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
 #define is_value(x)	((x) != THE_NON_VALUE)
 
 /* binary object access methods */
-#define is_binary_header(x)	(((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_REFC_BIN)
+#define is_binary_header(x) \
+	((((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_REFC_BIN) || \
+	 (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HEAP_BIN) || \
+	 (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_SUB_BIN))
 #define make_binary(x)	make_boxed((Eterm*)(x))
 #define is_binary(x)	(is_boxed((x)) && is_binary_header(*boxed_val((x))))
 #define is_not_binary(x) (!is_binary((x)))
@@ -1064,8 +1069,8 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 
 /*
  * Backwards compatibility definitions:
- * - #define virtal *_DEF constants with values that fit term order:
- *   number < atom < ref < fun < port < pid < tuple < nil < cons < binary
+ * - #define virtual *_DEF constants with values that fit term order:
+ *   number < atom < ref < fun < port < pid < tuple < map < nil < cons < binary
  * - tag_val_def() function generates virtual _DEF tag
  * - not_eq_tags() and NUMBER_CODE() defined in terms
  *   of the tag_val_def() function
@@ -1074,19 +1079,20 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define BINARY_DEF		0x0
 #define LIST_DEF		0x1
 #define NIL_DEF			0x2
-#define TUPLE_DEF		0x3
-#define PID_DEF			0x4
-#define EXTERNAL_PID_DEF	0x5
-#define PORT_DEF		0x6
-#define EXTERNAL_PORT_DEF	0x7
-#define EXPORT_DEF		0x8
-#define FUN_DEF			0x9
-#define REF_DEF			0xa
-#define EXTERNAL_REF_DEF	0xb
-#define ATOM_DEF		0xc
-#define FLOAT_DEF		0xd
-#define BIG_DEF			0xe
-#define SMALL_DEF		0xf
+#define MAP_DEF			0x3
+#define TUPLE_DEF		0x4
+#define PID_DEF			0x5
+#define EXTERNAL_PID_DEF	0x6
+#define PORT_DEF		0x7
+#define EXTERNAL_PORT_DEF	0x8
+#define EXPORT_DEF		0x9
+#define FUN_DEF			0xa
+#define REF_DEF			0xb
+#define EXTERNAL_REF_DEF	0xc
+#define ATOM_DEF		0xd
+#define FLOAT_DEF		0xe
+#define BIG_DEF			0xf
+#define SMALL_DEF		0x10
 
 #if ET_DEBUG
 extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
@@ -1096,8 +1102,8 @@ extern unsigned tag_val_def(Wterm);
 #endif
 #define not_eq_tags(X,Y)	(tag_val_def((X)) ^ tag_val_def((Y)))
 
-#define NUMBER_CODE(x,y)	((tag_val_def(x) << 4) | tag_val_def(y))
-#define _NUMBER_CODE(TX,TY)	((TX << 4) | TY)
+#define NUMBER_CODE(x,y)	((tag_val_def(x) << 5) | tag_val_def(y))
+#define _NUMBER_CODE(TX,TY)	((TX << 5) | TY)
 #define SMALL_SMALL	_NUMBER_CODE(SMALL_DEF,SMALL_DEF)
 #define SMALL_BIG 	_NUMBER_CODE(SMALL_DEF,BIG_DEF)
 #define SMALL_FLOAT 	_NUMBER_CODE(SMALL_DEF,FLOAT_DEF)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index e0776cf67d..6cdfd1c989 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -31,6 +31,7 @@
 #include "bif.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
+#include "erl_map.h"
 #include "packet_parser.h"
 #include "erl_gc.h"
 #define ERTS_WANT_DB_INTERNAL__
@@ -785,10 +786,10 @@ Uint32 make_hash(Eterm term_arg)
     unsigned op;
 
     /* Must not collide with the real tag_val_def's: */
-#define MAKE_HASH_TUPLE_OP 0x10
-#define MAKE_HASH_FUN_OP 0x11
-#define MAKE_HASH_CDR_PRE_OP 0x12
-#define	MAKE_HASH_CDR_POST_OP 0x13
+#define MAKE_HASH_TUPLE_OP 0x11
+#define MAKE_HASH_FUN_OP 0x12
+#define MAKE_HASH_CDR_PRE_OP 0x13
+#define	MAKE_HASH_CDR_POST_OP 0x14
 
     /* 
     ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit 
@@ -2007,6 +2008,18 @@ tailrecur_ne:
 		    ++bb;
 		    goto term_array;
 		}
+	    case MAP_SUBTAG:
+		{
+		    aa = map_val_rel(a, a_base);
+		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
+			goto not_equal;
+		    bb = map_val_rel(b,b_base);
+		    if ((sz = map_get_size((map_t*)aa)) == 0) goto pop_next;
+		    aa += 2;
+		    bb += 2;
+		    sz += 1; /* increment for tuple-keys */
+		    goto term_array;
+		}
 	    case REFC_BINARY_SUBTAG:
 	    case HEAP_BINARY_SUBTAG:
 	    case SUB_BINARY_SUBTAG:
@@ -2281,7 +2294,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2)
  *
  * According to the Erlang Standard, types are orderered as follows:
  *   numbers < (characters) < atoms < refs < funs < ports < pids <
- *   tuples < [] < conses < binaries.
+ *   tuples < maps < [] < conses < binaries.
  *
  * Note that characters are currently not implemented.
  *
@@ -2464,7 +2477,25 @@ tailrecur_ne:
 		++aa;
 		++bb;
 		goto term_array;
+	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) :
+		if (!is_map_rel(b,b_base)) {
+		    a_tag = MAP_DEF;
+		    goto mixed_types;
+		}
+		aa = (Eterm *)map_val_rel(a,a_base);
+		bb = (Eterm *)map_val_rel(b,b_base);
 
+		i = map_get_size((map_t*)aa);
+		if (i != map_get_size((map_t*)bb)) {
+		    RETURN_NEQ((int)(i - map_get_size((map_t*)bb)));
+		}
+		if (i == 0) {
+		    goto pop_next;
+		}
+		aa += 2;
+		bb += 2;
+		i  += 1; /* increment for tuple-keys */
+		goto term_array;
 	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
 		if (!is_float_rel(b,b_base)) {
 		    a_tag = FLOAT_DEF;
-- 
cgit v1.2.3


From 92303a2e1abdf74aa3bc3af095131a59a601c45e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 1 Oct 2013 19:27:54 +0200
Subject: erts: Add phash2 Map functionality

---
 erts/emulator/beam/utils.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 6cdfd1c989..bc7e91295d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1088,6 +1088,7 @@ make_hash2(Eterm term)
 #define HCONST_13 0x08d12e65UL
 #define HCONST_14 0xa708a81eUL
 #define HCONST_15 0x454021d7UL
+#define HCONST_16 0xe3779b90UL
 
 #define UINT32_HASH_2(Expr1, Expr2, AConst)       \
          do {                                     \
@@ -1190,6 +1191,28 @@ make_hash2(Eterm term)
 		term = elem[1];
 	    }
 	    break;
+	    case MAP_SUBTAG:
+	    {
+		map_t *mp = (map_t *)map_val(term);
+		int i;
+		int size = map_get_size(mp);
+		Eterm *ks = map_get_keys(mp);
+		Eterm *vs = map_get_values(mp);
+		UINT32_HASH(size, HCONST_16);
+		if (size == 0) /* Empty Map */
+		    goto hash2_common;
+
+		for (i = size - 1; i >= 0; i--) {
+		    tmp = vs[i];
+		    ESTACK_PUSH(s, tmp);
+		}
+		for (i = size - 1; i >= 1; i--) {
+		    tmp = ks[i];
+		    ESTACK_PUSH(s, tmp);
+		}
+		term = ks[0];
+	    }
+	    break;
 	    case EXPORT_SUBTAG:
 	    {
 		Export* ep = *((Export **) (export_val(term) + 1));
-- 
cgit v1.2.3


From 63ef0bbfdfb70673fe7f3ce2fc6fa4f0f801747d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 19 Sep 2013 17:48:28 +0200
Subject: erts: Add the size-testing guard BIF map_size/1

---
 erts/emulator/beam/beam_emu.c      |  2 ++
 erts/emulator/beam/beam_load.c     |  2 ++
 erts/emulator/beam/bif.tab         |  1 +
 erts/emulator/beam/erl_bif_guard.c | 23 +++++++++++++++++++++++
 erts/emulator/beam/erl_map.c       | 23 +++++++++++++++++++++--
 erts/emulator/beam/global.h        |  1 +
 6 files changed, 50 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 5c955b9566..1cad31be2c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5080,6 +5080,8 @@ translate_gc_bif(void* gcf)
 	return bit_size_1;
     } else if (gcf == erts_gc_byte_size_1) {
 	return byte_size_1;
+    } else if (gcf == erts_gc_map_size_1) {
+	return map_size_1;
     } else if (gcf == erts_gc_abs_1) {
 	return abs_1;
     } else if (gcf == erts_gc_float_1) {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 58207ec75b..b589d1c930 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3783,6 +3783,8 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 	op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1;
     } else if (bf == byte_size_1) {
 	op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1;
+    } else if (bf == map_size_1) {
+	op->a[1].val = (BeamInstr) (void *) erts_gc_map_size_1;
     } else if (bf == abs_1) {
 	op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1;
     } else if (bf == float_1) {
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 42515d0494..a8623fcf61 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -579,6 +579,7 @@ bif os:unsetenv/1
 bif re:inspect/2
 
 ubif erlang:is_map/1
+ubif erlang:map_size/1
 bif map:to_list/1
 bif map:new/0
 bif map:get/2
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index a715756c15..bbd8aa31d9 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -33,6 +33,7 @@
 #include "bif.h"
 #include "big.h"
 #include "erl_binary.h"
+#include "erl_map.h"
 
 static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live);
 
@@ -455,6 +456,28 @@ Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live)
     }
 }
 
+Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live)
+{
+    Eterm arg = reg[live];
+    if (is_map(arg)) {
+	map_t *mp = (map_t*)map_val(arg);
+	Uint size = map_get_size(mp);
+	if (IS_USMALL(0, size)) {
+	    return make_small(size);
+	} else {
+	    Eterm* hp;
+	    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
+		erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
+	    }
+	    hp = p->htop;
+	    p->htop += BIG_UINT_HEAP_SIZE;
+	    return uint_to_big(size, hp);
+	}
+    } else {
+	BIF_ERROR(p, BADARG);
+    }
+}
+
 Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg;
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 1fd7943c30..6a59fa29b8 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -30,10 +30,8 @@
 #include "error.h"
 #include "bif.h"
 
-
 #include "erl_map.h"
 
-
 BIF_RETTYPE map_to_list_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
 	Uint n;
@@ -58,6 +56,27 @@ BIF_RETTYPE map_to_list_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
+
+/* erlang:map_size/1
+ * the corresponding instruction is implemented in:
+ *     beam/erl_bif_guard.c
+ */
+
+BIF_RETTYPE map_size_1(BIF_ALIST_1) {
+    if (is_map(BIF_ARG_1)) {
+	Eterm *hp;
+	Uint hsz  = 0;
+	map_t *mp = (map_t*)map_val(BIF_ARG_1);
+	Uint n    = map_get_size(mp);
+
+	erts_bld_uint(NULL, &hsz, n);
+	hp = HAlloc(BIF_P, hsz);
+	BIF_RET(erts_bld_uint(&hp, NULL, n));
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 BIF_RETTYPE map_new_0(BIF_ALIST_0) {
     Eterm* hp;
     Eterm tup;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 83a8911a36..8fcb95d0e2 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -998,6 +998,7 @@ Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live);
+Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live);
-- 
cgit v1.2.3


From 43ed0cc716039c3b2f65a5395f00424169639309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 21 May 2013 18:11:39 +0200
Subject: erts: Add the type-testing guard BIF is_map/1

To add a type-testing guard BIF, the following steps are needed:

* The BIF itself is added to bif.tab (note that it should be declared
  using "ubif", not "bif"), and its implementation to erl_bif_op.c.

* erl_internal must be modified in 3 places: The type test must be
  recognized as guard BIF, as a type test, and it must be auto-imported.

* There must be an instruction that implements the same type test as
  the BIF (it will be used in guards). beam_utils:bif_to_test/3 must
  be updated to recognize the new guard BIF.
---
 erts/emulator/beam/bif.tab | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index a8623fcf61..8f85f931c9 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -572,6 +572,7 @@ bif erlang:binary_to_float/1
 bif io:printable_range/0
 bif os:unsetenv/1
 
+
 #
 # New in R17A
 #
-- 
cgit v1.2.3


From bc0960c095e9482ba0f452c9df3623f5c9c54e1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 30 Sep 2013 20:13:07 +0200
Subject: erts: Introduce more Maps BIFs

 * map:remove/2

 * map:keys/1

 * map:values/1

 * map:is_key/2

 * map:update/3
   - Equivalent to ':=' operator in #{ K := V } maps.

 * map:from_list/1
   - map:from_list/1 takes any unsorted key/value list, [{K,V}], and
     produces a map. Duplicate keys are removed. The latest key is kept.

 * map:find/2
  - Searches for a pair that *equals* input key.

 * map:merge/2
  - Merge two maps to one map.
---
 erts/emulator/beam/bif.tab   |  11 +-
 erts/emulator/beam/erl_map.c | 565 ++++++++++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_map.h |   6 +-
 3 files changed, 547 insertions(+), 35 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 8f85f931c9..306861a4c5 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -572,7 +572,6 @@ bif erlang:binary_to_float/1
 bif io:printable_range/0
 bif os:unsetenv/1
 
-
 #
 # New in R17A
 #
@@ -582,9 +581,17 @@ bif re:inspect/2
 ubif erlang:is_map/1
 ubif erlang:map_size/1
 bif map:to_list/1
-bif map:new/0
+bif map:find/2
 bif map:get/2
+bif map:from_list/1
+bif map:is_key/2
+bif map:keys/1
+bif map:merge/2
+bif map:new/0
 bif map:put/3
+bif map:remove/2
+bif map:update/3
+bif map:values/1
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 6a59fa29b8..6fd60f0689 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -32,6 +32,57 @@
 
 #include "erl_map.h"
 
+/* BIFs
+ *
+ * DONE:
+ * - erlang:is_map/1
+ * - erlang:map_size/1
+ *
+ * - map:find/2
+ * - map:from_list/1
+ * - map:get/2
+ * - map:is_key/2
+ * - map:keys/1
+ * - map:merge/2
+ * - map:new/0
+ * - map:put/3
+ * - map:remove/2
+ * - map:to_list/1
+ * - map:update/3
+ * - map:values/1
+ *
+ * TODO:
+ * - map:foldl/3
+ * - map:foldr/3
+ * - map:map/3
+ * - map:size/1
+ * - map:without/2
+ *
+ */
+
+/* erlang:map_size/1
+ * the corresponding instruction is implemented in:
+ *     beam/erl_bif_guard.c
+ */
+
+BIF_RETTYPE map_size_1(BIF_ALIST_1) {
+    if (is_map(BIF_ARG_1)) {
+	Eterm *hp;
+	Uint hsz  = 0;
+	map_t *mp = (map_t*)map_val(BIF_ARG_1);
+	Uint n    = map_get_size(mp);
+
+	erts_bld_uint(NULL, &hsz, n);
+	hp = HAlloc(BIF_P, hsz);
+	BIF_RET(erts_bld_uint(&hp, NULL, n));
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* map:to_list/1
+ */
+
 BIF_RETTYPE map_to_list_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
 	Uint n;
@@ -56,43 +107,40 @@ BIF_RETTYPE map_to_list_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-
-/* erlang:map_size/1
- * the corresponding instruction is implemented in:
- *     beam/erl_bif_guard.c
+/* map:find/2
+ * return value if key *equals* a key in the map
  */
 
-BIF_RETTYPE map_size_1(BIF_ALIST_1) {
-    if (is_map(BIF_ARG_1)) {
-	Eterm *hp;
-	Uint hsz  = 0;
-	map_t *mp = (map_t*)map_val(BIF_ARG_1);
-	Uint n    = map_get_size(mp);
+BIF_RETTYPE map_find_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+	Eterm *hp, *ks,*vs, key, res;
+	map_t *mp;
+	Uint n,i;
 
-	erts_bld_uint(NULL, &hsz, n);
-	hp = HAlloc(BIF_P, hsz);
-	BIF_RET(erts_bld_uint(&hp, NULL, n));
-    }
+	mp  = (map_t*)map_val(BIF_ARG_2);
+	key = BIF_ARG_1;
+	n   = map_get_size(mp);
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
 
+	for( i = 0; i < n; i++) {
+	    if (CMP(ks[i], key)==0) {
+		hp    = HAlloc(BIF_P, 3);
+		res   = make_tuple(hp);
+		*hp++ = make_arityval(2);
+		*hp++ = am_ok;
+		*hp++ = vs[i];
+		BIF_RET(res);
+	    }
+	}
+	BIF_RET(am_error);
+    }
     BIF_ERROR(BIF_P, BADARG);
 }
-
-BIF_RETTYPE map_new_0(BIF_ALIST_0) {
-    Eterm* hp;
-    Eterm tup;
-    map_t *mp;
-
-    hp    = HAlloc(BIF_P, (3 + 1));
-    tup   = make_tuple(hp);
-    *hp++ = make_arityval(0);
-
-    mp    = (map_t*)hp;
-    mp->thing_word = MAP_HEADER;
-    mp->size = 0;
-    mp->keys = tup;
-
-    BIF_RET(make_map(mp));
-}
+/* map:get/2
+ * return value if key *matches* a key in the map
+ * exception bad_key if none matches
+ */
 
 BIF_RETTYPE map_get_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
@@ -136,6 +184,297 @@ error:
     BIF_ERROR(BIF_P, BADARG);
 }
 
+/* map:from_list/1
+ * List may be unsorted [{K,V}]
+ */
+
+BIF_RETTYPE map_from_list_1(BIF_ALIST_1) {
+    Eterm *kv, item = BIF_ARG_1;
+    Eterm *hp, *thp,*vs, *ks, keys, res;
+    map_t *mp;
+    Uint  size = 0, unused_size = 0;
+    Sint  c = 0;
+    Sint  idx = 0;
+
+    if (is_list(item) || is_nil(item)) {
+
+	/* Calculate size and check validity */
+
+	while(is_list(item)) {
+	    res = CAR(list_val(item));
+	    if (is_not_tuple(res))
+		goto error;
+
+	    kv = tuple_val(res);
+	    if (*kv != make_arityval(2))
+		goto error;
+
+	    size++;
+	    item = CDR(list_val(item));
+	}
+
+	if (is_not_nil(item))
+	    goto error;
+
+	hp    = HAlloc(BIF_P, 3 + 1 + (2 * size));
+	thp   = hp;
+	keys  = make_tuple(hp);
+	*hp++ = make_arityval(size);
+	ks    = hp;
+	hp   += size;
+	mp    = (map_t*)hp;
+	res   = make_map(mp);
+	hp   += MAP_HEADER_SIZE;
+	vs    = hp;
+
+	mp->thing_word = MAP_HEADER;
+	mp->size = size; /* set later, might shrink*/
+	mp->keys = keys;
+
+	if (size == 0)
+	    BIF_RET(res);
+
+	item  = BIF_ARG_1;
+
+	/* first entry */
+	kv    = tuple_val(CAR(list_val(item)));
+	ks[0] = kv[1];
+	vs[0] = kv[2];
+	size  = 1;
+	item  = CDR(list_val(item));
+
+	/* insert sort key/value pairs */
+	while(is_list(item)) {
+
+	    kv = tuple_val(CAR(list_val(item)));
+
+	    /* compare ks backwards
+	     * idx represent word index to be written (hole position).
+	     * We cannot copy the elements when searching since we might
+	     * have an equal key. So we search for just the index first =(
+	     *
+	     * It is perhaps faster to move the values in the first pass.
+	     * Check for uniqueness during insert phase and then have a
+	     * second phace compacting the map if duplicates are found
+	     * during insert. .. or do someother sort .. shell-sort perhaps.
+	     */
+
+	    idx = size;
+
+	    while(idx > 0 && (c = CMP(kv[1],ks[idx-1])) < 0) { idx--; }
+
+	    if (c == 0) {
+		/* last compare was equal,
+		 * i.e. we have to release memory
+		 * and overwrite that key/value
+		 */
+		ks[idx-1] = kv[1];
+		vs[idx-1] = kv[2];
+		unused_size++;
+	    } else {
+		Uint i = size;
+		while(i > idx) {
+		    ks[i] = ks[i-1];
+		    vs[i] = vs[i-1];
+		    i--;
+		}
+		ks[idx] = kv[1];
+		vs[idx] = kv[2];
+		size++;
+	    }
+	    item = CDR(list_val(item));
+	}
+
+	if (unused_size) {
+	    /* the key tuple is embedded in the heap
+	     * write a bignum to clear it.
+	     */
+	    /* release values as normal since they are on the top of the heap */
+
+	    ks[size] = make_pos_bignum_header(unused_size - 1);
+	    HRelease(BIF_P, vs + size + unused_size, vs + size);
+	}
+
+	*thp = make_arityval(size);
+	mp->size = size;
+	BIF_RET(res);
+    }
+
+error:
+
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* map:is_key/2
+ */
+
+BIF_RETTYPE map_is_key_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+	Eterm *ks, key;
+	map_t *mp;
+	Uint n,i;
+
+	mp  = (map_t*)map_val(BIF_ARG_2);
+	key = BIF_ARG_1;
+	n   = map_get_size(mp);
+	ks  = map_get_keys(mp);
+
+	if (n == 0)
+	    BIF_RET(am_false);
+
+	if (is_immed(key)) {
+	    for( i = 0; i < n; i++) {
+		if (ks[i] == key) {
+		    BIF_RET(am_true);
+		}
+	    }
+	}
+
+	for( i = 0; i < n; i++) {
+	    if (eq(ks[i], key)) {
+		BIF_RET(am_true);
+	    }
+	}
+	BIF_RET(am_false);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* map:keys/1
+ */
+
+BIF_RETTYPE map_keys_1(BIF_ALIST_1) {
+    if (is_map(BIF_ARG_1)) {
+	Eterm *hp, *ks, res = NIL;
+	map_t *mp;
+	Uint n;
+
+	mp  = (map_t*)map_val(BIF_ARG_1);
+	n   = map_get_size(mp);
+
+	if (n == 0)
+	    BIF_RET(res);
+
+	hp  = HAlloc(BIF_P, (2 * n));
+	ks  = map_get_keys(mp);
+
+	while(n--) {
+	    res = CONS(hp, ks[n], res); hp += 2;
+	}
+
+	BIF_RET(res);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+/* map:merge/2
+ */
+
+BIF_RETTYPE map_merge_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_1) && is_map(BIF_ARG_2)) {
+	Eterm *hp,*thp;
+	Eterm tup;
+	Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2;
+	map_t *mp1,*mp2,*mp_new;
+	Uint n1,n2,i1,i2,need,unused_size=0;
+	int c = 0;
+
+	mp1  = (map_t*)map_val(BIF_ARG_1);
+	mp2  = (map_t*)map_val(BIF_ARG_2);
+	n1   = map_get_size(mp1);
+	n2   = map_get_size(mp2);
+
+	need = MAP_HEADER_SIZE + 1 + 2*(n1 + n2);
+
+	hp     = HAlloc(BIF_P, need);
+	thp    = hp;
+	tup    = make_tuple(thp);
+	ks     = hp + 1; hp += 1 + n1 + n2;
+	mp_new = (map_t*)hp; hp += MAP_HEADER_SIZE;
+	vs     = hp; hp += n1 + n2;
+
+	mp_new->thing_word = MAP_HEADER;
+	mp_new->size = 0;
+	mp_new->keys = tup;
+
+	i1  = 0; i2 = 0;
+	ks1 = map_get_keys(mp1);
+	vs1 = map_get_values(mp1);
+	ks2 = map_get_keys(mp2);
+	vs2 = map_get_values(mp2);
+
+	while(i1 < n1 && i2 < n2) {
+	    c = CMP(ks1[i1],ks2[i2]);
+	    if ( c == 0) {
+		/* use righthand side arguments map value,
+		 * but advance both maps */
+		*ks++ = ks2[i2];
+		*vs++ = vs2[i2];
+		i1++, i2++, unused_size++;
+	    } else if ( c < 0) {
+		*ks++ = ks1[i1];
+		*vs++ = vs1[i1];
+		i1++;
+	    } else {
+		*ks++ = ks2[i2];
+		*vs++ = vs2[i2];
+		i2++;
+	    }
+	}
+
+	/* copy remaining */
+	while (i1 < n1) {
+	    *ks++ = ks1[i1];
+	    *vs++ = vs1[i1];
+	    i1++;
+	}
+
+	while (i2 < n2) {
+	    *ks++ = ks2[i2];
+	    *vs++ = vs2[i2];
+	    i2++;
+	}
+
+	if (unused_size) {
+	    /* the key tuple is embedded in the heap, write a bignum to clear it.
+	     *
+	     * release values as normal since they are on the top of the heap
+	     * size = n1 + n1 - unused_size
+	     */
+
+	    *ks = make_pos_bignum_header(unused_size - 1);
+	    HRelease(BIF_P, vs + unused_size, vs);
+	}
+
+	mp_new->size = n1 + n2 - unused_size;
+	*thp = make_arityval(n1 + n2 - unused_size);
+
+	BIF_RET(make_map(mp_new));
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+/* map:new/2
+ */
+
+BIF_RETTYPE map_new_0(BIF_ALIST_0) {
+    Eterm* hp;
+    Eterm tup;
+    map_t *mp;
+
+    hp    = HAlloc(BIF_P, (3 + 1));
+    tup   = make_tuple(hp);
+    *hp++ = make_arityval(0);
+
+    mp    = (map_t*)hp;
+    mp->thing_word = MAP_HEADER;
+    mp->size = 0;
+    mp->keys = tup;
+
+    BIF_RET(make_map(mp));
+}
+
+/* map:put/3
+ */
+
 BIF_RETTYPE map_put_3(BIF_ALIST_3) {
     if (is_map(BIF_ARG_3)) {
 	Sint n,i;
@@ -242,3 +581,167 @@ BIF_RETTYPE map_put_3(BIF_ALIST_3) {
 
     BIF_ERROR(BIF_P, BADARG);
 }
+
+/* map:remove/3
+ */
+
+BIF_RETTYPE map_remove_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+	Sint n;
+	Sint found = 0;
+	Uint need;
+	Eterm *thp, *mhp;
+	Eterm *ks, *vs, res, key,tup;
+	map_t *mp = (map_t*)map_val(BIF_ARG_2);
+
+	key = BIF_ARG_1;
+	n   = map_get_size(mp);
+
+	if (n == 0)
+	    BIF_RET(BIF_ARG_2);
+
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
+
+	/* Assume key exists.
+	 * Release allocated if it didn't.
+	 * Allocate key tuple first.
+	 */
+
+	need   = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */
+	thp    = HAlloc(BIF_P, need);
+	mhp    = thp + n;               /* offset with tuple heap size */
+
+	tup    = make_tuple(thp);
+	*thp++ = make_arityval(n - 1);
+
+	res    = make_map(mhp);
+	*mhp++ = MAP_HEADER;
+	*mhp++ = n - 1;
+	*mhp++ = tup;
+
+	if (is_immed(key)) {
+	    while(n--) {
+		if (*ks == key) {
+		    ks++;
+		    vs++;
+		    found = 1;
+		} else {
+		    *mhp++ = *vs++;
+		    *thp++ = *ks++;
+		}
+	    }
+	} else {
+	    while(n--) {
+		if (eq(*ks, key)) {
+		    ks++;
+		    vs++;
+		    found = 1;
+		} else {
+		    *mhp++ = *vs++;
+		    *thp++ = *ks++;
+		}
+	    }
+	}
+
+	if (found)
+	    BIF_RET(res);
+
+	/* Not found, remove allocated memory
+	 * and return previous map.
+	 */
+	HRelease(BIF_P, thp + need, thp);
+	BIF_RET(BIF_ARG_2);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* map:update/3
+ */
+
+BIF_RETTYPE map_update_3(BIF_ALIST_3) {
+    if (is_map(BIF_ARG_3)) {
+	Sint n,i;
+	Sint found = 0;
+	Eterm* hp,*shp;
+	Eterm *ks,*vs, res, key;
+	map_t *mp = (map_t*)map_val(BIF_ARG_3);
+
+	key = BIF_ARG_1;
+	n   = map_get_size(mp);
+
+	if (n == 0) {
+	    BIF_ERROR(BIF_P, BADARG);
+	}
+
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
+
+	/* only allocate for values,
+	 * assume key-tuple will be intact
+	 */
+
+	hp  = HAlloc(BIF_P, 3 + n);
+	shp = hp;
+	res = make_map(hp);
+	*hp++ = MAP_HEADER;
+	*hp++ = n;
+	*hp++ = mp->keys;
+
+	if (is_immed(key)) {
+	    for( i = 0; i < n; i ++) {
+		if (ks[i] == key) {
+		    *hp++ = BIF_ARG_2;
+		    vs++;
+		    found = 1;
+		} else {
+		    *hp++ = *vs++;
+		}
+	    }
+	} else {
+	    for( i = 0; i < n; i ++) {
+		if (eq(ks[i], key)) {
+		    *hp++ = BIF_ARG_2;
+		    vs++;
+		    found = 1;
+		} else {
+		    *hp++ = *vs++;
+		}
+	    }
+	}
+
+	if (found)
+	    BIF_RET(res);
+
+	HRelease(BIF_P, shp + 3 + n, shp);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+
+/* map:values/1
+ */
+
+BIF_RETTYPE map_values_1(BIF_ALIST_1) {
+    if (is_map(BIF_ARG_1)) {
+	Eterm *hp, *vs, res = NIL;
+	map_t *mp;
+	Uint n;
+
+	mp  = (map_t*)map_val(BIF_ARG_1);
+	n   = map_get_size(mp);
+
+	if (n == 0)
+	    BIF_RET(res);
+
+	hp  = HAlloc(BIF_P, (2 * n));
+	vs  = map_get_values(mp);
+
+	while(n--) {
+	    res = CONS(hp, vs[n], res); hp += 2;
+	}
+
+	BIF_RET(res);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index ca8ce0f974..4f0d26e100 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -33,11 +33,12 @@ typedef struct map_s {
  *
  * -----------
  * Eterm   THING	
- * Eterm   Keys -> {K1, K2, K3, ..., Kn} where n = arity
+ * Uint    size
+ * Eterm   Keys -> {K1, K2, K3, ..., Kn} where n = size
  * ----
  * Eterm   V1
  * ...
- * Eterm   Vn, where n = arity
+ * Eterm   Vn, where n = size
  * -----------
  */
 
@@ -59,6 +60,7 @@ typedef struct map_s {
 #define map_get_size(x)        (((map_t*)(x))->size)
 
 #define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
+#define MAP_HEADER_SIZE        (sizeof(map_t) / sizeof(Eterm))
 
 #endif
 
-- 
cgit v1.2.3


From c2702eb35db00ad67f922708eeea48616d908306 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 3 Oct 2013 14:06:57 +0200
Subject: compiler: Implement different instructions for => and :=

---
 erts/emulator/beam/beam_emu.c | 301 +++++++++++++++++++++++++++++-------------
 erts/emulator/beam/ops.tab    |  11 +-
 2 files changed, 219 insertions(+), 93 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1cad31be2c..b666b7c3f7 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -959,8 +959,10 @@ static BeamInstr* apply_fun(Process* p, Eterm fun,
 static Eterm new_fun(Process* p, Eterm* reg,
 		     ErlFunEntry* fe, int num_free) NOINLINE;
 static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE;
-static Eterm update_map(Process* p, Eterm* reg,
-			Eterm map, BeamInstr* I) NOINLINE;
+static Eterm update_map_assoc(Process* p, Eterm* reg,
+			      Eterm map, BeamInstr* I) NOINLINE;
+static Eterm update_map_exact(Process* p, Eterm* reg,
+			      Eterm map, BeamInstr* I) NOINLINE;
 static int has_not_map_field(Eterm map, Eterm key);
 static Eterm get_map_element(Eterm map, Eterm key);
 
@@ -2353,21 +2355,39 @@ void process_main(void)
      Next(4+Arg(3));
  }
 
- OpCase(update_map_jddII): {
+ OpCase(update_map_assoc_jddII): {
      Eterm res;
      Eterm map;
 
      GetArg1(1, map);
      x(0) = r(0);
      SWAPOUT;
-     res = update_map(c_p, reg, map, I);
+     res = update_map_assoc(c_p, reg, map, I);
      SWAPIN;
-     if (res) {
+     if (is_value(res)) {
 	 r(0) = x(0);
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
      } else {
-	 goto lb_Cl_error;
+	 goto badarg;
+     }
+ }
+
+ OpCase(update_map_exact_jddII): {
+     Eterm res;
+     Eterm map;
+
+     GetArg1(1, map);
+     x(0) = r(0);
+     SWAPOUT;
+     res = update_map_exact(c_p, reg, map, I);
+     SWAPIN;
+     if (is_value(res)) {
+	 r(0) = x(0);
+	 StoreResult(res, Arg(2));
+	 Next(5+Arg(4));
+     } else {
+	 goto badarg;
      }
  }
 
@@ -6387,18 +6407,10 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     return make_map(mp);
 }
 
-
-/* This entire instruction will be split into two.
- * 1) update_map_exact (literals) <- this can be much more optimized
- * 2) update_map_assoc (literals)
- * Also update_map is pretty bad code as it stands now.
- */
-
 static Eterm
-update_map(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
+update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 {
     Uint n;
-    Uint i;
     Uint num_old;
     Uint num_updates;
     Uint need;
@@ -6413,7 +6425,7 @@ update_map(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Eterm* kp;
 
     if (is_not_map(map)) {
-	return 0;
+	return THE_NON_VALUE;
     }
 
     old_mp = (map_t *) map_val(map);
@@ -6428,11 +6440,12 @@ update_map(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     }
 
     /*
-     * Allocate heap space for the worst case (i.e. all keys are new).
+     * Allocate heap space for the worst case (i.e. all keys in the
+     * update list are new).
      */
 
     num_updates = Arg(4) / 2;
-    need = 2*(num_old+num_updates) + 4;
+    need = 2*(num_old+num_updates) + 1 + sizeof(map_t) / sizeof(Eterm);
     if (HeapWordsLeft(p) < need) {
 	Uint live = Arg(3);
 	reg[live] = map;
@@ -6442,67 +6455,37 @@ update_map(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     }
 
     /*
-     * Update map, optimistically assuming that there are no
-     * new keys, allowing us to keep the old key tuple.
+     * Build the skeleton for the map, ready to be filled in.
+     *
+     * +-----------------------------------+
+     * | (Space for aritvyal for keys)     | <-----------+
+     * +-----------------------------------+		 |
+     * | (Space for key 1)		   |		 |    <-- kp
+     * +-----------------------------------+		 |
+     *        .				    		 |
+     *        .				    		 |
+     *        .				    		 |
+     * +-----------------------------------+		 |
+     * | (Space for last key)		   |		 |
+     * +-----------------------------------+		 |
+     * | MAP_HEADER			   |		 |
+     * +-----------------------------------+		 |
+     * | (Space for number of keys/values) |		 |
+     * +-----------------------------------+		 |
+     * | Boxed tuple pointer            >----------------+
+     * +-----------------------------------+
+     * | (Space for value 1)		   |                  <-- hp
+     * +-----------------------------------+
      */
 
-    hp = p->htop;
-    E  = p->stop;
-
-    old_vals = map_get_values(old_mp);
-    old_keys = map_get_keys(old_mp);
+    E = p->stop;
+    kp = p->htop + 1;		/* Point to first key */
+    hp = kp + num_old + num_updates;
 
     res = make_map(hp);
-    mp = (map_t *)hp; hp += 3;
+    mp = (map_t *)hp;
+    hp += sizeof(map_t) / sizeof(Eterm);
     mp->thing_word = MAP_HEADER;
-    mp->size = num_old;
-    mp->keys = old_mp->keys;
-
-    ASSERT(num_updates > 0);
-
-    /* Get array of key/value pairs to be updated */
-    new_p = &Arg(5);
-    GET_TERM(*new_p, new_key);
-    
-    n = num_updates;
-
-    for (i = 0; i < num_old; i++) {
-	if (new_key == THE_NON_VALUE || !eq(*old_keys, new_key)) {
-	    /* not same keys */
-	    *hp++ = *old_vals;
-	} else {
-	    GET_TERM(new_p[1], *hp);
-	    hp++;
-	    n--;
-	    if (n == 0) {
-		new_key = THE_NON_VALUE;
-	    } else {
-		new_p += 2;
-		GET_TERM(*new_p, new_key);
-	    }
-	}
-	old_vals++, old_keys++;
-    }
-
-    /*
-     * If we have exhausted the update list we are done.
-     */
-
-    if (n == 0) {
-	p->htop = hp;
-	return res;
-    }
-
-    /*
-     * There were some new keys. We'll have to start over and rebuild
-     * the key tuple too.
-     */
-
-    kp    = p->htop;
-    *kp++ = make_arityval(0);
-
-    res = make_map(hp);
-    mp  = (map_t *)hp; hp += 3;
     mp->keys = make_tuple(kp-1);
 
     old_vals = map_get_values(old_mp);
@@ -6512,20 +6495,28 @@ update_map(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     GET_TERM(*new_p, new_key);
     n = num_updates;
 
+    /*
+     * Fill in keys and values, until we run out of either updates
+     * or old values and keys.
+     */
+
     for (;;) {
 	Eterm key;
 	Sint c;
 
 	ASSERT(kp < (Eterm *)mp);
 	key = *old_keys;
-	if ((c = cmp(key, new_key)) < 0) {
+	if ((c = CMP(key, new_key)) < 0) {
+	    /* Copy old key and value */
 	    *kp++ = key;
 	    *hp++ = *old_vals;
 	    old_keys++, old_vals++, num_old--;
 	} else {		/* Replace or insert new */
-	    *kp++ = new_key;
 	    GET_TERM(new_p[1], *hp++);
-	    if (c == 0) {	/* If replacement */
+	    if (c > 0) {	/* If new new key */
+		*kp++ = new_key;
+	    } else {		/* If replacement */
+		*kp++ = key;
 		old_keys++, old_vals++, num_old--;
 	    }
 	    n--;
@@ -6541,32 +6532,162 @@ update_map(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	}
     }
 
-    while (n-- > 0) {
-	GET_TERM(new_p[0], *kp++);
-	GET_TERM(new_p[1], *hp++);
-	new_p += 2;
-    }
-
     /*
-     * All updates done. Now copy the remaining part of the frame's
-     * keys and values.
+     * At this point, we have run out of either old keys and values,
+     * or the update list. In other words, at least of one n and
+     * num_old must be zero.
      */
 
-    while (num_old-- > 0) {
-	ASSERT(kp < (Eterm *)mp);
-	*kp++ = *old_keys++;
-	*hp++ = *old_vals++;
+    if (n > 0) {
+	/*
+	 * All old keys and values have been copied, but there
+	 * are still new keys and values in the update list that
+	 * must be copied.
+	 */
+	ASSERT(num_old == 0);
+	while (n-- > 0) {
+	    GET_TERM(new_p[0], *kp++);
+	    GET_TERM(new_p[1], *hp++);
+	    new_p += 2;
+	}
+    } else {
+	/*
+	 * All updates are now done. We may still have old
+	 * keys and values that we must copy.
+	 */
+	ASSERT(n == 0);
+	while (num_old-- > 0) {
+	    ASSERT(kp < (Eterm *)mp);
+	    *kp++ = *old_keys++;
+	    *hp++ = *old_vals++;
+	}
     }
+
+    /*
+     * Calculate how many values that are unused at the end of the
+     * key tuple and fill it out with a bignum header.
+     */
     if ((n = (Eterm *)mp - kp) > 0) {
 	*kp = make_pos_bignum_header(n-1);
     }
+
+    /*
+     * Fill in the size of the map in both the key tuple and in the map.
+     */
+
     n = kp - p->htop - 1;	/* Actual number of keys/values */
     *p->htop = make_arityval(n);
-    mp->thing_word = MAP_HEADER;
     mp->size = n;
     p->htop = hp;
     return res;
 }
+
+/*
+ * Update values for keys that already exist in the map.
+ */
+
+static Eterm
+update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
+{
+    Uint n;
+    Uint i;
+    Uint num_old;
+    Uint need;
+    map_t *old_mp, *mp;
+    Eterm res;
+    Eterm* hp;
+    Eterm* E;
+    Eterm* old_keys;
+    Eterm* old_vals;
+    Eterm* new_p;
+    Eterm new_key;
+
+    if (is_not_map(map)) {
+	return THE_NON_VALUE;
+    }
+
+    old_mp = (map_t *) map_val(map);
+    num_old = map_get_size(old_mp);
+
+    /*
+     * If the old map is empty, create a new map.
+     */
+
+    if (num_old == 0) {
+	return new_map(p, reg, I+1);
+    }
+
+    /*
+     * Allocate the exact heap space needed.
+     */
+
+    need = num_old + sizeof(map_t) / sizeof(Eterm);
+    if (HeapWordsLeft(p) < need) {
+	Uint live = Arg(3);
+	reg[live] = map;
+	erts_garbage_collect(p, need, reg, live+1);
+	map      = reg[live];
+	old_mp   = (map_t *)map_val(map);
+    }
+
+    /*
+     * Update map, keeping the old key tuple.
+     */
+
+    hp = p->htop;
+    E = p->stop;
+
+    old_vals = map_get_values(old_mp);
+    old_keys = map_get_keys(old_mp);
+
+    res = make_map(hp);
+    mp = (map_t *)hp;
+    hp += sizeof(map_t) / sizeof(Eterm);
+    mp->thing_word = MAP_HEADER;
+    mp->size = num_old;
+    mp->keys = old_mp->keys;
+
+    /* Get array of key/value pairs to be updated */
+    new_p = &Arg(5);
+    GET_TERM(*new_p, new_key);
+
+    /* Update all values */
+    n = Arg(4) / 2;		/* Number of values to be updated */
+    ASSERT(n > 0);
+    for (i = 0; i < num_old; i++) {
+	if (!EQ(*old_keys, new_key)) {
+	    /* Not same keys */
+	    *hp++ = *old_vals;
+	} else {
+	    GET_TERM(new_p[1], *hp);
+	    hp++;
+	    n--;
+	    if (n == 0) {
+		/*
+		 * All updates done. Copy remaining values
+		 * and return the result.
+		 */
+		for (i++, old_vals++; i < num_old; i++) {
+		    *hp++ = *old_vals++;
+		}
+		ASSERT(hp == p->htop + need);
+		p->htop = hp;
+		return res;
+	    } else {
+		new_p += 2;
+		GET_TERM(*new_p, new_key);
+	    }
+	}
+	old_vals++, old_keys++;
+    }
+
+    /*
+     * Updates left. That means that at least one the keys in the
+     * update list did not previously exist.
+     */
+    ASSERT(hp == p->htop + need);
+    return THE_NON_VALUE;
+}
 #undef GET_TERM
 
 int catchlevel(Process *p)
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b89bdb2e3a..f35997efee 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1474,11 +1474,16 @@ apply_last I P
 # has_map_field Fail Src Key => jump Fail
 # get_map_element Fail Src Key Dst => jump Fail
 
-put_map F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
-put_map F Src Dst Live Size Rest=* => update_map F Src Dst Live Size Rest
+put_map_assoc F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
+put_map_exact F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
+put_map_assoc F Src Dst Live Size Rest=* => \
+	       update_map_assoc F Src Dst Live Size Rest
+put_map_exact F Src Dst Live Size Rest=* => \
+	      update_map_exact F Src Dst Live Size Rest
 
 new_map j d I I
-update_map j d d I I
+update_map_assoc j d d I I
+update_map_exact j d d I I
 
 is_map Fail cq => jump Fail
 
-- 
cgit v1.2.3


From 813f61de8e7872481a0369de3297596c1b11881d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 9 Oct 2013 15:46:03 +0200
Subject: erts: Fixup map instructions for halfword

---
 erts/emulator/beam/beam_emu.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b666b7c3f7..fe2e196785 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6353,7 +6353,7 @@ static Eterm get_map_element(Eterm map, Eterm key)
 
 #define GET_TERM(term, dest)					\
 do {								\
-    Eterm src = term;						\
+    Eterm src = (Eterm)(term);					\
     switch (src & _TAG_IMMED1_MASK) {				\
     case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
 	dest = x(0);						\
@@ -6380,7 +6380,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     Eterm keys;
     Eterm *mhp,*thp;
     Eterm *E;
-    Eterm *tp;
+    BeamInstr *ptr;
     map_t *mp;
 
     if (HeapWordsLeft(p) < need) {
@@ -6390,18 +6390,18 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     thp    = p->htop;
     mhp    = thp + 1 + n/2;
     E      = p->stop;
-    tp     = &Arg(4);
+    ptr    = &Arg(4);
     keys   = make_tuple(thp);
     *thp++ = make_arityval(n/2);
 
-    mp = (map_t *)mhp; mhp += 3;
+    mp = (map_t *)mhp; mhp += MAP_HEADER_SIZE;
     mp->thing_word = MAP_HEADER;
     mp->size = n/2;
     mp->keys = keys;
 
     for (i = 0; i < n/2; i++) {
-	GET_TERM(*tp++, *thp++);
-	GET_TERM(*tp++, *mhp++);
+	GET_TERM(*ptr++, *thp++);
+	GET_TERM(*ptr++, *mhp++);
     }
     p->htop = mhp;
     return make_map(mp);
@@ -6420,7 +6420,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Eterm* E;
     Eterm* old_keys;
     Eterm* old_vals;
-    Eterm* new_p;
+    BeamInstr* new_p;
     Eterm new_key;
     Eterm* kp;
 
@@ -6445,7 +6445,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      */
 
     num_updates = Arg(4) / 2;
-    need = 2*(num_old+num_updates) + 1 + sizeof(map_t) / sizeof(Eterm);
+    need = 2*(num_old+num_updates) + 1 + MAP_HEADER_SIZE;
     if (HeapWordsLeft(p) < need) {
 	Uint live = Arg(3);
 	reg[live] = map;
@@ -6484,7 +6484,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     res = make_map(hp);
     mp = (map_t *)hp;
-    hp += sizeof(map_t) / sizeof(Eterm);
+    hp += MAP_HEADER_SIZE;
     mp->thing_word = MAP_HEADER;
     mp->keys = make_tuple(kp-1);
 
@@ -6599,7 +6599,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Eterm* E;
     Eterm* old_keys;
     Eterm* old_vals;
-    Eterm* new_p;
+    BeamInstr* new_p;
     Eterm new_key;
 
     if (is_not_map(map)) {
@@ -6621,7 +6621,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      * Allocate the exact heap space needed.
      */
 
-    need = num_old + sizeof(map_t) / sizeof(Eterm);
+    need = num_old + MAP_HEADER_SIZE;
     if (HeapWordsLeft(p) < need) {
 	Uint live = Arg(3);
 	reg[live] = map;
@@ -6642,7 +6642,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     res = make_map(hp);
     mp = (map_t *)hp;
-    hp += sizeof(map_t) / sizeof(Eterm);
+    hp += MAP_HEADER_SIZE;
     mp->thing_word = MAP_HEADER;
     mp->size = num_old;
     mp->keys = old_mp->keys;
-- 
cgit v1.2.3


From eb218eb375d353bcd78ae895805d1d45f927b3a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 2 Oct 2013 16:19:22 +0200
Subject: erts: Teach term_to_binary/1 and binary_to_term/1 Map encoding

---
 erts/emulator/beam/external.c | 197 ++++++++++++++++++++++++++++++++++++++----
 erts/emulator/beam/external.h |   1 +
 2 files changed, 183 insertions(+), 15 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 5e7a5cab6e..a31b2731be 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -42,6 +42,7 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "erl_zlib.h"
+#include "erl_map.h"
 
 #ifdef HIPE
 #include "hipe_mode_switch.h"
@@ -2555,6 +2556,38 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    }
 	    break;
 
+	case MAP_DEF:
+	    {
+		map_t *mp = (map_t*)map_val(obj);
+		Uint size = map_get_size(mp);
+		Eterm *mptr;
+
+		*ep++ = MAP_EXT;
+		put_int32(size, ep); ep += 4;
+
+		/* Push values first */
+		if (size > 0) {
+		    mptr = map_get_values(mp);
+		    for (i = size-1; i >= 1; i--) {
+			WSTACK_PUSH(s, ENC_TERM);
+			WSTACK_PUSH(s, (UWord) mptr[i]);
+		    }
+
+		    WSTACK_PUSH(s, ENC_TERM);
+		    WSTACK_PUSH(s, (UWord) mptr[0]);
+
+		    mptr = map_get_keys(mp);
+		    for (i = size-1; i >= 1; i--) {
+			WSTACK_PUSH(s, ENC_TERM);
+			WSTACK_PUSH(s, (UWord) mptr[i]);
+		    }
+
+		    obj = mptr[0];
+		    goto L_jump_start;
+		}
+	    }
+	    break;
+
 	case FLOAT_DEF:
 	    GET_DOUBLE(obj, f);
 	    if (dflags & DFLAG_NEW_FLOATS) {
@@ -2845,6 +2878,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
     int n;
     ErtsAtomEncoding char_enc;
     register Eterm* hp;        /* Please don't take the address of hp */
+    Eterm *maps_head = NULL; /* for validation of maps */
     Eterm* next;
     SWord reds;
 
@@ -3469,6 +3503,65 @@ dec_term_atom_common:
 		break;
 	    }
 	    break;
+	case MAP_EXT:
+	    {
+		map_t *mp;
+		Uint32 size,n;
+		Eterm *kptr,*vptr;
+		Eterm keys;
+
+		size = get_int32(ep); ep += 4;
+
+		keys  = make_tuple(hp);
+		*hp++ = make_arityval(size);
+		kptr  = hp;
+		hp   += size;
+
+		mp    = (map_t*)hp;
+		hp   += MAP_HEADER_SIZE;
+		vptr  = hp;
+		hp   += size;
+
+		/* kptr, first word for keys
+		 * vptr, first word for values
+		 */
+
+		/*
+		 * Use thing_word to link through decoded maps.
+		 * The list of maps is for later validation.
+		 */
+
+		mp->thing_word = (Eterm) COMPRESS_POINTER(maps_head);
+		maps_head      = (Eterm *) mp;
+
+		mp->size       = size;
+		mp->keys       = keys;
+		*objp          = make_map(mp);
+
+		/* We assume the map is wellformed, meaning:
+		 * - ascending key order
+		 * - unique keys
+		 */
+
+		objp  = vptr + size - 1;
+		n     = size;
+
+		while (n-- > 0) {
+		    *objp = (Eterm) COMPRESS_POINTER(next);
+		    next  = objp;
+		    objp--;
+		}
+
+		objp  = kptr + size - 1;
+		n     = size;
+
+		while (n-- > 0) {
+		    *objp = (Eterm) COMPRESS_POINTER(next);
+		    next  = objp;
+		    objp--;
+		}
+	    }
+	    break;
 	case NEW_FUN_EXT:
 	    {
 		ErlFunThing* funp = (ErlFunThing *) hp;
@@ -3678,21 +3771,7 @@ dec_term_atom_common:
 	    }
 
 	default:
-	error:
-	    /* UNDO:
-	     * Must unlink all off-heap objects that may have been
-	     * linked into the process. 
-	     */
-	    if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */
-		hp = *hpp;   /* the largest must be the freshest */
-	    }
-	    undo_offheap_in_area(off_heap, hp_saved, hp);
-	    *hpp = hp_saved;
-            if (ctx) {
-                ctx->state = B2TDecodeFail;
-		ctx->reds = reds;
-            }
-            return NULL;
+	    goto error;
 	}
 
         if (--reds <= 0) {
@@ -3710,12 +3789,53 @@ dec_term_atom_common:
             }
         }
     }
+
+    /* Iterate through all the maps and check for validity
+     * - done here for when we know it is complete.
+     */
+
+    while (maps_head) {
+	Eterm *keys;
+	Sint  arity;
+
+	next  = (Eterm *)(EXPAND_POINTER(*maps_head));
+	keys  = tuple_val(*(maps_head + 2));
+	arity = arityval(*keys++);
+
+	while(arity-- > 1) {
+	    if (CMP(keys[arity-1],keys[arity]) >= 0) {
+		goto error;
+	    }
+	}
+
+	*maps_head = MAP_HEADER;
+	maps_head  = next;
+    }
+
     if (ctx) {
         ctx->state = B2TDone;
 	ctx->reds = reds;
     }
+
     *hpp = hp;
     return ep;
+
+error:
+    /* UNDO:
+     * Must unlink all off-heap objects that may have been
+     * linked into the process. 
+     */
+    if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */
+	hp = *hpp;   /* the largest must be the freshest */
+    }
+    undo_offheap_in_area(off_heap, hp_saved, hp);
+    *hpp = hp_saved;
+    if (ctx) {
+	ctx->state = B2TDecodeFail;
+	ctx->reds = reds;
+    }
+        
+    return NULL;
 }
 
 /* returns the number of bytes needed to encode an object
@@ -3885,6 +4005,46 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		goto outer_loop;
 	    }
 	    break;
+	case MAP_DEF:
+	    {
+		map_t *mp = (map_t*)map_val(obj);
+		Uint size = map_get_size(mp);
+		Uint i;
+		Eterm *ptr;
+
+		result += 1 + 4; /* tag + 4 bytes size */
+
+		/* push values first */
+		ptr = map_get_values(mp);
+		i   = size;
+		while(i--) {
+		    if (is_list(*ptr)) {
+			if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) {
+			    result += m + 2 + 1;
+			} else {
+			    result += 5;
+			}
+		    }
+		    ESTACK_PUSH(s,*ptr);
+		    ++ptr;
+		}
+
+		ptr = map_get_keys(mp);
+		i   = size;
+		while(i--) {
+		    if (is_list(*ptr)) {
+			if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) {
+			    result += m + 2 + 1;
+			} else {
+			    result += 5;
+			}
+		    }
+		    ESTACK_PUSH(s,*ptr);
+		    ++ptr;
+		}
+		goto outer_loop;
+	    }
+	    break;
 	case FLOAT_DEF:
 	    if (dflags & DFLAG_NEW_FLOATS) {
 		result += 9;
@@ -4175,6 +4335,13 @@ init_done:
 	    ADDTERMS(n);
 	    heap_size += n + 1;
 	    break;
+	case MAP_EXT:
+	    CHKSIZE(4);
+	    n = get_int32(ep);
+	    ep += 4;
+	    ADDTERMS(2*n);
+	    heap_size += 3 + n + 1 + n;
+	    break;
 	case STRING_EXT:
 	    CHKSIZE(2);
 	    n = get_int16(ep);
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 83001b2c7e..bf00958eb1 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -50,6 +50,7 @@
 #define LARGE_BIG_EXT     'o'
 #define NEW_FUN_EXT       'p'
 #define EXPORT_EXT        'q'
+#define MAP_EXT           't'
 #define FUN_EXT           'u'
 #define ATOM_UTF8_EXT     'v'
 #define SMALL_ATOM_UTF8_EXT 'w'
-- 
cgit v1.2.3


From ac2954982d6089b0970798581b2230bdd1a065d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 15 Oct 2013 17:21:30 +0200
Subject: erts: Add Maps to erlang:phash/2 and erlang:hash/2

The hashing a map in these functions uses the same strategy
as the other terms. The exception being a prime number with size
so we do not get erlang:phash(#{}) -> 1 which would be the same
as erlang:phash({}) and erlang:phash(<<>>). Same argument for
erlang:hash/1.
---
 erts/emulator/beam/utils.c | 53 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 47 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index bc7e91295d..3774f379cc 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -735,6 +735,8 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
 #define FUNNY_NUMBER10 268440479
 #define FUNNY_NUMBER11 268440577
 #define FUNNY_NUMBER12 268440581
+#define FUNNY_NUMBER13 268440593
+#define FUNNY_NUMBER14 268440611
 
 static Uint32
 hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash)
@@ -787,7 +789,7 @@ Uint32 make_hash(Eterm term_arg)
 
     /* Must not collide with the real tag_val_def's: */
 #define MAKE_HASH_TUPLE_OP 0x11
-#define MAKE_HASH_FUN_OP 0x12
+#define MAKE_HASH_TERM_ARRAY_OP 0x12
 #define MAKE_HASH_CDR_PRE_OP 0x13
 #define	MAKE_HASH_CDR_POST_OP 0x14
 
@@ -878,7 +880,7 @@ tail_recur:
 	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
 	    if (num_free > 0) {
 		if (num_free > 1) {
-		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP);
 		}
 		term = funp->env[0];
 		goto tail_recur;
@@ -968,6 +970,24 @@ tail_recur:
 	    hash *= is_neg ? FUNNY_NUMBER4 : FUNNY_NUMBER3;
 	    break;
 	}	
+    case MAP_DEF:
+	{
+	    map_t *mp = (map_t *)map_val(term);
+	    int size  = map_get_size(mp);
+	    Eterm *ks = map_get_keys(mp);
+	    Eterm *vs = map_get_values(mp);
+
+	    /* Use a prime with size to remedy some of
+	     * the {} and <<>> hash problems */
+	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size;
+	    if (size == 0)
+		break;
+
+	    /* push values first */
+	    WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
+	    WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
+	    break;
+	}
     case TUPLE_DEF: 
 	{
 	    Eterm* ptr = tuple_val(term);
@@ -977,7 +997,7 @@ tail_recur:
 	    op = MAKE_HASH_TUPLE_OP;	    
 	}/*fall through*/
     case MAKE_HASH_TUPLE_OP:
-    case MAKE_HASH_FUN_OP:
+    case MAKE_HASH_TERM_ARRAY_OP:
 	{
 	    Uint i = (Uint) WSTACK_POP(stack);
 	    Eterm* ptr = (Eterm*) WSTACK_POP(stack);
@@ -1206,6 +1226,9 @@ make_hash2(Eterm term)
 		    tmp = vs[i];
 		    ESTACK_PUSH(s, tmp);
 		}
+		/* We do not want to expose the tuple representation.
+		 * Do not push the keys as a tuple.
+		 */
 		for (i = size - 1; i >= 1; i--) {
 		    tmp = ks[i];
 		    ESTACK_PUSH(s, tmp);
@@ -1514,7 +1537,7 @@ tail_recur:
 	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
 	    if (num_free > 0) {
 		if (num_free > 1) {
-		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP);
 		}
 		term = funp->env[0];
 		goto tail_recur;
@@ -1627,6 +1650,24 @@ tail_recur:
 	}
 	break;
 
+    case MAP_DEF:
+	{
+	    map_t *mp = (map_t *)map_val(term);
+	    int size  = map_get_size(mp);
+	    Eterm *ks = map_get_keys(mp);
+	    Eterm *vs = map_get_values(mp);
+
+	    /* Use a prime with size to remedy some of
+	     * the {} and <<>> hash problems */
+	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size;
+	    if (size == 0)
+		break;
+
+	    /* push values first */
+	    WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
+	    WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
+	    break;
+	}
     case TUPLE_DEF: 
 	{
 	    Eterm* ptr = tuple_val(term);
@@ -1636,7 +1677,7 @@ tail_recur:
 	    op = MAKE_HASH_TUPLE_OP;
 	}/*fall through*/ 
     case MAKE_HASH_TUPLE_OP:
-    case MAKE_HASH_FUN_OP:
+    case MAKE_HASH_TERM_ARRAY_OP:
 	{
 	    Uint i = (Uint) WSTACK_POP(stack);
 	    Eterm* ptr = (Eterm*) WSTACK_POP(stack);
@@ -1664,7 +1705,7 @@ tail_recur:
     return hash;
     
 #undef MAKE_HASH_TUPLE_OP
-#undef MAKE_HASH_FUN_OP
+#undef MAKE_HASH_TERM_ARRAY_OP
 #undef MAKE_HASH_CDR_PRE_OP
 #undef MAKE_HASH_CDR_POST_OP
 }
-- 
cgit v1.2.3


From ab54731c41f28a21bce526249e582b56106d4965 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 16 Dec 2013 17:07:54 +0100
Subject: erts: erlang:phash2 should hash Maps independent of order

---
 erts/emulator/beam/utils.c | 63 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 55 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 3774f379cc..8958d334ae 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1091,9 +1091,11 @@ Uint32
 make_hash2(Eterm term)
 {
     Uint32 hash;
+    Uint32 hash_xor_keys   = 0;
+    Uint32 hash_xor_values = 0;
     DeclareTmpHeapNoproc(tmp_big,2);
 
-/* (HCONST * {2, ..., 14}) mod 2^32 */
+/* (HCONST * {2, ..., 16}) mod 2^32 */
 #define HCONST_2 0x3c6ef372UL
 #define HCONST_3 0xdaa66d2bUL
 #define HCONST_4 0x78dde6e4UL
@@ -1110,6 +1112,10 @@ make_hash2(Eterm term)
 #define HCONST_15 0x454021d7UL
 #define HCONST_16 0xe3779b90UL
 
+#define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF))
+#define HASH_MAP_KEY  (_make_header(2,_TAG_HEADER_REF))
+#define HASH_MAP_VAL  (_make_header(3,_TAG_HEADER_REF))
+
 #define UINT32_HASH_2(Expr1, Expr2, AConst)       \
          do {                                     \
 	    Uint32 a,b;                           \
@@ -1204,36 +1210,45 @@ make_hash2(Eterm term)
 		UINT32_HASH(arity, HCONST_9);
 		if (arity == 0) /* Empty tuple */ 
 		    goto hash2_common;
-		for (i = arity; i >= 2; i--) {
+		for (i = arity; i >= 1; i--) {
 		    tmp = elem[i];
 		    ESTACK_PUSH(s, tmp);
 		}
-		term = elem[1];
+		goto hash2_common;
 	    }
 	    break;
 	    case MAP_SUBTAG:
 	    {
 		map_t *mp = (map_t *)map_val(term);
 		int i;
-		int size = map_get_size(mp);
+		int size  = map_get_size(mp);
 		Eterm *ks = map_get_keys(mp);
 		Eterm *vs = map_get_values(mp);
 		UINT32_HASH(size, HCONST_16);
-		if (size == 0) /* Empty Map */
+		if (size == 0) {
 		    goto hash2_common;
-
+		}
+		ESTACK_PUSH(s, hash_xor_values);
+		ESTACK_PUSH(s, hash_xor_keys);
+		ESTACK_PUSH(s, hash);
+		ESTACK_PUSH(s, HASH_MAP_TAIL);
+		hash = 0;
+		hash_xor_keys = 0;
+		hash_xor_values = 0;
 		for (i = size - 1; i >= 0; i--) {
 		    tmp = vs[i];
+		    ESTACK_PUSH(s, HASH_MAP_VAL);
 		    ESTACK_PUSH(s, tmp);
 		}
 		/* We do not want to expose the tuple representation.
 		 * Do not push the keys as a tuple.
 		 */
-		for (i = size - 1; i >= 1; i--) {
+		for (i = size - 1; i >= 0; i--) {
 		    tmp = ks[i];
+		    ESTACK_PUSH(s, HASH_MAP_KEY);
 		    ESTACK_PUSH(s, tmp);
 		}
-		term = ks[0];
+		goto hash2_common;
 	    }
 	    break;
 	    case EXPORT_SUBTAG:
@@ -1427,15 +1442,47 @@ make_hash2(Eterm term)
 	default:
 	    erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
 	hash2_common:
+
+	    /* Uint32 hash always has the hash value of the previous term,
+	     * compounded or otherwise.
+	     */
+
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
 		UnUseTmpHeapNoproc(2);
 		return hash;
 	    }
+
 	    term = ESTACK_POP(s);
+
+	    switch (term) {
+		case HASH_MAP_TAIL: {
+		    hash = (Uint32) ESTACK_POP(s);
+		    UINT32_HASH(hash_xor_keys, HCONST_16);
+		    UINT32_HASH(hash_xor_values, HCONST_16);
+		    hash_xor_keys = (Uint32) ESTACK_POP(s);
+		    hash_xor_values = (Uint32) ESTACK_POP(s);
+		    goto hash2_common;
+		}
+		case HASH_MAP_KEY:
+		    hash_xor_keys ^= hash;
+		    hash = 0;
+		    goto hash2_common;
+		case HASH_MAP_VAL:
+		    hash_xor_values ^= hash;
+		    hash = 0;
+		    goto hash2_common;
+		default:
+		    break;
+	    }
 	}
     }
     }
+
+#undef HASH_MAP_TAIL
+#undef HASH_MAP_KEY
+#undef HASH_MAP_VAL
+
 #undef UINT32_HASH_2
 #undef UINT32_HASH
 #undef SINT32_HASH
-- 
cgit v1.2.3


From 457b29c9ddd2b338e80916a5e7bad8dfe4b36ffc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 25 Oct 2013 16:28:50 +0200
Subject: erts,stdlib: Teach matchspec compiler map guards

---
 erts/emulator/beam/erl_db_util.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index d903053aa4..3986ccd4d3 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -565,6 +565,12 @@ static DMCGuardBif guard_tab[] =
 	1,
 	DBIF_ALL
     },
+    {
+	am_is_map,
+	&is_map_1,
+	1,
+	DBIF_ALL
+    },
     {
 	am_is_binary,
 	&is_binary_1,
@@ -631,6 +637,12 @@ static DMCGuardBif guard_tab[] =
 	1,
 	DBIF_ALL
     },
+    {
+	am_map_size,
+	&map_size_1,
+	1,
+	DBIF_ALL
+    },
     {
 	am_bit_size,
 	&bit_size_1,
-- 
cgit v1.2.3


From 6fdad74f41803089a0f9026c98f319daecda9a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 24 Oct 2013 19:06:34 +0200
Subject: erts,stdlib: Change map module name to maps

Name conforms to EEP.
---
 erts/emulator/beam/bif.tab   | 24 ++++++-------
 erts/emulator/beam/erl_map.c | 82 ++++++++++++++++++++++----------------------
 2 files changed, 53 insertions(+), 53 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 306861a4c5..b623e47b9a 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -580,18 +580,18 @@ bif re:inspect/2
 
 ubif erlang:is_map/1
 ubif erlang:map_size/1
-bif map:to_list/1
-bif map:find/2
-bif map:get/2
-bif map:from_list/1
-bif map:is_key/2
-bif map:keys/1
-bif map:merge/2
-bif map:new/0
-bif map:put/3
-bif map:remove/2
-bif map:update/3
-bif map:values/1
+bif maps:to_list/1
+bif maps:find/2
+bif maps:get/2
+bif maps:from_list/1
+bif maps:is_key/2
+bif maps:keys/1
+bif maps:merge/2
+bif maps:new/0
+bif maps:put/3
+bif maps:remove/2
+bif maps:update/3
+bif maps:values/1
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 6fd60f0689..27734276f9 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -38,25 +38,25 @@
  * - erlang:is_map/1
  * - erlang:map_size/1
  *
- * - map:find/2
- * - map:from_list/1
- * - map:get/2
- * - map:is_key/2
- * - map:keys/1
- * - map:merge/2
- * - map:new/0
- * - map:put/3
- * - map:remove/2
- * - map:to_list/1
- * - map:update/3
- * - map:values/1
+ * - maps:find/2
+ * - maps:from_list/1
+ * - maps:get/2
+ * - maps:is_key/2
+ * - maps:keys/1
+ * - maps:merge/2
+ * - maps:new/0
+ * - maps:put/3
+ * - maps:remove/2
+ * - maps:to_list/1
+ * - maps:update/3
+ * - maps:values/1
  *
  * TODO:
- * - map:foldl/3
- * - map:foldr/3
- * - map:map/3
- * - map:size/1
- * - map:without/2
+ * - maps:foldl/3
+ * - maps:foldr/3
+ * - maps:map/3
+ * - maps:size/1
+ * - maps:without/2
  *
  */
 
@@ -80,10 +80,10 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* map:to_list/1
+/* maps:to_list/1
  */
 
-BIF_RETTYPE map_to_list_1(BIF_ALIST_1) {
+BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
 	Uint n;
 	Eterm* hp;
@@ -107,11 +107,11 @@ BIF_RETTYPE map_to_list_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* map:find/2
+/* maps:find/2
  * return value if key *equals* a key in the map
  */
 
-BIF_RETTYPE map_find_2(BIF_ALIST_2) {
+BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
 	Eterm *hp, *ks,*vs, key, res;
 	map_t *mp;
@@ -137,12 +137,12 @@ BIF_RETTYPE map_find_2(BIF_ALIST_2) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
-/* map:get/2
+/* maps:get/2
  * return value if key *matches* a key in the map
  * exception bad_key if none matches
  */
 
-BIF_RETTYPE map_get_2(BIF_ALIST_2) {
+BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
 	Eterm *hp, *ks,*vs, key, error;
 	map_t *mp;
@@ -184,11 +184,11 @@ error:
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* map:from_list/1
+/* maps:from_list/1
  * List may be unsorted [{K,V}]
  */
 
-BIF_RETTYPE map_from_list_1(BIF_ALIST_1) {
+BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) {
     Eterm *kv, item = BIF_ARG_1;
     Eterm *hp, *thp,*vs, *ks, keys, res;
     map_t *mp;
@@ -305,10 +305,10 @@ error:
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* map:is_key/2
+/* maps:is_key/2
  */
 
-BIF_RETTYPE map_is_key_2(BIF_ALIST_2) {
+BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
 	Eterm *ks, key;
 	map_t *mp;
@@ -340,10 +340,10 @@ BIF_RETTYPE map_is_key_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* map:keys/1
+/* maps:keys/1
  */
 
-BIF_RETTYPE map_keys_1(BIF_ALIST_1) {
+BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
 	Eterm *hp, *ks, res = NIL;
 	map_t *mp;
@@ -366,10 +366,10 @@ BIF_RETTYPE map_keys_1(BIF_ALIST_1) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
-/* map:merge/2
+/* maps:merge/2
  */
 
-BIF_RETTYPE map_merge_2(BIF_ALIST_2) {
+BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_1) && is_map(BIF_ARG_2)) {
 	Eterm *hp,*thp;
 	Eterm tup;
@@ -452,10 +452,10 @@ BIF_RETTYPE map_merge_2(BIF_ALIST_2) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
-/* map:new/2
+/* maps:new/2
  */
 
-BIF_RETTYPE map_new_0(BIF_ALIST_0) {
+BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
     Eterm* hp;
     Eterm tup;
     map_t *mp;
@@ -472,10 +472,10 @@ BIF_RETTYPE map_new_0(BIF_ALIST_0) {
     BIF_RET(make_map(mp));
 }
 
-/* map:put/3
+/* maps:put/3
  */
 
-BIF_RETTYPE map_put_3(BIF_ALIST_3) {
+BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
     if (is_map(BIF_ARG_3)) {
 	Sint n,i;
 	Sint c = 0;
@@ -582,10 +582,10 @@ BIF_RETTYPE map_put_3(BIF_ALIST_3) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* map:remove/3
+/* maps:remove/3
  */
 
-BIF_RETTYPE map_remove_2(BIF_ALIST_2) {
+BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
 	Sint n;
 	Sint found = 0;
@@ -656,10 +656,10 @@ BIF_RETTYPE map_remove_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* map:update/3
+/* maps:update/3
  */
 
-BIF_RETTYPE map_update_3(BIF_ALIST_3) {
+BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
     if (is_map(BIF_ARG_3)) {
 	Sint n,i;
 	Sint found = 0;
@@ -719,10 +719,10 @@ BIF_RETTYPE map_update_3(BIF_ALIST_3) {
 }
 
 
-/* map:values/1
+/* maps:values/1
  */
 
-BIF_RETTYPE map_values_1(BIF_ALIST_1) {
+BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
 	Eterm *hp, *vs, res = NIL;
 	map_t *mp;
-- 
cgit v1.2.3


From e0eb6d5bafcebc1c24b0a538e50a1d55a3724f01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 8 Nov 2013 15:34:05 +0100
Subject: erts: Add NIFs for Maps

- int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM map)
- int enif_get_map_size(ErlNifEnv *env, ERL_NIF_TERM, int*)
- ERL_NIF_TERM enif_make_new_map(ErlNifEnv *env)
- int enif_make_map_put(ErlNifEnv *env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out)
- int enif_get_map_value(ErlNifEnv *env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value)
- int enif_find_map_value(ErlNifEnv *env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value)
- int enif_make_map_update(ErlNifEnv *env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out)
- int enif_make_map_remove(ErlNifEnv *env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM* map_out)

- int  enif_map_iterator_create(ErlNifEnv *env, ERL_NIF_TERM map, ErlNifMapIterator *iter)
- void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter)
- int  enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
- int  enif_map_iterator_get_pair(ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value)
---
 erts/emulator/beam/erl_map.c           | 443 ++++++++++++++++++---------------
 erts/emulator/beam/erl_map.h           |   6 +
 erts/emulator/beam/erl_nif.c           | 209 ++++++++++++++++
 erts/emulator/beam/erl_nif.h           |  26 +-
 erts/emulator/beam/erl_nif_api_funcs.h |  33 +++
 5 files changed, 511 insertions(+), 206 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 27734276f9..98aeee634b 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -111,28 +111,39 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
  * return value if key *equals* a key in the map
  */
 
-BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
-	Eterm *hp, *ks,*vs, key, res;
-	map_t *mp;
-	Uint n,i;
+int erts_maps_find(Eterm key, Eterm map, Eterm *value) {
 
-	mp  = (map_t*)map_val(BIF_ARG_2);
-	key = BIF_ARG_1;
-	n   = map_get_size(mp);
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
+    Eterm *ks,*vs;
+    map_t *mp;
+    Uint n,i;
 
-	for( i = 0; i < n; i++) {
-	    if (CMP(ks[i], key)==0) {
-		hp    = HAlloc(BIF_P, 3);
-		res   = make_tuple(hp);
-		*hp++ = make_arityval(2);
-		*hp++ = am_ok;
-		*hp++ = vs[i];
-		BIF_RET(res);
-	    }
+    mp  = (map_t*)map_val(map);
+    n   = map_get_size(mp);
+    ks  = map_get_keys(mp);
+    vs  = map_get_values(mp);
+
+    for( i = 0; i < n; i++) {
+	if (CMP(ks[i], key)==0) {
+	    *value = vs[i];
+	    return 1;
+	}
+    }
+    return 0;
+}
+
+BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+	Eterm *hp, value,res;
+
+	if (erts_maps_find(BIF_ARG_1, BIF_ARG_2, &value)) {
+	    hp    = HAlloc(BIF_P, 3);
+	    res   = make_tuple(hp);
+	    *hp++ = make_arityval(2);
+	    *hp++ = am_ok;
+	    *hp++ = value;
+	    BIF_RET(res);
 	}
+
 	BIF_RET(am_error);
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -142,43 +153,54 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
  * exception bad_key if none matches
  */
 
-BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
-	Eterm *hp, *ks,*vs, key, error;
-	map_t *mp;
-	Uint n,i;
-	char *s_error;
 
-	mp  = (map_t*)map_val(BIF_ARG_2);
-	key = BIF_ARG_1;
-	n   = map_get_size(mp);
-	
-	if (n == 0)
-	    goto error;
+int erts_maps_get(Eterm key, Eterm map, Eterm *value) {
+    Eterm *ks,*vs;
+    map_t *mp;
+    Uint n,i;
 
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
+    mp  = (map_t*)map_val(map);
+    n   = map_get_size(mp);
 
-	if (is_immed(key)) {
-	    for( i = 0; i < n; i++) {
-		if (ks[i] == key) {
-		    BIF_RET(vs[i]);
-		}
-	    }
-	}
+    if (n == 0)
+	return 0;
+
+    ks  = map_get_keys(mp);
+    vs  = map_get_values(mp);
 
+    if (is_immed(key)) {
 	for( i = 0; i < n; i++) {
-	    if (eq(ks[i], key)) {
-		BIF_RET(vs[i]);
+	    if (ks[i] == key) {
+		*value = vs[i];
+		return 1;
 	    }
 	}
-error:
+    }
+
+    for( i = 0; i < n; i++) {
+	if (eq(ks[i], key)) {
+	    *value = vs[i];
+	    return 1;
+	}
+    }
+    return 0;
+}
+
+BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+	Eterm *hp;
+	Eterm value, error;
+	char *s_error;
+
+	if (erts_maps_get(BIF_ARG_1, BIF_ARG_2, &value)) {
+	    BIF_RET(value);
+	}
 
 	s_error = "bad_key";
 	error = am_atom_put(s_error, sys_strlen(s_error));
 
 	hp = HAlloc(BIF_P, 3);
-	BIF_P->fvalue = TUPLE2(hp, error, key);
+	BIF_P->fvalue = TUPLE2(hp, error, BIF_ARG_1);
 	BIF_ERROR(BIF_P, EXC_ERROR_2);
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -460,7 +482,7 @@ BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
     Eterm tup;
     map_t *mp;
 
-    hp    = HAlloc(BIF_P, (3 + 1));
+    hp    = HAlloc(BIF_P, (MAP_HEADER_SIZE + 1));
     tup   = make_tuple(hp);
     *hp++ = make_arityval(0);
 
@@ -475,183 +497,197 @@ BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
 /* maps:put/3
  */
 
-BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
-    if (is_map(BIF_ARG_3)) {
-	Sint n,i;
-	Sint c = 0;
-	Eterm* hp, *shp;
-	Eterm *ks,*vs, res, key, tup;
-	map_t *mp = (map_t*)map_val(BIF_ARG_3);
-
-	key = BIF_ARG_1;
-	n   = map_get_size(mp);
-
-	if (n == 0) {
-	    hp    = HAlloc(BIF_P, 4 + 2);
-	    tup   = make_tuple(hp);
-	    *hp++ = make_arityval(1);
-	    *hp++ = key;
-	    res   = make_map(hp);
-	    *hp++ = MAP_HEADER;
-	    *hp++ = 1;
-	    *hp++ = tup;
-	    *hp++ = BIF_ARG_2;
+Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
+    Sint n,i;
+    Sint c = 0;
+    Eterm* hp, *shp;
+    Eterm *ks,*vs, res, tup;
+    map_t *mp = (map_t*)map_val(map);
 
-	    BIF_RET(res);
-	}
-
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
-	/* only allocate for values,
-	 * assume key-tuple will be intact
-	 */
+    n = map_get_size(mp);
 
-	hp  = HAlloc(BIF_P, 3 + n);
-	shp = hp; /* save hp, used if optimistic update fails */
-	res = make_map(hp);
+    if (n == 0) {
+	hp    = HAlloc(p, MAP_HEADER_SIZE + 1 + 2);
+	tup   = make_tuple(hp);
+	*hp++ = make_arityval(1);
+	*hp++ = key;
+	res   = make_map(hp);
 	*hp++ = MAP_HEADER;
-	*hp++ = n;
-	*hp++ = mp->keys;
+	*hp++ = 1;
+	*hp++ = tup;
+	*hp++ = value;
 
-	if (is_immed(key)) {
-	    for( i = 0; i < n; i ++) {
-		if (ks[i] == key) {
-		    *hp++ = BIF_ARG_2;
-		    vs++;
-		    c = 1;
-		} else {
-		    *hp++ = *vs++;
-		}
+	return res;
+    }
+
+    ks  = map_get_keys(mp);
+    vs  = map_get_values(mp);
+
+    /* only allocate for values,
+     * assume key-tuple will be intact
+     */
+
+    hp  = HAlloc(p, MAP_HEADER_SIZE + n);
+    shp = hp; /* save hp, used if optimistic update fails */
+    res = make_map(hp);
+    *hp++ = MAP_HEADER;
+    *hp++ = n;
+    *hp++ = mp->keys;
+
+    if (is_immed(key)) {
+	for( i = 0; i < n; i ++) {
+	    if (ks[i] == key) {
+		*hp++ = value;
+		vs++;
+		c = 1;
+	    } else {
+		*hp++ = *vs++;
 	    }
-	} else {
-	    for( i = 0; i < n; i ++) {
-		if (eq(ks[i], key)) {
-		    *hp++ = BIF_ARG_2;
-		    vs++;
-		    c = 1;
-		} else {
-		    *hp++ = *vs++;
-		}
+	}
+    } else {
+	for( i = 0; i < n; i ++) {
+	    if (eq(ks[i], key)) {
+		*hp++ = value;
+		vs++;
+		c = 1;
+	    } else {
+		*hp++ = *vs++;
 	    }
 	}
+    }
 
-	if (c)
-	    BIF_RET(res);
+    if (c)
+	return res;
 
-	/* need to make a new tuple,
-	 * use old hp since it needs to be recreated anyway.
-	 */
-	tup    = make_tuple(shp);
-	*shp++ = make_arityval(n+1);
+    /* need to make a new tuple,
+     * use old hp since it needs to be recreated anyway.
+     */
+    tup    = make_tuple(shp);
+    *shp++ = make_arityval(n+1);
 
-	hp    = HAlloc(BIF_P, 3 + n + 1);
-	res   = make_map(hp);
-	*hp++ = MAP_HEADER;
-	*hp++ = n + 1;
-	*hp++ = tup;
+    hp    = HAlloc(p, 3 + n + 1);
+    res   = make_map(hp);
+    *hp++ = MAP_HEADER;
+    *hp++ = n + 1;
+    *hp++ = tup;
 
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
+    ks  = map_get_keys(mp);
+    vs  = map_get_values(mp);
 
-	ASSERT(n >= 0);
+    ASSERT(n >= 0);
 
-	/* copy map in order */
-	while (n && ((c = CMP(*ks, key)) < 0)) {
-	    *shp++ = *ks++;
-	    *hp++  = *vs++;
-	    n--;
-	}
+    /* copy map in order */
+    while (n && ((c = CMP(*ks, key)) < 0)) {
+	*shp++ = *ks++;
+	*hp++  = *vs++;
+	n--;
+    }
 
-	*shp++ = key;
-	*hp++  = BIF_ARG_2;
+    *shp++ = key;
+    *hp++  = value;
 
-	ASSERT(n >= 0);
+    ASSERT(n >= 0);
 
-	while(n--) {
-	    *shp++ = *ks++;
-	    *hp++  = *vs++;
-	}
-	/* we have one word remaining
-	 * this will work out fine once we get the size word
-	 * in the header.
-	 */
-	*shp = make_pos_bignum_header(0);
-	BIF_RET(res);
+    while(n--) {
+	*shp++ = *ks++;
+	*hp++  = *vs++;
     }
+    /* we have one word remaining
+     * this will work out fine once we get the size word
+     * in the header.
+     */
+    *shp = make_pos_bignum_header(0);
+    return res;
+}
 
+BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
+    if (is_map(BIF_ARG_3)) {
+	BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3));
+    }
     BIF_ERROR(BIF_P, BADARG);
 }
 
 /* maps:remove/3
  */
 
-BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
-	Sint n;
-	Sint found = 0;
-	Uint need;
-	Eterm *thp, *mhp;
-	Eterm *ks, *vs, res, key,tup;
-	map_t *mp = (map_t*)map_val(BIF_ARG_2);
+int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
+    Sint n;
+    Sint found = 0;
+    Uint need;
+    Eterm *thp, *mhp;
+    Eterm *ks, *vs, tup;
+    map_t *mp = (map_t*)map_val(map);
 
-	key = BIF_ARG_1;
-	n   = map_get_size(mp);
+    n = map_get_size(mp);
 
-	if (n == 0)
-	    BIF_RET(BIF_ARG_2);
+    if (n == 0) {
+	*res = map;
+	return 1;
+    }
 
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
+    ks = map_get_keys(mp);
+    vs = map_get_values(mp);
 
-	/* Assume key exists.
-	 * Release allocated if it didn't.
-	 * Allocate key tuple first.
-	 */
+    /* Assume key exists.
+     * Release allocated if it didn't.
+     * Allocate key tuple first.
+     */
 
-	need   = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */
-	thp    = HAlloc(BIF_P, need);
-	mhp    = thp + n;               /* offset with tuple heap size */
+    need   = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */
+    thp    = HAlloc(p, need);
+    mhp    = thp + n;               /* offset with tuple heap size */
 
-	tup    = make_tuple(thp);
-	*thp++ = make_arityval(n - 1);
+    tup    = make_tuple(thp);
+    *thp++ = make_arityval(n - 1);
 
-	res    = make_map(mhp);
-	*mhp++ = MAP_HEADER;
-	*mhp++ = n - 1;
-	*mhp++ = tup;
+    *res   = make_map(mhp);
+    *mhp++ = MAP_HEADER;
+    *mhp++ = n - 1;
+    *mhp++ = tup;
 
-	if (is_immed(key)) {
-	    while(n--) {
-		if (*ks == key) {
-		    ks++;
-		    vs++;
-		    found = 1;
-		} else {
-		    *mhp++ = *vs++;
-		    *thp++ = *ks++;
-		}
+    if (is_immed(key)) {
+	while(n--) {
+	    if (*ks == key) {
+		ks++;
+		vs++;
+		found = 1;
+	    } else {
+		*mhp++ = *vs++;
+		*thp++ = *ks++;
 	    }
-	} else {
-	    while(n--) {
-		if (eq(*ks, key)) {
-		    ks++;
-		    vs++;
-		    found = 1;
-		} else {
-		    *mhp++ = *vs++;
-		    *thp++ = *ks++;
-		}
+	}
+    } else {
+	while(n--) {
+	    if (eq(*ks, key)) {
+		ks++;
+		vs++;
+		found = 1;
+	    } else {
+		*mhp++ = *vs++;
+		*thp++ = *ks++;
 	    }
 	}
+    }
 
-	if (found)
-	    BIF_RET(res);
+    if (found) {
+	return 1;
+    }
 
-	/* Not found, remove allocated memory
-	 * and return previous map.
-	 */
-	HRelease(BIF_P, thp + need, thp);
-	BIF_RET(BIF_ARG_2);
+    /* Not found, remove allocated memory
+     * and return previous map.
+     */
+    HRelease(p, thp + need, thp);
+
+    *res = map;
+    return 1;
+}
+
+BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+	Eterm res;
+	if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) {
+	    BIF_RET(res);
+	}
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -659,19 +695,15 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
 /* maps:update/3
  */
 
-BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
-    if (is_map(BIF_ARG_3)) {
+int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
 	Sint n,i;
 	Sint found = 0;
 	Eterm* hp,*shp;
-	Eterm *ks,*vs, res, key;
-	map_t *mp = (map_t*)map_val(BIF_ARG_3);
+	Eterm *ks,*vs;
+	map_t *mp = (map_t*)map_val(map);
 
-	key = BIF_ARG_1;
-	n   = map_get_size(mp);
-
-	if (n == 0) {
-	    BIF_ERROR(BIF_P, BADARG);
+	if ((n = map_get_size(mp)) == 0) {
+	    return 0;
 	}
 
 	ks  = map_get_keys(mp);
@@ -681,9 +713,9 @@ BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
 	 * assume key-tuple will be intact
 	 */
 
-	hp  = HAlloc(BIF_P, 3 + n);
+	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
 	shp = hp;
-	res = make_map(hp);
+	*res = make_map(hp);
 	*hp++ = MAP_HEADER;
 	*hp++ = n;
 	*hp++ = mp->keys;
@@ -691,7 +723,7 @@ BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
 	if (is_immed(key)) {
 	    for( i = 0; i < n; i ++) {
 		if (ks[i] == key) {
-		    *hp++ = BIF_ARG_2;
+		    *hp++ = value;
 		    vs++;
 		    found = 1;
 		} else {
@@ -701,7 +733,7 @@ BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
 	} else {
 	    for( i = 0; i < n; i ++) {
 		if (eq(ks[i], key)) {
-		    *hp++ = BIF_ARG_2;
+		    *hp++ = value;
 		    vs++;
 		    found = 1;
 		} else {
@@ -710,10 +742,19 @@ BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
 	    }
 	}
 
-	if (found)
-	    BIF_RET(res);
+	if (found) {
+	    return 1;
+	}
+	HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
+	return 0;
+}
 
-	HRelease(BIF_P, shp + 3 + n, shp);
+BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
+    if (is_map(BIF_ARG_3)) {
+	Eterm res;
+	if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) {
+	    BIF_RET(res);
+	}
     }
     BIF_ERROR(BIF_P, BADARG);
 }
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 4f0d26e100..616ecd24ce 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -62,5 +62,11 @@ typedef struct map_s {
 #define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
 #define MAP_HEADER_SIZE        (sizeof(map_t) / sizeof(Eterm))
 
+Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
+int   erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
+int   erts_maps_find(Eterm key, Eterm map, Eterm *value);
+int   erts_maps_get(Eterm key, Eterm map, Eterm *value);
+int   erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
+
 #endif
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e1e213c4eb..c683847aaa 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -31,6 +31,7 @@
 #include "bif.h"
 #include "error.h"
 #include "big.h"
+#include "erl_map.h"
 #include "beam_bp.h"
 #include "erl_thr_progress.h"
 #include "dtrace-wrapper.h"
@@ -1602,6 +1603,214 @@ enif_have_dirty_schedulers()
 
 #endif /* ERL_NIF_DIRTY_SCHEDULER_SUPPORT */
 
+/* Maps */
+
+int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return is_map(term);
+}
+
+int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, int *size)
+{
+    if (is_map(term)) {
+	map_t *mp;
+	mp    = (map_t*)map_val(term);
+	*size = map_get_size(mp);
+	return 1;
+    }
+    return 0;
+}
+
+ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env)
+{
+    Eterm* hp = alloc_heap(env,MAP_HEADER_SIZE+1);
+    Eterm tup;
+    map_t *mp;
+
+    tup   = make_tuple(hp);
+    *hp++ = make_arityval(0);
+    mp    = (map_t*)hp;
+    mp->thing_word = MAP_HEADER;
+    mp->size = 0;
+    mp->keys = tup;
+
+    return make_map(mp);
+}
+
+int enif_make_map_put(ErlNifEnv* env,
+	              Eterm map_in,
+		      Eterm key,
+		      Eterm value,
+		      Eterm *map_out)
+{
+    if (is_not_map(map_in)) {
+	return 0;
+    }
+    flush_env(env);
+    *map_out = erts_maps_put(env->proc, key, value, map_in);
+    cache_env(env);
+    return 1;
+}
+
+int enif_get_map_value(ErlNifEnv* env,
+	               Eterm map,
+		       Eterm key,
+		       Eterm *value)
+{
+    if (is_not_map(map)) {
+	return 0;
+    }
+    return erts_maps_get(key, map, value);
+}
+
+int enif_find_map_value(ErlNifEnv* env,
+	                Eterm map,
+			Eterm key,
+			Eterm *value)
+{
+    if (is_not_map(map)) {
+	return 0;
+    }
+    return erts_maps_get(key, map, value);
+}
+
+int enif_make_map_update(ErlNifEnv* env,
+	                 Eterm map_in,
+			 Eterm key,
+			 Eterm value,
+			 Eterm *map_out)
+{
+    int res;
+    if (is_not_map(map_in)) {
+	return 0;
+    }
+
+    flush_env(env);
+    res = erts_maps_update(env->proc, key, value, map_in, map_out);
+    cache_env(env);
+    return res;
+}
+
+int enif_make_map_remove(ErlNifEnv* env,
+	                 Eterm map_in,
+			 Eterm key,
+			 Eterm *map_out)
+{
+    int res;
+    if (is_not_map(map_in)) {
+	return 0;
+    }
+    flush_env(env);
+    res = erts_maps_remove(env->proc, key, map_in, map_out);
+    cache_env(env);
+    return res;
+}
+
+int enif_map_iterator_create(ErlNifEnv *env,
+	                     Eterm map,
+			     ErlNifMapIterator *iter,
+			     ErlNifMapIteratorEntry entry)
+{
+    if (is_map(map)) {
+	map_t *mp = (map_t*)map_val(map);
+	size_t offset;
+
+	switch (entry) {
+	    case ERL_NIF_MAP_ITERATOR_HEAD: offset = 0; break;
+	    case ERL_NIF_MAP_ITERATOR_TAIL: offset = map_get_size(mp) - 1; break;
+	    default: goto error;
+	}
+
+	/* empty maps are ok but will leave the iterator
+	 * in bad shape.
+	 */
+
+	iter->map     = map;
+	iter->ks      = ((Eterm *)map_get_keys(mp)) + offset;
+	iter->vs      = ((Eterm *)map_get_values(mp)) + offset;
+	iter->t_limit = map_get_size(mp) + 1;
+	iter->h_limit = 0;
+	iter->idx     = offset + 1;
+
+	return 1;
+    }
+
+error:
+    iter->map = THE_NON_VALUE;
+    return 0;
+}
+
+void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter)
+{
+    /* not used */
+}
+
+int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)
+{
+    ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
+    if (is_map(iter->map) && (
+		(iter->t_limit - iter->h_limit) == 1 ||
+		 iter->idx == iter->t_limit)) {
+	return 1;
+    }
+    return 0;
+}
+
+int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)
+{
+    ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
+    if (is_map(iter->map) && (
+		(iter->t_limit - iter->h_limit) == 1 ||
+		 iter->idx == iter->h_limit)) {
+	return 1;
+    }
+    return 0;
+}
+
+
+int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
+{
+    if (is_map(iter->map) && iter->idx < iter->t_limit) {
+	iter->idx++;
+	if (iter->idx != iter->t_limit) {
+	    iter->ks++;
+	    iter->vs++;
+	}
+	return 1;
+    }
+    return 0;
+}
+
+int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
+{
+    if (is_map(iter->map) && iter->idx > iter->h_limit ) {
+	iter->idx--;
+	if (iter->idx != iter->h_limit ) {
+	    iter->ks--;
+	    iter->vs--;
+	}
+	return 1;
+    }
+    return 0;
+}
+
+int enif_map_iterator_get_pair(ErlNifEnv *env,
+			       ErlNifMapIterator *iter,
+			       Eterm *key,
+			       Eterm *value)
+{
+    if (is_map(iter->map) && iter->idx > iter->h_limit && iter->idx < iter->t_limit) {
+	ASSERT(iter->ks >= map_get_keys(map_val(iter->map)) &&
+	       iter->ks  < (map_get_keys(map_val(iter->map)) + map_get_size(map_val(iter->map))));
+	ASSERT(iter->vs >= map_get_values(map_val(iter->map)) &&
+	       iter->vs  < (map_get_values(map_val(iter->map)) + map_get_size(map_val(iter->map))));
+	*key   = *(iter->ks);
+	*value = *(iter->vs);
+	return 1;
+    }
+    return 0;
+}
+
 /***************************************************************************
  **                              load_nif/2                               **
  ***************************************************************************/
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index fb3c359ec9..3c1e13f8a4 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -38,14 +38,11 @@
 ** 2.2: R14B03 enif_is_exception
 ** 2.3: R15 enif_make_reverse_list, enif_is_number
 ** 2.4: R16 enif_consume_timeslice
-** 2.5: R17 dirty schedulers
+** 2.5: R17 Maps API additions
+**      R17 dirty schedulers
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 #define ERL_NIF_MINOR_VERSION 5
-#else
-#define ERL_NIF_MINOR_VERSION 4
-#endif
 
 #include <stdlib.h>
 
@@ -168,6 +165,7 @@ typedef int ErlNifTSDKey;
 
 typedef ErlDrvThreadOpts ErlNifThreadOpts;
 
+<<<<<<< HEAD
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 typedef enum
 {
@@ -175,6 +173,24 @@ typedef enum
     ERL_NIF_DIRTY_JOB_IO_BOUND  = 2
 }ErlNifDirtyTaskFlags;
 #endif
+=======
+typedef struct
+{
+    /* use a lot of memory, structure may change */
+    ERL_NIF_TERM map;
+    ErlNifUInt64 h_limit;
+    ErlNifUInt64 t_limit;
+    ErlNifUInt64 idx;
+    ERL_NIF_TERM *ks;
+    ERL_NIF_TERM *vs;
+} ErlNifMapIterator;
+
+typedef enum {
+    ERL_NIF_MAP_ITERATOR_HEAD = 1,
+    ERL_NIF_MAP_ITERATOR_TAIL = 2
+} ErlNifMapIteratorEntry;
+
+>>>>>>> erts: Add NIFs for Maps
 
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index f5b27dfdfa..2cabfd4ce1 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -149,6 +149,23 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
 #endif
 
+ERL_NIF_API_FUNC_DECL(int, enif_is_map, (ErlNifEnv* env, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term, int *size));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_new_map, (ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(int, enif_make_map_put, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out));
+ERL_NIF_API_FUNC_DECL(int, enif_get_map_value, (ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value));
+ERL_NIF_API_FUNC_DECL(int, enif_find_map_value, (ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value));
+ERL_NIF_API_FUNC_DECL(int, enif_make_map_update, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out));
+ERL_NIF_API_FUNC_DECL(int, enif_make_map_remove, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM* map_out));
+ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_create, (ErlNifEnv *env, ERL_NIF_TERM map, ErlNifMapIterator *iter, ErlNifMapIteratorEntry entry));
+ERL_NIF_API_FUNC_DECL(void, enif_map_iterator_destroy, (ErlNifEnv *env, ErlNifMapIterator *iter));
+ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_is_head, (ErlNifEnv *env, ErlNifMapIterator *iter));
+ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_is_tail, (ErlNifEnv *env, ErlNifMapIterator *iter));
+ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIterator *iter));
+ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter));
+ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value));
+
+
 /*
 ** Add new entries here to keep compatibility on Windows!!!
 */
@@ -281,6 +298,22 @@ ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
 #  define enif_have_dirty_schedulers ERL_NIF_API_FUNC_MACRO(enif_have_dirty_schedulers)
 #endif
 
+#  define enif_is_map ERL_NIF_API_FUNC_MACRO(enif_is_map)
+#  define enif_get_map_size ERL_NIF_API_FUNC_MACRO(enif_get_map_size)
+#  define enif_make_new_map ERL_NIF_FUNC_MACRO(enif_make_new_map)
+#  define enif_make_map_put ERL_NIF_FUNC_MACRO(enif_map_map_put)
+#  define enif_get_map_value ERL_NIF_FUNC_MACRO(enif_get_map_value)
+#  define enif_find_map_value ERL_NIF_FUNC_MACRO(enif_find_map_value)
+#  define enif_make_map_update ERL_NIF_FUNC_MACRO(enif_make_map_update)
+#  define enif_make_map_remove ERL_NIF_FUNC_MACRO(enif_make_map_remove)
+#  define enif_map_iterator_create ERL_NIF_FUNC_MACRO(enif_map_iterator_create)
+#  define enif_map_iterator_destroy ERL_NIF_FUNC_MACRO(enif_map_iterator_destroy)
+#  define enif_map_iterator_is_head ERL_NIF_FUNC_MACRO(enif_map_iterator_is_head)
+#  define enif_map_iterator_is_tail ERL_NIF_FUNC_MACRO(enif_map_iterator_is_tail)
+#  define enif_map_iterator_next ERL_NIF_FUNC_MACRO(enif_map_iterator_next)
+#  define enif_map_iterator_prev ERL_NIF_FUNC_MACRO(enif_map_iterator_prev)
+#  define enif_map_iterator_get_pair NIF_FUNC_MACRO(enif_map_iterator_get_pair)
+
 /*
 ** Add new entries here
 */
-- 
cgit v1.2.3


From faf92c5174c7b86d2b5829c928b187753b3918e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 29 Nov 2013 12:12:53 +0100
Subject: erts: NIFs Map API fixup

---
 erts/emulator/beam/erl_nif_api_funcs.h | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 2cabfd4ce1..4a5aacad48 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -300,19 +300,19 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa
 
 #  define enif_is_map ERL_NIF_API_FUNC_MACRO(enif_is_map)
 #  define enif_get_map_size ERL_NIF_API_FUNC_MACRO(enif_get_map_size)
-#  define enif_make_new_map ERL_NIF_FUNC_MACRO(enif_make_new_map)
-#  define enif_make_map_put ERL_NIF_FUNC_MACRO(enif_map_map_put)
-#  define enif_get_map_value ERL_NIF_FUNC_MACRO(enif_get_map_value)
-#  define enif_find_map_value ERL_NIF_FUNC_MACRO(enif_find_map_value)
-#  define enif_make_map_update ERL_NIF_FUNC_MACRO(enif_make_map_update)
-#  define enif_make_map_remove ERL_NIF_FUNC_MACRO(enif_make_map_remove)
-#  define enif_map_iterator_create ERL_NIF_FUNC_MACRO(enif_map_iterator_create)
-#  define enif_map_iterator_destroy ERL_NIF_FUNC_MACRO(enif_map_iterator_destroy)
-#  define enif_map_iterator_is_head ERL_NIF_FUNC_MACRO(enif_map_iterator_is_head)
-#  define enif_map_iterator_is_tail ERL_NIF_FUNC_MACRO(enif_map_iterator_is_tail)
-#  define enif_map_iterator_next ERL_NIF_FUNC_MACRO(enif_map_iterator_next)
-#  define enif_map_iterator_prev ERL_NIF_FUNC_MACRO(enif_map_iterator_prev)
-#  define enif_map_iterator_get_pair NIF_FUNC_MACRO(enif_map_iterator_get_pair)
+#  define enif_make_new_map ERL_NIF_API_FUNC_MACRO(enif_make_new_map)
+#  define enif_make_map_put ERL_NIF_API_FUNC_MACRO(enif_map_map_put)
+#  define enif_get_map_value ERL_NIF_API_FUNC_MACRO(enif_get_map_value)
+#  define enif_find_map_value ERL_NIF_API_FUNC_MACRO(enif_find_map_value)
+#  define enif_make_map_update ERL_NIF_API_FUNC_MACRO(enif_make_map_update)
+#  define enif_make_map_remove ERL_NIF_API_FUNC_MACRO(enif_make_map_remove)
+#  define enif_map_iterator_create ERL_NIF_API_FUNC_MACRO(enif_map_iterator_create)
+#  define enif_map_iterator_destroy ERL_NIF_API_FUNC_MACRO(enif_map_iterator_destroy)
+#  define enif_map_iterator_is_head ERL_NIF_API_FUNC_MACRO(enif_map_iterator_is_head)
+#  define enif_map_iterator_is_tail ERL_NIF_API_FUNC_MACRO(enif_map_iterator_is_tail)
+#  define enif_map_iterator_next ERL_NIF_API_FUNC_MACRO(enif_map_iterator_next)
+#  define enif_map_iterator_prev ERL_NIF_API_FUNC_MACRO(enif_map_iterator_prev)
+#  define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair)
 
 /*
 ** Add new entries here
-- 
cgit v1.2.3


From d5c238473b9cec819d93faaef4ccc00ddb60465f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Sat, 26 Oct 2013 21:01:41 +0200
Subject: erts: Add cmp_term to compare

Uses total order of types meaning int < float
---
 erts/emulator/beam/erl_utils.h | 46 +++++++++++++++++++++++++++---------------
 erts/emulator/beam/utils.c     | 13 ++++++++++--
 2 files changed, 41 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 292d135946..c6247c7246 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -202,23 +202,37 @@ int eq(Eterm, Eterm);
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
 #if HALFWORD_HEAP
-Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*);
-#define CMP(A,B) cmp_rel(A,NULL,B,NULL)
+Sint cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int);
+#define cmp_rel(A,A_BASE,B,B_BASE)       cmp_rel_opt(A,A_BASE,B,B_BASE,0)
+#define cmp_rel_exact(A,A_BASE,B,B_BASE) cmp_rel_opt(A,A_BASE,B,B_BASE,1)
+#define CMP(A,B)                         cmp_rel(A,NULL,B,NULL,0)
+#define CMP_TERM(A,B)                    cmp_rel(A,NULL,B,NULL,1)
 #else
-Sint cmp(Eterm, Eterm);
-#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B)
-#define CMP(A,B) cmp(A,B)
+Sint cmp(Eterm, Eterm, int);
+#define cmp_rel(A,A_BASE,B,B_BASE)       cmp(A,B,0)
+#define cmp_rel_term(A,A_BASE,B,B_BASE)  cmp(A,B,1)
+#define CMP(A,B)                         cmp(A,B,0)
+#define CMP_TERM(A,B)                    cmp(A,B,1)
 #endif
-#define cmp_lt(a,b)	(CMP((a),(b)) < 0)
-#define cmp_le(a,b)	(CMP((a),(b)) <= 0)
-#define cmp_eq(a,b)	(CMP((a),(b)) == 0)
-#define cmp_ne(a,b)	(CMP((a),(b)) != 0)
-#define cmp_ge(a,b)	(CMP((a),(b)) >= 0)
-#define cmp_gt(a,b)	(CMP((a),(b)) > 0)
-
-#define CMP_LT(a,b)	((a) != (b) && cmp_lt((a),(b)))
-#define CMP_GE(a,b)	((a) == (b) || cmp_ge((a),(b)))
-#define CMP_EQ(a,b)	((a) == (b) || cmp_eq((a),(b)))
-#define CMP_NE(a,b)	((a) != (b) && cmp_ne((a),(b)))
+
+#define cmp_lt(a,b)          (CMP((a),(b)) <  0)
+#define cmp_le(a,b)          (CMP((a),(b)) <= 0)
+#define cmp_eq(a,b)          (CMP((a),(b)) == 0)
+#define cmp_ne(a,b)          (CMP((a),(b)) != 0)
+#define cmp_ge(a,b)          (CMP((a),(b)) >= 0)
+#define cmp_gt(a,b)          (CMP((a),(b)) >  0)
+
+#define cmp_lt_term(a,b)     (CMP_TERM((a),(b)) <  0)
+#define cmp_le_term(a,b)     (CMP_TERM((a),(b)) <= 0)
+#define cmp_ge_term(a,b)     (CMP_TERM((a),(b)) >= 0)
+#define cmp_gt_term(a,b)     (CMP_TERM((a),(b)) >  0)
+
+#define CMP_LT(a,b)          ((a) != (b) && cmp_lt((a),(b)))
+#define CMP_GE(a,b)          ((a) == (b) || cmp_ge((a),(b)))
+#define CMP_EQ(a,b)          ((a) == (b) || cmp_eq((a),(b)))
+#define CMP_NE(a,b)          ((a) != (b) && cmp_ne((a),(b)))
+
+#define CMP_LT_TERM(a,b)     ((a) != (b) && cmp_lt_term((a),(b)))
+#define CMP_GE_TERM(a,b)     ((a) == (b) || cmp_ge_term((a),(b)))
 
 #endif
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 8958d334ae..bc4a05d385 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2425,10 +2425,14 @@ static int cmp_atoms(Eterm a, Eterm b)
 		    bb->name+3, bb->len-3);
 }
 
+/* cmp(Eterm a, Eterm b, int exact)
+ * exact = 1 -> term-based compare
+ * exact = 0 -> arith-based compare
+ */
 #if HALFWORD_HEAP
-Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+Sint cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact)
 #else
-Sint cmp(Eterm a, Eterm b)
+Sint cmp(Eterm a, Eterm b, int exact)
 #endif
 {
     DECLARE_WSTACK(stack);
@@ -2852,6 +2856,7 @@ tailrecur_ne:
 	    j = big_sign(aw) ? -1 : 1;
 	    break;
 	case SMALL_FLOAT:
+	    if (exact) goto exact_fall_through;
 	    GET_DOUBLE(bw, f2);
 	    if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) {
 		/* Float is within the no loss limit */
@@ -2877,12 +2882,14 @@ tailrecur_ne:
 #endif /* ERTS_SIZEOF_ETERM == 8 */
 	    break;
         case FLOAT_BIG:
+	    if (exact) goto exact_fall_through;
 	{
 	    Wterm tmp = aw;
 	    aw = bw;
 	    bw = tmp;
 	}/* fall through */
 	case BIG_FLOAT:
+	    if (exact) goto exact_fall_through;
 	    GET_DOUBLE(bw, f2);
 	    if ((f2.fd < (double) (MAX_SMALL + 1))
 		    && (f2.fd > (double) (MIN_SMALL - 1))) {
@@ -2912,6 +2919,7 @@ tailrecur_ne:
 	    }
 	    break;
 	case FLOAT_SMALL:
+	    if (exact) goto exact_fall_through;
 	    GET_DOUBLE(aw, f1);
 	    if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) {
 		/* Float is within the no loss limit */
@@ -2936,6 +2944,7 @@ tailrecur_ne:
 	    }
 #endif /* ERTS_SIZEOF_ETERM == 8 */
 	    break;
+exact_fall_through:
 	default:
 	    j = b_tag - a_tag;
 	}
-- 
cgit v1.2.3


From 76b8ea8ab1eb4ce099f88ccb8d1721c438d0ada4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 12 Dec 2013 10:58:59 +0100
Subject: erts: Add BIF erts_internal:cmp_term/2

Compares terms where integer() < float().
---
 erts/emulator/beam/bif.c   | 11 +++++++++++
 erts/emulator/beam/bif.tab |  2 ++
 2 files changed, 13 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 61c1abedb5..9c4801041f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4615,6 +4615,17 @@ BIF_RETTYPE bump_reductions_1(BIF_ALIST_1)
     BIF_RET2(am_true, reds);
 }
 
+BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) {
+    int res = CMP_TERM(BIF_ARG_1,BIF_ARG_2);
+
+    /* ensure -1, 0, 1 result */
+    if (res < 0) {
+	BIF_RET(make_small(-1));
+    } else if (res > 0) {
+	BIF_RET(make_small(1));
+    }
+    BIF_RET(make_small(0));
+}
 /*
  * Processes doing yield on return in a bif ends up in bif_return_trap().
  */
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index b623e47b9a..2d888862bf 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -593,6 +593,8 @@ bif maps:remove/2
 bif maps:update/3
 bif maps:values/1
 
+bif erts_internal:cmp_term/2
+
 #
 # Obsolete
 #
-- 
cgit v1.2.3


From 862728a458729f4a71630f4a8fa93f1f26744c7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 12 Dec 2013 11:15:20 +0100
Subject: erts: Update maps BIFs to use term order

Maps internally uses term order to store keys in an ordered fashion.
---
 erts/emulator/beam/erl_map.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 98aeee634b..455ba25e11 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -108,7 +108,7 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
 }
 
 /* maps:find/2
- * return value if key *equals* a key in the map
+ * return value if key *matches* a key in the map
  */
 
 int erts_maps_find(Eterm key, Eterm map, Eterm *value) {
@@ -123,7 +123,7 @@ int erts_maps_find(Eterm key, Eterm map, Eterm *value) {
     vs  = map_get_values(mp);
 
     for( i = 0; i < n; i++) {
-	if (CMP(ks[i], key)==0) {
+	if (EQ(ks[i], key)) {
 	    *value = vs[i];
 	    return 1;
 	}
@@ -178,7 +178,7 @@ int erts_maps_get(Eterm key, Eterm map, Eterm *value) {
     }
 
     for( i = 0; i < n; i++) {
-	if (eq(ks[i], key)) {
+	if (EQ(ks[i], key)) {
 	    *value = vs[i];
 	    return 1;
 	}
@@ -283,7 +283,7 @@ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) {
 
 	    idx = size;
 
-	    while(idx > 0 && (c = CMP(kv[1],ks[idx-1])) < 0) { idx--; }
+	    while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; }
 
 	    if (c == 0) {
 		/* last compare was equal,
@@ -353,7 +353,7 @@ BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
 	}
 
 	for( i = 0; i < n; i++) {
-	    if (eq(ks[i], key)) {
+	    if (EQ(ks[i], key)) {
 		BIF_RET(am_true);
 	    }
 	}
@@ -425,7 +425,7 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
 	vs2 = map_get_values(mp2);
 
 	while(i1 < n1 && i2 < n2) {
-	    c = CMP(ks1[i1],ks2[i2]);
+	    c = CMP_TERM(ks1[i1],ks2[i2]);
 	    if ( c == 0) {
 		/* use righthand side arguments map value,
 		 * but advance both maps */
@@ -546,7 +546,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 	}
     } else {
 	for( i = 0; i < n; i ++) {
-	    if (eq(ks[i], key)) {
+	    if (EQ(ks[i], key)) {
 		*hp++ = value;
 		vs++;
 		c = 1;
@@ -577,7 +577,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
     ASSERT(n >= 0);
 
     /* copy map in order */
-    while (n && ((c = CMP(*ks, key)) < 0)) {
+    while (n && ((c = CMP_TERM(*ks, key)) < 0)) {
 	*shp++ = *ks++;
 	*hp++  = *vs++;
 	n--;
@@ -658,7 +658,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 	}
     } else {
 	while(n--) {
-	    if (eq(*ks, key)) {
+	    if (EQ(*ks, key)) {
 		ks++;
 		vs++;
 		found = 1;
@@ -732,7 +732,7 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res)
 	    }
 	} else {
 	    for( i = 0; i < n; i ++) {
-		if (eq(ks[i], key)) {
+		if (EQ(ks[i], key)) {
 		    *hp++ = value;
 		    vs++;
 		    found = 1;
-- 
cgit v1.2.3


From f3821597383fa20d1093dab70fa75b4a1018a6b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 12 Dec 2013 11:20:04 +0100
Subject: erts: Update maps instructions to use term order

Maps internally uses term order to store keys in an ordered fashion.
---
 erts/emulator/beam/beam_emu.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index fe2e196785..89d9442526 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6316,7 +6316,7 @@ static int has_not_map_field(Eterm map, Eterm key)
 	}
     } else {
 	for (i = 0; i <  n; i++) {
-	    if (eq(keys[i], key)) {
+	    if (EQ(keys[i], key)) {
 		return 0;
 	    }
 	}
@@ -6343,7 +6343,7 @@ static Eterm get_map_element(Eterm map, Eterm key)
 	}
     } else {
 	for (i = 0; i < n; i++) {
-	    if (eq(ks[i], key)) {
+	    if (EQ(ks[i], key)) {
 		return vs[i];
 	    }
 	}
@@ -6506,7 +6506,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
 	ASSERT(kp < (Eterm *)mp);
 	key = *old_keys;
-	if ((c = CMP(key, new_key)) < 0) {
+	if ((c = CMP_TERM(key, new_key)) < 0) {
 	    /* Copy old key and value */
 	    *kp++ = key;
 	    *hp++ = *old_vals;
-- 
cgit v1.2.3


From c334d4841d9d600237184233518ee13f7421f91c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 10 Jan 2014 18:04:00 +0100
Subject: erts: Update maps serializing to use term order

* erlang:term_to_binary/1
* erlang:binary_to_term/1
---
 erts/emulator/beam/external.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index a31b2731be..e8b77b9f37 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3803,7 +3803,7 @@ dec_term_atom_common:
 	arity = arityval(*keys++);
 
 	while(arity-- > 1) {
-	    if (CMP(keys[arity-1],keys[arity]) >= 0) {
+	    if (CMP_TERM(keys[arity-1],keys[arity]) >= 0) {
 		goto error;
 	    }
 	}
-- 
cgit v1.2.3


From 0422247a9d8259ef1a28704500f07caf827f42f6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 16 Jan 2014 22:00:04 +0100
Subject: erts: Fix bug in erts_maps_remove

HRelease was called with wrong arguments and left garbage on heap
when key was not found.
---
 erts/emulator/beam/erl_map.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 455ba25e11..e68482be58 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2013. All Rights Reserved.
+ * Copyright Ericsson AB 2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -614,6 +614,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     Sint n;
     Sint found = 0;
     Uint need;
+    Eterm *hp_start;
     Eterm *thp, *mhp;
     Eterm *ks, *vs, tup;
     map_t *mp = (map_t*)map_val(map);
@@ -634,7 +635,8 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
      */
 
     need   = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */
-    thp    = HAlloc(p, need);
+    hp_start = HAlloc(p, need);
+    thp    = hp_start;
     mhp    = thp + n;               /* offset with tuple heap size */
 
     tup    = make_tuple(thp);
@@ -676,7 +678,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     /* Not found, remove allocated memory
      * and return previous map.
      */
-    HRelease(p, thp + need, thp);
+    HRelease(p, hp_start + need, hp_start);
 
     *res = map;
     return 1;
-- 
cgit v1.2.3


From f1003b26524c2321b1eef36fb2d3997aaf0b9e10 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 16 Jan 2014 23:03:33 +0100
Subject: erts: Optimize erts_map_update/remove

to not continue comparing keys once it has been found.
---
 erts/emulator/beam/erl_map.c | 42 ++++++++++++++++++++----------------------
 1 file changed, 20 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index e68482be58..57f156509c 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -612,7 +612,6 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
 
 int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     Sint n;
-    Sint found = 0;
     Uint need;
     Eterm *hp_start;
     Eterm *thp, *mhp;
@@ -650,9 +649,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     if (is_immed(key)) {
 	while(n--) {
 	    if (*ks == key) {
-		ks++;
-		vs++;
-		found = 1;
+		goto found_key;
 	    } else {
 		*mhp++ = *vs++;
 		*thp++ = *ks++;
@@ -661,9 +658,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     } else {
 	while(n--) {
 	    if (EQ(*ks, key)) {
-		ks++;
-		vs++;
-		found = 1;
+		goto found_key;
 	    } else {
 		*mhp++ = *vs++;
 		*thp++ = *ks++;
@@ -671,10 +666,6 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 	}
     }
 
-    if (found) {
-	return 1;
-    }
-
     /* Not found, remove allocated memory
      * and return previous map.
      */
@@ -682,6 +673,14 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 
     *res = map;
     return 1;
+
+found_key:
+    /* Copy rest of keys and values */
+    if (n) {
+	sys_memcpy(mhp, vs+1, n*sizeof(Eterm));
+	sys_memcpy(thp, ks+1, n*sizeof(Eterm));
+    }
+    return 1;
 }
 
 BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
@@ -699,7 +698,6 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
 
 int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
 	Sint n,i;
-	Sint found = 0;
 	Eterm* hp,*shp;
 	Eterm *ks,*vs;
 	map_t *mp = (map_t*)map_val(map);
@@ -717,7 +715,6 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res)
 
 	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
 	shp = hp;
-	*res = make_map(hp);
 	*hp++ = MAP_HEADER;
 	*hp++ = n;
 	*hp++ = mp->keys;
@@ -725,9 +722,7 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res)
 	if (is_immed(key)) {
 	    for( i = 0; i < n; i ++) {
 		if (ks[i] == key) {
-		    *hp++ = value;
-		    vs++;
-		    found = 1;
+		    goto found_key;
 		} else {
 		    *hp++ = *vs++;
 		}
@@ -735,20 +730,23 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res)
 	} else {
 	    for( i = 0; i < n; i ++) {
 		if (EQ(ks[i], key)) {
-		    *hp++ = value;
-		    vs++;
-		    found = 1;
+		    goto found_key;
 		} else {
 		    *hp++ = *vs++;
 		}
 	    }
 	}
 
-	if (found) {
-	    return 1;
-	}
 	HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
 	return 0;
+
+found_key:
+	*hp++ = value;
+	vs++;
+	if (++i < n)
+	    sys_memcpy(hp, vs, (n - i)*sizeof(Eterm));
+	*res = make_map(shp);
+	return 1;
 }
 
 BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
-- 
cgit v1.2.3


From 39c35199f5118a59f337b695a934c6bfcbf0813b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 13 Jan 2014 16:23:56 +0100
Subject: erts: Let erlang:binary_to_term/1 handle unsorted Maps

Maps may be encoded with keys in arbitrary order. This is fine,
as long as keys are unique.
---
 erts/emulator/beam/external.c | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index e8b77b9f37..57251286c8 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3790,24 +3790,41 @@ dec_term_atom_common:
         }
     }
 
-    /* Iterate through all the maps and check for validity
+    /* Iterate through all the maps and check for validity and sort keys
      * - done here for when we know it is complete.
      */
 
     while (maps_head) {
-	Eterm *keys;
-	Sint  arity;
+	map_t *mp  = (map_t*)maps_head;
+	Eterm *ks  = map_get_keys(mp);
+	Eterm *vs  = map_get_values(mp);
+	Uint   sz  = map_get_size(mp);
+	Uint   ix,jx;
+	Eterm  tmp;
+	int c;
 
 	next  = (Eterm *)(EXPAND_POINTER(*maps_head));
-	keys  = tuple_val(*(maps_head + 2));
-	arity = arityval(*keys++);
 
-	while(arity-- > 1) {
-	    if (CMP_TERM(keys[arity-1],keys[arity]) >= 0) {
-		goto error;
+	/* sort */
+
+	for ( ix = 1; ix < sz; ix++) {
+	    jx = ix;
+	    while( jx > 0 && (c = CMP_TERM(ks[jx],ks[jx-1])) <= 0 ) {
+		/* identical key -> error */
+		if (c == 0) goto error;
+
+		tmp = ks[jx];
+		ks[jx] = ks[jx - 1];
+		ks[jx - 1] = tmp;
+
+		tmp = vs[jx];
+		vs[jx] = vs[jx - 1];
+		vs[jx - 1] = tmp;
+
+		jx--;
 	    }
-	}
 
+	}
 	*maps_head = MAP_HEADER;
 	maps_head  = next;
     }
-- 
cgit v1.2.3


From c8ecb6962a923ceb0b7e599d22adef773d042e4a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 13 Jan 2014 22:15:13 +0100
Subject: erts: Remove enif_find_map_value

as it does the same thing as enif_get_map_value.

Replace with placeholder to be ABI backward compatible on Windows
as long as enif_find_map_value is not called.
---
 erts/emulator/beam/erl_nif.c           | 13 +------------
 erts/emulator/beam/erl_nif_api_funcs.h |  5 ++---
 2 files changed, 3 insertions(+), 15 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c683847aaa..1a539c730f 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1663,17 +1663,6 @@ int enif_get_map_value(ErlNifEnv* env,
     return erts_maps_get(key, map, value);
 }
 
-int enif_find_map_value(ErlNifEnv* env,
-	                Eterm map,
-			Eterm key,
-			Eterm *value)
-{
-    if (is_not_map(map)) {
-	return 0;
-    }
-    return erts_maps_get(key, map, value);
-}
-
 int enif_make_map_update(ErlNifEnv* env,
 	                 Eterm map_in,
 			 Eterm key,
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 4a5aacad48..71abb5dc8b 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -154,7 +154,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_new_map, (ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(int, enif_make_map_put, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out));
 ERL_NIF_API_FUNC_DECL(int, enif_get_map_value, (ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value));
-ERL_NIF_API_FUNC_DECL(int, enif_find_map_value, (ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value));
+ERL_NIF_API_FUNC_DECL(void, __enif_PLACEHOLDER__, (void));
 ERL_NIF_API_FUNC_DECL(int, enif_make_map_update, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out));
 ERL_NIF_API_FUNC_DECL(int, enif_make_map_remove, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM* map_out));
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_create, (ErlNifEnv *env, ERL_NIF_TERM map, ErlNifMapIterator *iter, ErlNifMapIteratorEntry entry));
@@ -303,7 +303,6 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa
 #  define enif_make_new_map ERL_NIF_API_FUNC_MACRO(enif_make_new_map)
 #  define enif_make_map_put ERL_NIF_API_FUNC_MACRO(enif_map_map_put)
 #  define enif_get_map_value ERL_NIF_API_FUNC_MACRO(enif_get_map_value)
-#  define enif_find_map_value ERL_NIF_API_FUNC_MACRO(enif_find_map_value)
 #  define enif_make_map_update ERL_NIF_API_FUNC_MACRO(enif_make_map_update)
 #  define enif_make_map_remove ERL_NIF_API_FUNC_MACRO(enif_make_map_remove)
 #  define enif_map_iterator_create ERL_NIF_API_FUNC_MACRO(enif_map_iterator_create)
-- 
cgit v1.2.3


From b8a7203f04a1ed6d2756e6b782cd4c95a8c7c491 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Jan 2014 18:22:34 +0100
Subject: erts: Increase version for NIF API and reject experimental v2.5

---
 erts/emulator/beam/erl_nif.c           | 3 ++-
 erts/emulator/beam/erl_nif.h           | 9 ++++-----
 erts/emulator/beam/erl_nif_api_funcs.h | 1 -
 3 files changed, 6 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1a539c730f..1eca7822eb 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1996,7 +1996,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful");
     }
     else if (entry->major != ERL_NIF_MAJOR_VERSION
-	     || entry->minor > ERL_NIF_MINOR_VERSION) {
+	     || entry->minor > ERL_NIF_MINOR_VERSION
+	     || (entry->major==2 && entry->minor == 5)) { /* experimental maps */
 	
 	ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).",
 			     entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION);
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 3c1e13f8a4..ae7f63f923 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -38,11 +38,13 @@
 ** 2.2: R14B03 enif_is_exception
 ** 2.3: R15 enif_make_reverse_list, enif_is_number
 ** 2.4: R16 enif_consume_timeslice
+** 2.5: First experimental maps API additions (libs of this version is not compatible with any other VM)
 ** 2.5: R17 Maps API additions
+** 2.6: R17 with maps
 **      R17 dirty schedulers
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 5
+#define ERL_NIF_MINOR_VERSION 6
 
 #include <stdlib.h>
 
@@ -165,7 +167,6 @@ typedef int ErlNifTSDKey;
 
 typedef ErlDrvThreadOpts ErlNifThreadOpts;
 
-<<<<<<< HEAD
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 typedef enum
 {
@@ -173,7 +174,7 @@ typedef enum
     ERL_NIF_DIRTY_JOB_IO_BOUND  = 2
 }ErlNifDirtyTaskFlags;
 #endif
-=======
+
 typedef struct
 {
     /* use a lot of memory, structure may change */
@@ -190,8 +191,6 @@ typedef enum {
     ERL_NIF_MAP_ITERATOR_TAIL = 2
 } ErlNifMapIteratorEntry;
 
->>>>>>> erts: Add NIFs for Maps
-
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
 typedef struct {
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 71abb5dc8b..9c386a4c41 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -154,7 +154,6 @@ ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_new_map, (ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(int, enif_make_map_put, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out));
 ERL_NIF_API_FUNC_DECL(int, enif_get_map_value, (ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value));
-ERL_NIF_API_FUNC_DECL(void, __enif_PLACEHOLDER__, (void));
 ERL_NIF_API_FUNC_DECL(int, enif_make_map_update, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out));
 ERL_NIF_API_FUNC_DECL(int, enif_make_map_remove, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM* map_out));
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_create, (ErlNifEnv *env, ERL_NIF_TERM map, ErlNifMapIterator *iter, ErlNifMapIteratorEntry entry));
-- 
cgit v1.2.3


From f321ea89fecdb343e58e2485c057326d08fed69c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 13 Jan 2014 22:16:40 +0100
Subject: erts: Do not allow map iterator created without map

---
 erts/emulator/beam/erl_nif.c | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1eca7822eb..ca82590a78 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1725,21 +1725,27 @@ int enif_map_iterator_create(ErlNifEnv *env,
     }
 
 error:
+#ifdef DEBUG
     iter->map = THE_NON_VALUE;
+#endif
     return 0;
 }
 
 void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
     /* not used */
+#ifdef DEBUG
+    iter->map = THE_NON_VALUE;
+#endif
+
 }
 
 int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
+    ASSERT(iter && is_map(iter->map));
     ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
-    if (is_map(iter->map) && (
-		(iter->t_limit - iter->h_limit) == 1 ||
-		 iter->idx == iter->t_limit)) {
+    if ((iter->t_limit - iter->h_limit) == 1
+	|| iter->idx == iter->t_limit) {
 	return 1;
     }
     return 0;
@@ -1747,10 +1753,10 @@ int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)
 
 int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
+    ASSERT(iter && is_map(iter->map));
     ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
-    if (is_map(iter->map) && (
-		(iter->t_limit - iter->h_limit) == 1 ||
-		 iter->idx == iter->h_limit)) {
+    if ((iter->t_limit - iter->h_limit) == 1
+	|| iter->idx == iter->h_limit) {
 	return 1;
     }
     return 0;
@@ -1759,7 +1765,8 @@ int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)
 
 int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    if (is_map(iter->map) && iter->idx < iter->t_limit) {
+    ASSERT(iter && is_map(iter->map));
+    if (iter->idx < iter->t_limit) {
 	iter->idx++;
 	if (iter->idx != iter->t_limit) {
 	    iter->ks++;
@@ -1772,7 +1779,8 @@ int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
 
 int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    if (is_map(iter->map) && iter->idx > iter->h_limit ) {
+    ASSERT(iter && is_map(iter->map));
+    if (iter->idx > iter->h_limit ) {
 	iter->idx--;
 	if (iter->idx != iter->h_limit ) {
 	    iter->ks--;
@@ -1788,7 +1796,8 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
 			       Eterm *key,
 			       Eterm *value)
 {
-    if (is_map(iter->map) && iter->idx > iter->h_limit && iter->idx < iter->t_limit) {
+    ASSERT(iter && is_map(iter->map));
+    if (iter->idx > iter->h_limit && iter->idx < iter->t_limit) {
 	ASSERT(iter->ks >= map_get_keys(map_val(iter->map)) &&
 	       iter->ks  < (map_get_keys(map_val(iter->map)) + map_get_size(map_val(iter->map))));
 	ASSERT(iter->vs >= map_get_values(map_val(iter->map)) &&
-- 
cgit v1.2.3


From 609d835ce04a1f6320377780a11fb32382e1e419 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Jan 2014 11:25:34 +0100
Subject: erts: Let enif_map_iterator_next/prev return 0 to signal end of map.

---
 erts/emulator/beam/erl_nif.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ca82590a78..def9a7efe1 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1771,8 +1771,8 @@ int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
 	if (iter->idx != iter->t_limit) {
 	    iter->ks++;
 	    iter->vs++;
+	    return 1;
 	}
-	return 1;
     }
     return 0;
 }
@@ -1785,8 +1785,8 @@ int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
 	if (iter->idx != iter->h_limit ) {
 	    iter->ks--;
 	    iter->vs--;
+	    return 1;
 	}
-	return 1;
     }
     return 0;
 }
-- 
cgit v1.2.3


From d39affeac223a7e2891b509395e7b19c931b5018 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Jan 2014 20:32:46 +0100
Subject: erts: Remove use of h_limit which is always zero.

---
 erts/emulator/beam/erl_nif.c | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index def9a7efe1..94c2512650 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1718,7 +1718,6 @@ int enif_map_iterator_create(ErlNifEnv *env,
 	iter->ks      = ((Eterm *)map_get_keys(mp)) + offset;
 	iter->vs      = ((Eterm *)map_get_values(mp)) + offset;
 	iter->t_limit = map_get_size(mp) + 1;
-	iter->h_limit = 0;
 	iter->idx     = offset + 1;
 
 	return 1;
@@ -1744,8 +1743,7 @@ int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
     ASSERT(iter && is_map(iter->map));
     ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
-    if ((iter->t_limit - iter->h_limit) == 1
-	|| iter->idx == iter->t_limit) {
+    if (iter->t_limit == 1 || iter->idx == iter->t_limit) {
 	return 1;
     }
     return 0;
@@ -1755,8 +1753,7 @@ int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
     ASSERT(iter && is_map(iter->map));
     ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
-    if ((iter->t_limit - iter->h_limit) == 1
-	|| iter->idx == iter->h_limit) {
+    if (iter->t_limit == 1 || iter->idx == 0) {
 	return 1;
     }
     return 0;
@@ -1780,9 +1777,9 @@ int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
 int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
     ASSERT(iter && is_map(iter->map));
-    if (iter->idx > iter->h_limit ) {
+    if (iter->idx > 0) {
 	iter->idx--;
-	if (iter->idx != iter->h_limit ) {
+	if (iter->idx != 0) {
 	    iter->ks--;
 	    iter->vs--;
 	    return 1;
@@ -1797,7 +1794,7 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
 			       Eterm *value)
 {
     ASSERT(iter && is_map(iter->map));
-    if (iter->idx > iter->h_limit && iter->idx < iter->t_limit) {
+    if (iter->idx > 0 && iter->idx < iter->t_limit) {
 	ASSERT(iter->ks >= map_get_keys(map_val(iter->map)) &&
 	       iter->ks  < (map_get_keys(map_val(iter->map)) + map_get_size(map_val(iter->map))));
 	ASSERT(iter->vs >= map_get_values(map_val(iter->map)) &&
-- 
cgit v1.2.3


From ada36d9024b4ceda974f32a78b5fe39b64f59319 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Jan 2014 20:55:10 +0100
Subject: erts: Simplify some map iterator code

---
 erts/emulator/beam/erl_nif.c | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 94c2512650..c6f7c8adb5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1743,20 +1743,14 @@ int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
     ASSERT(iter && is_map(iter->map));
     ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
-    if (iter->t_limit == 1 || iter->idx == iter->t_limit) {
-	return 1;
-    }
-    return 0;
+    return (iter->t_limit == 1 || iter->idx == iter->t_limit);
 }
 
 int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
     ASSERT(iter && is_map(iter->map));
     ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
-    if (iter->t_limit == 1 || iter->idx == 0) {
-	return 1;
-    }
-    return 0;
+    return (iter->t_limit == 1 || iter->idx == 0);
 }
 
 
-- 
cgit v1.2.3


From 00f9be42e43913bce9b110382e55bfbdaa9406d0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Jan 2014 18:13:45 +0100
Subject: erts: Fix map iterator bug when reverting from end of map position

and simplify code by ignoring h_limit which is always zero.
---
 erts/emulator/beam/erl_nif.c | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c6f7c8adb5..6fceb8a0ba 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1759,13 +1759,10 @@ int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
     ASSERT(iter && is_map(iter->map));
     if (iter->idx < iter->t_limit) {
 	iter->idx++;
-	if (iter->idx != iter->t_limit) {
-	    iter->ks++;
-	    iter->vs++;
-	    return 1;
-	}
+	iter->ks++;
+	iter->vs++;
     }
-    return 0;
+    return (iter->idx != iter->t_limit);
 }
 
 int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
@@ -1773,13 +1770,10 @@ int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
     ASSERT(iter && is_map(iter->map));
     if (iter->idx > 0) {
 	iter->idx--;
-	if (iter->idx != 0) {
-	    iter->ks--;
-	    iter->vs--;
-	    return 1;
-	}
+	iter->ks--;
+	iter->vs--;
     }
-    return 0;
+    return (iter->idx > 0);
 }
 
 int enif_map_iterator_get_pair(ErlNifEnv *env,
-- 
cgit v1.2.3


From a18853e6814ab42188e56343e6363ff94c794bb8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Jan 2014 18:21:26 +0100
Subject: erts: Optimize struct ErlNifMapIterator

No need to use 64bit integers on 32bit machines.
---
 erts/emulator/beam/erl_nif.h | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index ae7f63f923..7613446f64 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -103,6 +103,8 @@ typedef unsigned long long ERL_NIF_TERM;
 #  endif
 #endif
 
+typedef ERL_NIF_TERM ERL_NIF_UINT;
+
 struct enif_environment_t;
 typedef struct enif_environment_t ErlNifEnv;
 
@@ -175,15 +177,14 @@ typedef enum
 }ErlNifDirtyTaskFlags;
 #endif
 
-typedef struct
+typedef struct /* All fields all internal and may change */
 {
-    /* use a lot of memory, structure may change */
     ERL_NIF_TERM map;
-    ErlNifUInt64 h_limit;
-    ErlNifUInt64 t_limit;
-    ErlNifUInt64 idx;
+    ERL_NIF_UINT t_limit;
+    ERL_NIF_UINT idx;
     ERL_NIF_TERM *ks;
     ERL_NIF_TERM *vs;
+    void* __spare__[2]; /* for future additions to be ABI compatible (same struct size) */
 } ErlNifMapIterator;
 
 typedef enum {
-- 
cgit v1.2.3


From 9eddc8eb4c7f4ef45b51650750334a787c5f9ef7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 16 Jan 2014 23:45:42 +0100
Subject: erts: Fix compile error for halfword emulator

---
 erts/emulator/beam/erl_utils.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index c6247c7246..5b81d814c6 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -204,9 +204,9 @@ int eq(Eterm, Eterm);
 #if HALFWORD_HEAP
 Sint cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int);
 #define cmp_rel(A,A_BASE,B,B_BASE)       cmp_rel_opt(A,A_BASE,B,B_BASE,0)
-#define cmp_rel_exact(A,A_BASE,B,B_BASE) cmp_rel_opt(A,A_BASE,B,B_BASE,1)
-#define CMP(A,B)                         cmp_rel(A,NULL,B,NULL,0)
-#define CMP_TERM(A,B)                    cmp_rel(A,NULL,B,NULL,1)
+#define cmp_rel_term(A,A_BASE,B,B_BASE)  cmp_rel_opt(A,A_BASE,B,B_BASE,1)
+#define CMP(A,B)                         cmp_rel_opt(A,NULL,B,NULL,0)
+#define CMP_TERM(A,B)                    cmp_rel_opt(A,NULL,B,NULL,1)
 #else
 Sint cmp(Eterm, Eterm, int);
 #define cmp_rel(A,A_BASE,B,B_BASE)       cmp(A,B,0)
-- 
cgit v1.2.3


From f2b9a5623ec5c9a314560add82ef20a03ba91235 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 17 Jan 2014 12:32:42 +0100
Subject: erts: Change 'size' argument of enif_get_map_size from int* to
 size_t*

---
 erts/emulator/beam/erl_nif.c           | 2 +-
 erts/emulator/beam/erl_nif_api_funcs.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 6fceb8a0ba..c35f1fc2c6 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1610,7 +1610,7 @@ int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term)
     return is_map(term);
 }
 
-int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, int *size)
+int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)
 {
     if (is_map(term)) {
 	map_t *mp;
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 9c386a4c41..4121378e1a 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -150,7 +150,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
 #endif
 
 ERL_NIF_API_FUNC_DECL(int, enif_is_map, (ErlNifEnv* env, ERL_NIF_TERM term));
-ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term, int *size));
+ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term, size_t *size));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_new_map, (ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(int, enif_make_map_put, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out));
 ERL_NIF_API_FUNC_DECL(int, enif_get_map_value, (ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value));
-- 
cgit v1.2.3


From bc34b5dafaebf07d7185900310246f5752c0a6c4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 21 Jan 2014 12:21:11 +0100
Subject: erts: Add map construction to driver API

erl_drv_output_term() and erl_drv_send_term() can send messages
containing maps with the use of the new ERL_DRV_MAP.

The driver API minor version is updated as new functionality is added.
---
 erts/emulator/beam/erl_driver.h |  4 +++-
 erts/emulator/beam/erl_map.c    | 31 +++++++++++++++++++++++++++++
 erts/emulator/beam/erl_map.h    |  4 ++--
 erts/emulator/beam/external.c   | 31 ++---------------------------
 erts/emulator/beam/io.c         | 44 ++++++++++++++++++++++++++++++++++++++++-
 5 files changed, 81 insertions(+), 33 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 2bd3181bdc..ab9ee63104 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -605,6 +605,8 @@ EXTERN int null_func(void);
 #define ERL_DRV_INT64       ((ErlDrvTermData) 15) /* ErlDrvSInt64 * */
 #define ERL_DRV_UINT64      ((ErlDrvTermData) 16) /* ErlDrvUInt64 * */
 
+#define ERL_DRV_MAP         ((ErlDrvTermData) 17) /* ErlDrvUInt */
+
 #ifndef ERL_DRIVER_TYPES_ONLY
 
 /* make terms for driver_output_term and driver_send_term */
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 57f156509c..2fff7f9390 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -786,3 +786,34 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
+
+int erts_validate_and_sort_map(map_t* mp)
+{
+    Eterm *ks  = map_get_keys(mp);
+    Eterm *vs  = map_get_values(mp);
+    Uint   sz  = map_get_size(mp);
+    Uint   ix,jx;
+    Eterm  tmp;
+    int c;
+
+    /* sort */
+
+    for (ix = 1; ix < sz; ix++) {
+	jx = ix;
+	while( jx > 0 && (c = CMP_TERM(ks[jx],ks[jx-1])) <= 0 ) {
+	    /* identical key -> error */
+	    if (c == 0) return 0;
+
+	    tmp = ks[jx];
+	    ks[jx] = ks[jx - 1];
+	    ks[jx - 1] = tmp;
+
+	    tmp = vs[jx];
+	    vs[jx] = vs[jx - 1];
+	    vs[jx - 1] = tmp;
+
+	    jx--;
+	}
+    }
+    return 1;
+}
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 616ecd24ce..cfacb2ec28 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2013. All Rights Reserved.
+ * Copyright Ericsson AB 2014. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -67,6 +67,6 @@ int   erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res
 int   erts_maps_find(Eterm key, Eterm map, Eterm *value);
 int   erts_maps_get(Eterm key, Eterm map, Eterm *value);
 int   erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
-
+int   erts_validate_and_sort_map(map_t* map);
 #endif
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 57251286c8..a4cc3435c3 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3795,37 +3795,10 @@ dec_term_atom_common:
      */
 
     while (maps_head) {
-	map_t *mp  = (map_t*)maps_head;
-	Eterm *ks  = map_get_keys(mp);
-	Eterm *vs  = map_get_values(mp);
-	Uint   sz  = map_get_size(mp);
-	Uint   ix,jx;
-	Eterm  tmp;
-	int c;
-
 	next  = (Eterm *)(EXPAND_POINTER(*maps_head));
-
-	/* sort */
-
-	for ( ix = 1; ix < sz; ix++) {
-	    jx = ix;
-	    while( jx > 0 && (c = CMP_TERM(ks[jx],ks[jx-1])) <= 0 ) {
-		/* identical key -> error */
-		if (c == 0) goto error;
-
-		tmp = ks[jx];
-		ks[jx] = ks[jx - 1];
-		ks[jx - 1] = tmp;
-
-		tmp = vs[jx];
-		vs[jx] = vs[jx - 1];
-		vs[jx - 1] = tmp;
-
-		jx--;
-	    }
-
-	}
 	*maps_head = MAP_HEADER;
+	if (!erts_validate_and_sort_map((map_t*)maps_head))
+	    goto error;
 	maps_head  = next;
     }
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 49af86b36a..3b16cdeb4a 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -46,6 +46,7 @@
 #define ERTS_WANT_EXTERNAL_TAGS
 #include "external.h"
 #include "dtrace-wrapper.h"
+#include "erl_map.h"
 
 extern ErlDrvEntry fd_driver_entry;
 extern ErlDrvEntry vanilla_driver_entry;
@@ -5293,6 +5294,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    depth++;
 	    break;
 	}
+	case ERL_DRV_MAP: { /* int */
+	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
+	    if ((int) ptr[0] < 0) ERTS_DDT_FAIL;
+	    need += MAP_HEADER_SIZE + 1 + 2*ptr[0];
+	    depth -= 2*ptr[0];
+	    if (depth < 0) ERTS_DDT_FAIL;
+	    ptr++;
+	    depth++;
+	    break;
+	}
+
 	default:
 	    ERTS_DDT_FAIL;
 	}
@@ -5529,6 +5541,36 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    ptr += 2;
 	    break;
 
+	case ERL_DRV_MAP: { /* int */
+	    int size = (int)ptr[0];
+	    Eterm* tp = hp;
+	    Eterm* vp;
+	    map_t *mp;
+
+	    *tp = make_arityval(size);
+
+	    hp += 1 + size;
+	    mp = (map_t*)hp;
+	    mp->thing_word = MAP_HEADER;
+	    mp->size = size;
+	    mp->keys = make_tuple(tp);
+	    mess = make_map(mp);
+
+	    hp += MAP_HEADER_SIZE + size;   /* advance "heap" pointer */
+
+	    tp += size;    /* point at last key */
+	    vp = hp - 1;   /* point at last value */
+
+	    while(size--) {
+		*vp-- = ESTACK_POP(stack);
+		*tp-- = ESTACK_POP(stack);
+	    }
+	    if (!erts_validate_and_sort_map(mp))
+		ERTS_DDT_FAIL;
+	    ptr++;
+	    break;
+	}
+
 	}
 	ESTACK_PUSH(stack, mess);
     }
-- 
cgit v1.2.3


From 543074e7aab966a597e405bc52c94e57358663a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 27 Jan 2014 14:47:42 +0100
Subject: erts: Fixup enif_make_map_put on windows

---
 erts/emulator/beam/erl_nif_api_funcs.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 4121378e1a..d7c554e60b 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -300,7 +300,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa
 #  define enif_is_map ERL_NIF_API_FUNC_MACRO(enif_is_map)
 #  define enif_get_map_size ERL_NIF_API_FUNC_MACRO(enif_get_map_size)
 #  define enif_make_new_map ERL_NIF_API_FUNC_MACRO(enif_make_new_map)
-#  define enif_make_map_put ERL_NIF_API_FUNC_MACRO(enif_map_map_put)
+#  define enif_make_map_put ERL_NIF_API_FUNC_MACRO(enif_make_map_put)
 #  define enif_get_map_value ERL_NIF_API_FUNC_MACRO(enif_get_map_value)
 #  define enif_make_map_update ERL_NIF_API_FUNC_MACRO(enif_make_map_update)
 #  define enif_make_map_remove ERL_NIF_API_FUNC_MACRO(enif_make_map_remove)
-- 
cgit v1.2.3


From b7c7cdb8ac922f571d0ace3a3cd0cd5149606480 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 29 Jan 2014 16:51:06 +0100
Subject: Make it easier to revert +M<S>acul default

---
 erts/emulator/beam/erl_alloc.c | 117 +++++++++++++----------------------------
 1 file changed, 38 insertions(+), 79 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index c6b324dc15..ca7e39041b 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -242,32 +242,13 @@ do {									\
     sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init));	\
 } while (0)
 
-#if ERTS_ALC_DEFAULT_ACUL \
-    || ERTS_ALC_DEFAULT_ACUL_LL_ALLOC \
-    || ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
-
-static ERTS_INLINE void
-set_default_acul(struct au_init *ip, int acul)
-{
-    ip->thr_spec = 1;
-    ip->atype = AOFIRSTFIT;
-    ip->init.aoff.flavor = AOFF_BF;
-    ip->init.util.acul = acul;
-}
-
-#endif
-
 static void
 set_default_sl_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
-#else
     ip->thr_spec		= 1;
     ip->atype			= GOODFIT;
-#endif
     ip->init.util.name_prefix	= "sl_";
     ip->init.util.alloc_no	= ERTS_ALC_A_SHORT_LIVED;
 #ifndef SMALL_MEMORY
@@ -281,7 +262,7 @@ set_default_sl_alloc_opts(struct au_init *ip)
     ip->init.util.force         = 1;
     ip->init.util.low_mem       = 1;
 #endif
-
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
 static void
@@ -289,12 +270,8 @@ set_default_std_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
-#else
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
-#endif
     ip->init.util.name_prefix	= "std_";
     ip->init.util.alloc_no	= ERTS_ALC_A_STANDARD;
 #ifndef SMALL_MEMORY
@@ -303,6 +280,7 @@ set_default_std_alloc_opts(struct au_init *ip)
     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_STANDARD;
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
 static void
@@ -310,13 +288,9 @@ set_default_ll_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL_LL_ALLOC
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_LL_ALLOC);
-#else
     ip->thr_spec		= 0;
     ip->atype			= BESTFIT;
     ip->init.bf.ao		= 1;
-#endif
     ip->init.util.ramv		= 0;
     ip->init.util.mmsbc		= 0;
     ip->init.util.sbct		= ~((UWord) 0);
@@ -332,6 +306,7 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.util.rsbcst	= 0;
     ip->init.util.rsbcmt	= 0;
     ip->init.util.rmbcmt	= 0;
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL_LL_ALLOC;
 }
 
 static void
@@ -363,12 +338,8 @@ set_default_eheap_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC);
-#else
     ip->thr_spec		= 1;
     ip->atype			= GOODFIT;
-#endif
     ip->init.util.name_prefix	= "eheap_";
     ip->init.util.alloc_no	= ERTS_ALC_A_EHEAP;
 #ifndef SMALL_MEMORY
@@ -382,6 +353,7 @@ set_default_eheap_alloc_opts(struct au_init *ip)
     ip->init.util.force         = 1;
     ip->init.util.low_mem       = 1;
 #endif
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC;
 }
 
 static void
@@ -389,12 +361,8 @@ set_default_binary_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
-#else
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
-#endif
     ip->init.util.name_prefix	= "binary_";
     ip->init.util.alloc_no	= ERTS_ALC_A_BINARY;
 #ifndef SMALL_MEMORY
@@ -403,6 +371,7 @@ set_default_binary_alloc_opts(struct au_init *ip)
     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_BINARY;
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
 static void
@@ -410,12 +379,8 @@ set_default_ets_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
-#else
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
-#endif
     ip->init.util.name_prefix	= "ets_";
     ip->init.util.alloc_no	= ERTS_ALC_A_ETS;
 #ifndef SMALL_MEMORY
@@ -424,6 +389,7 @@ set_default_ets_alloc_opts(struct au_init *ip)
     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_ETS;
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
 static void
@@ -431,12 +397,8 @@ set_default_driver_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
-#else
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
-#endif
     ip->init.util.name_prefix	= "driver_";
     ip->init.util.alloc_no	= ERTS_ALC_A_DRIVER;
 #ifndef SMALL_MEMORY
@@ -445,6 +407,7 @@ set_default_driver_alloc_opts(struct au_init *ip)
     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_DRIVER;
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
 static void
@@ -453,12 +416,8 @@ set_default_fix_alloc_opts(struct au_init *ip,
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
-#if ERTS_ALC_DEFAULT_ACUL
-    set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
-#else
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
-#endif
     ip->init.bf.ao = 1;
     ip->init.util.name_prefix	= "fix_";
     ip->init.util.fix_type_size	= fix_type_sizes;
@@ -469,6 +428,7 @@ set_default_fix_alloc_opts(struct au_init *ip,
     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_FIXED_SIZE;
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
 #ifdef ERTS_SMP
@@ -560,26 +520,25 @@ strategy_support_carrier_migration(struct au_init *auip)
 }
 
 static ERTS_INLINE void
-check_disable_carrier_migration(struct au_init *auip)
-{
-    if (!strategy_support_carrier_migration(auip) || !auip->thr_spec)
-	auip->init.util.acul = 0;
-}
-
-static ERTS_INLINE void
-ensure_carrier_migration_support(struct au_init *auip)
+adjust_carrier_migration_support(struct au_init *auip)
 {
-    auip->thr_spec = 1; /* Need thread preferred */
+#ifdef ERTS_SMP
+    if (auip->init.util.acul) {
+	auip->thr_spec = -1; /* Need thread preferred */
 
-    /*
-     * If strategy cannot handle carrier migration,
-     * default to a strategy that can...
-     */
-    if (!strategy_support_carrier_migration(auip)) {
-	/* Default to aoffcbf */
-	auip->atype = AOFIRSTFIT;
-	auip->init.aoff.flavor = AOFF_BF;
+	/*
+	 * If strategy cannot handle carrier migration,
+	 * default to a strategy that can...
+	 */
+	if (!strategy_support_carrier_migration(auip)) {
+	    /* Default to aoffcbf */
+	    auip->atype = AOFIRSTFIT;
+	    auip->init.aoff.flavor = AOFF_BF;
+	}
     }
+#else
+    auip->init.util.acul = 0;
+#endif
 }
 
 void
@@ -683,15 +642,16 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.fix_alloc.thr_spec = 0;	
     }
 
-    check_disable_carrier_migration(&init.sl_alloc);
-    check_disable_carrier_migration(&init.std_alloc);
-    check_disable_carrier_migration(&init.ll_alloc);
-    check_disable_carrier_migration(&init.eheap_alloc);
-    check_disable_carrier_migration(&init.binary_alloc);
-    check_disable_carrier_migration(&init.ets_alloc);
-    check_disable_carrier_migration(&init.driver_alloc);
-    check_disable_carrier_migration(&init.fix_alloc);
-
+    /* Make adjustments for carrier migration support */
+    init.temp_alloc.init.util.acul = 0;
+    adjust_carrier_migration_support(&init.sl_alloc);
+    adjust_carrier_migration_support(&init.std_alloc);
+    adjust_carrier_migration_support(&init.ll_alloc);
+    adjust_carrier_migration_support(&init.eheap_alloc);
+    adjust_carrier_migration_support(&init.binary_alloc);
+    adjust_carrier_migration_support(&init.ets_alloc);
+    adjust_carrier_migration_support(&init.driver_alloc);
+    adjust_carrier_migration_support(&init.fix_alloc);
 
 #ifdef ERTS_SMP
     /* Only temp_alloc can use thread specific interface */
@@ -1290,8 +1250,6 @@ handle_au_arg(struct au_init *auip,
 		    break;
 		}
 	    }
-	    ensure_carrier_migration_support(auip);
-
 	    auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip);
 	}
 	else if(has_prefix("asbcst", sub_param)) {
@@ -1328,7 +1286,8 @@ handle_au_arg(struct au_init *auip,
 	    else {
 		bad_value(param, sub_param + 1, alg);
 	    }
-	    check_disable_carrier_migration(auip);
+	    if (!strategy_support_carrier_migration(auip))
+		auip->init.util.acul = 0;
 	}
 	else
 	    goto bad_switch;
@@ -1409,7 +1368,7 @@ handle_au_arg(struct au_init *auip,
 	}
 	else if (res == 0) {
 	    auip->thr_spec = 0;
-	    check_disable_carrier_migration(auip);
+	    auip->init.util.acul = 0;
 	    break;
 	}
 	goto bad_switch;
@@ -1607,7 +1566,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 
 			    for (a = 0; a < aui_sz; a++) {
 				aui[a]->thr_spec = 0;
-				check_disable_carrier_migration(aui[a]);
+				aui[a]->init.util.acul = 0;
 				aui[a]->init.util.ramv = 0;
 				aui[a]->init.util.lmbcs = 5*1024*1024;
 			    }
-- 
cgit v1.2.3


From 4a0aa563b9776e33bca19459bfefc81b7ac900c9 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 31 Jan 2014 16:52:51 +0100
Subject: Fix +Mea config

---
 erts/emulator/beam/erl_alloc.c | 37 +++++++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index ca7e39041b..64b3e01ed4 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -629,8 +629,21 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     init.fix_alloc.thr_spec = 0;
 #endif
 
+    /* Make adjustments for carrier migration support */
+    init.temp_alloc.init.util.acul = 0;
+    adjust_carrier_migration_support(&init.sl_alloc);
+    adjust_carrier_migration_support(&init.std_alloc);
+    adjust_carrier_migration_support(&init.ll_alloc);
+    adjust_carrier_migration_support(&init.eheap_alloc);
+    adjust_carrier_migration_support(&init.binary_alloc);
+    adjust_carrier_migration_support(&init.ets_alloc);
+    adjust_carrier_migration_support(&init.driver_alloc);
+    adjust_carrier_migration_support(&init.fix_alloc);
+
     if (init.erts_alloc_config) {
 	/* Adjust flags that erts_alloc_config won't like */
+
+	/* No thread specific instances */
 	init.temp_alloc.thr_spec = 0;
 	init.sl_alloc.thr_spec = 0;
 	init.std_alloc.thr_spec = 0;
@@ -639,20 +652,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.binary_alloc.thr_spec = 0;
 	init.ets_alloc.thr_spec = 0;
 	init.driver_alloc.thr_spec = 0;
-	init.fix_alloc.thr_spec = 0;	
+	init.fix_alloc.thr_spec = 0;
+
+	/* No carrier migration */
+	init.temp_alloc.init.util.acul = 0;
+	init.sl_alloc.init.util.acul = 0;
+	init.std_alloc.init.util.acul = 0;
+	init.ll_alloc.init.util.acul = 0;
+	init.eheap_alloc.init.util.acul = 0;
+	init.binary_alloc.init.util.acul = 0;
+	init.ets_alloc.init.util.acul = 0;
+	init.driver_alloc.init.util.acul = 0;
+	init.fix_alloc.init.util.acul = 0;
     }
 
-    /* Make adjustments for carrier migration support */
-    init.temp_alloc.init.util.acul = 0;
-    adjust_carrier_migration_support(&init.sl_alloc);
-    adjust_carrier_migration_support(&init.std_alloc);
-    adjust_carrier_migration_support(&init.ll_alloc);
-    adjust_carrier_migration_support(&init.eheap_alloc);
-    adjust_carrier_migration_support(&init.binary_alloc);
-    adjust_carrier_migration_support(&init.ets_alloc);
-    adjust_carrier_migration_support(&init.driver_alloc);
-    adjust_carrier_migration_support(&init.fix_alloc);
-
 #ifdef ERTS_SMP
     /* Only temp_alloc can use thread specific interface */
     if (init.temp_alloc.thr_spec)
-- 
cgit v1.2.3


From 9ae4a522ea841a880d94e2b1e683e470cf2c5037 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 3 Feb 2014 17:58:24 +0100
Subject: erts: Fix bug in binary_to_term for maps

'maps_head' was not restored when yielding. Risk for crash increases with
size and number of maps in term.
---
 erts/emulator/beam/external.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index a4cc3435c3..9fb2dbd8bf 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1169,6 +1169,7 @@ typedef struct {
     Eterm* hp_end;
     int remaining_n;
     char* remaining_bytes;
+    Eterm* maps_head;
 } B2TDecodeContext;
 
 typedef struct {
@@ -1486,6 +1487,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size);
             ctx->u.dc.hp       = ctx->u.dc.hp_start;
             ctx->u.dc.hp_end   = ctx->u.dc.hp_start + ctx->heap_size;
+	    ctx->u.dc.maps_head = NULL;
             ctx->state = B2TDecode;
             /*fall through*/
 	case B2TDecode:
@@ -2878,7 +2880,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
     int n;
     ErtsAtomEncoding char_enc;
     register Eterm* hp;        /* Please don't take the address of hp */
-    Eterm *maps_head = NULL; /* for validation of maps */
+    Eterm *maps_head;   /* for validation of maps */
     Eterm* next;
     SWord reds;
 
@@ -2888,6 +2890,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
         next     = ctx->u.dc.next;
         ep       = ctx->u.dc.ep;
         hpp      = &ctx->u.dc.hp;
+	maps_head = ctx->u.dc.maps_head;
 
         if (ctx->state != B2TDecode) {
             int n_limit = reds;
@@ -2968,6 +2971,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
         reds = ERTS_SWORD_MAX;
         next = objp;
         *next = (Eterm) (UWord) NULL;
+	maps_head = NULL;
     }
     hp = *hpp;
 
@@ -3780,6 +3784,7 @@ dec_term_atom_common:
                     ctx->u.dc.ep = ep;
                     ctx->u.dc.next = next;
                     ctx->u.dc.hp = hp;
+		    ctx->u.dc.maps_head = maps_head;
                     ctx->reds = 0;
                     return NULL;
                 }
-- 
cgit v1.2.3


From 1f7324d0fd72b4137fb1f21b0b759ffebdf7ea90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 5 Feb 2014 19:07:22 +0100
Subject: erts: Fix Maps for beam_load

Map source may be anything, not only registers.
---
 erts/emulator/beam/beam_emu.c | 4 ++--
 erts/emulator/beam/ops.tab    | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 89d9442526..4d8ea354ce 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2355,7 +2355,7 @@ void process_main(void)
      Next(4+Arg(3));
  }
 
- OpCase(update_map_assoc_jddII): {
+ OpCase(update_map_assoc_jsdII): {
      Eterm res;
      Eterm map;
 
@@ -2373,7 +2373,7 @@ void process_main(void)
      }
  }
 
- OpCase(update_map_exact_jddII): {
+ OpCase(update_map_exact_jsdII): {
      Eterm res;
      Eterm map;
 
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index f35997efee..9ea3a3a8ea 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1482,8 +1482,8 @@ put_map_exact F Src Dst Live Size Rest=* => \
 	      update_map_exact F Src Dst Live Size Rest
 
 new_map j d I I
-update_map_assoc j d d I I
-update_map_exact j d d I I
+update_map_assoc j s d I I
+update_map_exact j s d I I
 
 is_map Fail cq => jump Fail
 
-- 
cgit v1.2.3


From 4082c3d1ad3d0e8bb85706c5f3b2bde97bb43fcb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 5 Feb 2014 19:41:28 +0100
Subject: erts: Fix NIF bug when load/upgrade fails after
 enif_open_resource_type

..has been successfully called.

Opened resource types (created or taken-over) were left "hanging"
leading both to memory leakage and other more strange and serious behavior.

Now a proper rollback is done.
---
 erts/emulator/beam/erl_nif.c | 113 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 91 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index dc285b3cf7..b2fd3dcd24 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1235,6 +1235,19 @@ static void steal_resource_type(ErlNifResourceType* type)
     }
 }
 
+/* The opened_rt_list is used by enif_open_resource_type()
+ * in order to rollback "creates" and "take-overs" in case the load fails.
+ */
+struct opened_resource_type
+{
+    struct opened_resource_type* next;
+
+    ErlNifResourceFlags op;
+    ErlNifResourceType* type;
+    ErlNifResourceDtor* new_dtor;
+};
+static struct opened_resource_type* opened_rt_list = NULL;
+
 ErlNifResourceType*
 enif_open_resource_type(ErlNifEnv* env,
 			const char* module_str, 
@@ -1256,22 +1269,21 @@ enif_open_resource_type(ErlNifEnv* env,
     if (type == NULL) {
 	if (flags & ERL_NIF_RT_CREATE) {
 	    type = erts_alloc(ERTS_ALC_T_NIF,
-			      sizeof(struct enif_resource_type_t)); 
-	    type->dtor = dtor;
+			      sizeof(struct enif_resource_type_t));
 	    type->module = module_am;
 	    type->name = name_am;
 	    erts_refc_init(&type->refc, 1);
-	    type->owner = env->mod_nif;
-	    type->prev = &resource_type_list;
-	    type->next = resource_type_list.next;
-	    type->next->prev = type;
-	    type->prev->next = type;
 	    op = ERL_NIF_RT_CREATE;
+	#ifdef DEBUG
+	    type->dtor = (void*)1;
+	    type->owner = (void*)2;
+	    type->prev = (void*)3;
+	    type->next = (void*)4;
+	#endif
 	}
     }
     else {
-	if (flags & ERL_NIF_RT_TAKEOVER) {	
-	    steal_resource_type(type);
+	if (flags & ERL_NIF_RT_TAKEOVER) {
 	    op = ERL_NIF_RT_TAKEOVER;
 	}
 	else {
@@ -1279,12 +1291,13 @@ enif_open_resource_type(ErlNifEnv* env,
 	}
     }
     if (type != NULL) {
-	type->owner = env->mod_nif;
-	type->dtor = dtor;
-	if (type->dtor != NULL) {
-	    erts_refc_inc(&type->owner->rt_dtor_cnt, 1);
-	}
-	erts_refc_inc(&type->owner->rt_cnt, 1);    
+	struct opened_resource_type* ort = erts_alloc(ERTS_ALC_T_TMP,
+						sizeof(struct opened_resource_type));
+	ort->op = op;
+	ort->type = type;
+	ort->new_dtor = dtor;
+	ort->next = opened_rt_list;
+	opened_rt_list = ort;
     }
     if (tried != NULL) {
 	*tried = op;
@@ -1292,6 +1305,51 @@ enif_open_resource_type(ErlNifEnv* env,
     return type;
 }
 
+static void commit_opened_resource_types(struct erl_module_nif* lib)
+{
+    while (opened_rt_list) {
+	struct opened_resource_type* ort = opened_rt_list;
+
+	ErlNifResourceType* type = ort->type;
+
+	if (ort->op == ERL_NIF_RT_CREATE) {
+	    type->prev = &resource_type_list;
+	    type->next = resource_type_list.next;
+	    type->next->prev = type;
+	    type->prev->next = type;
+	}
+	else { /* ERL_NIF_RT_TAKEOVER */
+	    steal_resource_type(type);
+	}
+
+	type->owner = lib;
+	type->dtor = ort->new_dtor;
+
+	if (type->dtor != NULL) {
+	    erts_refc_inc(&lib->rt_dtor_cnt, 1);
+	}
+	erts_refc_inc(&lib->rt_cnt, 1);
+
+	opened_rt_list = ort->next;
+	erts_free(ERTS_ALC_T_TMP, ort);
+    }
+}
+
+static void rollback_opened_resource_types(void)
+{
+    while (opened_rt_list) {
+	struct opened_resource_type* ort = opened_rt_list;
+
+	if (ort->op == ERL_NIF_RT_CREATE) {
+	    erts_free(ERTS_ALC_T_NIF, ort->type);
+	}
+
+	opened_rt_list = ort->next;
+	erts_free(ERTS_ALC_T_TMP, ort);
+    }
+}
+
+
 static void nif_resource_dtor(Binary* bin)
 {
     ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin);
@@ -1317,6 +1375,8 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 {
     Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor);
     ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin);
+
+    ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */
     resource->type = type;
     erts_refc_inc(&bin->refc, 1);
 #ifdef DEBUG
@@ -1696,9 +1756,15 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     lib->entry = entry;
     erts_refc_init(&lib->rt_cnt, 0);
     erts_refc_init(&lib->rt_dtor_cnt, 0);
+    ASSERT(opened_rt_list == NULL);
     lib->mod = mod;
     env.mod_nif = lib;
-    if (mod->curr.nif != NULL) { /* Reload */
+    if (mod->curr.nif != NULL) { /*************** Reload ******************/
+	/*
+	 * Repeated load_nif calls from same Erlang module instance ("reload")
+	 * is deprecated and was only ment as a development feature not to
+	 * be used in production systems. (See warning below)
+	 */
 	int k;
         lib->priv_data = mod->curr.nif->priv_data;
 
@@ -1730,6 +1796,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful.");
 	}
 	else {
+	    commit_opened_resource_types(lib);
 	    mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */
 	    erts_unload_nif(mod->curr.nif);
 	    reload_warning = 1;
@@ -1737,7 +1804,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     }
     else {
 	lib->priv_data = NULL;
-	if (mod->old.nif != NULL) { /* Upgrade */
+	if (mod->old.nif != NULL) { /**************** Upgrade ***************/
 	    void* prev_old_data = mod->old.nif->priv_data;
 	    if (entry->upgrade == NULL) {
 		ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library.");
@@ -1750,17 +1817,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		mod->old.nif->priv_data = prev_old_data;
 		ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful.");
 	    }
-	    /*else if (mod->old_nif->priv_data != prev_old_data) {
-		refresh_cached_nif_data(mod->old_code, mod->old_nif);
-	    }*/
+	    else
+		commit_opened_resource_types(lib);
 	}
-	else if (entry->load != NULL) { /* Initial load */
+	else if (entry->load != NULL) { /********* Initial load ***********/
 	    erts_pre_nif(&env, BIF_P, lib);
 	    veto = entry->load(&env, &lib->priv_data, BIF_ARG_2);
 	    erts_post_nif(&env);
 	    if (veto) {
 		ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful.");
 	    }
+	    else
+		commit_opened_resource_types(lib);
 	}
     }
     if (ret == am_ok) {
@@ -1789,6 +1857,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     }
     else {
     error:
+	rollback_opened_resource_types();
 	ASSERT(ret != am_ok);
         if (lib != NULL) {
 	    erts_free(ERTS_ALC_T_NIF, lib);
-- 
cgit v1.2.3


From 963c06a5b72c2ac8d88a6910ee60248d6dca747d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 5 Feb 2014 19:11:07 +0100
Subject: erts: Maps must fail on exact updates of empty Maps

Exact updates on empty Maps must fail directly.
---
 erts/emulator/beam/beam_emu.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4d8ea354ce..1ae413d46e 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6614,7 +6614,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      */
 
     if (num_old == 0) {
-	return new_map(p, reg, I+1);
+	return THE_NON_VALUE;
     }
 
     /*
-- 
cgit v1.2.3


From 57573ab98e88ede4b6bca07574e537b6e4f82027 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Tue, 28 Jan 2014 22:26:14 -0500
Subject: allow optional whitespace in dirty scheduler erl options

The +SDcpu, +SDPcpu, and +SDio options did not properly handle having their
arguments immediately following them without intervening whitespace,
e.g. +SDio20 was treated as an error. Fix all the dirty scheduler command
line options so they handle optional whitespace between them and their
associated arguments.
---
 erts/emulator/beam/erl_init.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index c17256f466..e6c05adec1 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -835,7 +835,7 @@ early_init(int *argc, char **argv) /*
 		    else if (argv[i][2] == 'D') {
 			char *arg;
 			char *type = argv[i]+3;
-			if (strcmp(type, "Pcpu") == 0) {
+			if (strncmp(type, "Pcpu", 4) == 0) {
 			    int ptot, ponln;
 			    arg = get_arg(argv[i]+7, argv[i+1], &i);
 			    switch (sscanf(arg, "%d:%d", &ptot, &ponln)) {
@@ -872,7 +872,7 @@ early_init(int *argc, char **argv) /*
 			    VERBOSE(DEBUG_SYSTEM,
 				    ("using %d:%d dirty CPU scheduler percentages\n",
 				     dirty_cpu_scheds_pctg, dirty_cpu_scheds_onln_pctg));
-			} else if (strcmp(type, "cpu") == 0) {
+			} else if (strncmp(type, "cpu", 3) == 0) {
 			    int tot, onln;
 			    arg = get_arg(argv[i]+6, argv[i+1], &i);
 			    switch (sscanf(arg, "%d:%d", &tot, &onln)) {
@@ -923,7 +923,7 @@ early_init(int *argc, char **argv) /*
 			    }
 			    VERBOSE(DEBUG_SYSTEM,
 				    ("using %d:%d dirty CPU scheduler(s)\n", tot, onln));
-			} else if (strcmp(type, "io") == 0) {
+			} else if (strncmp(type, "io", 2) == 0) {
 			    arg = get_arg(argv[i]+5, argv[i+1], &i);
 			    dirty_io_scheds = atoi(arg);
 			    if (dirty_io_scheds < 0 ||
-- 
cgit v1.2.3


From 04f4d9656724f465352df9582773393a165c8803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 19 Feb 2014 14:16:06 +0100
Subject: beam/utils.c: Change back cmp() to only take two arguments

Commit d5c238473b9cec819d93faaef4ccc00ddb60465f added a third
argument to the cmp() function, but did not change the code generation
of native code that called cmp().

Name the new functions with an "erts_" prefix, and reinstate the
old cmp() function with two arguments.

We don't have to keep a cmp() function in the halfword emulator,
since HiPE does not support the halfword emulator.
---
 erts/emulator/beam/erl_utils.h | 21 +++++++++++----------
 erts/emulator/beam/utils.c     | 16 +++++++++++++---
 2 files changed, 24 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 5b81d814c6..43a19e97f1 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -202,17 +202,18 @@ int eq(Eterm, Eterm);
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
 #if HALFWORD_HEAP
-Sint cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int);
-#define cmp_rel(A,A_BASE,B,B_BASE)       cmp_rel_opt(A,A_BASE,B,B_BASE,0)
-#define cmp_rel_term(A,A_BASE,B,B_BASE)  cmp_rel_opt(A,A_BASE,B,B_BASE,1)
-#define CMP(A,B)                         cmp_rel_opt(A,NULL,B,NULL,0)
-#define CMP_TERM(A,B)                    cmp_rel_opt(A,NULL,B,NULL,1)
+Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int);
+#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0)
+#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1)
+#define CMP(A,B)                         erts_cmp_rel_opt(A,NULL,B,NULL,0)
+#define CMP_TERM(A,B)                    erts_cmp_rel_opt(A,NULL,B,NULL,1)
 #else
-Sint cmp(Eterm, Eterm, int);
-#define cmp_rel(A,A_BASE,B,B_BASE)       cmp(A,B,0)
-#define cmp_rel_term(A,A_BASE,B,B_BASE)  cmp(A,B,1)
-#define CMP(A,B)                         cmp(A,B,0)
-#define CMP_TERM(A,B)                    cmp(A,B,1)
+Sint cmp(Eterm, Eterm);
+Sint erts_cmp(Eterm, Eterm, int);
+#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp(A,B,0)
+#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp(A,B,1)
+#define CMP(A,B)                         erts_cmp(A,B,0)
+#define CMP_TERM(A,B)                    erts_cmp(A,B,1)
 #endif
 
 #define cmp_lt(a,b)          (CMP((a),(b)) <  0)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index bc4a05d385..7da555b18d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2425,14 +2425,24 @@ static int cmp_atoms(Eterm a, Eterm b)
 		    bb->name+3, bb->len-3);
 }
 
-/* cmp(Eterm a, Eterm b, int exact)
+#if !HALFWORD_HEAP
+/* cmp(Eterm a, Eterm b)
+ *  For compatibility with HiPE - arith-based compare.
+ */
+Sint cmp(Eterm a, Eterm b)
+{
+    return erts_cmp(a, b, 0);
+}
+#endif
+
+/* erts_cmp(Eterm a, Eterm b, int exact)
  * exact = 1 -> term-based compare
  * exact = 0 -> arith-based compare
  */
 #if HALFWORD_HEAP
-Sint cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact)
+Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact)
 #else
-Sint cmp(Eterm a, Eterm b, int exact)
+Sint erts_cmp(Eterm a, Eterm b, int exact)
 #endif
 {
     DECLARE_WSTACK(stack);
-- 
cgit v1.2.3


From 3fd4e3e770fbed95fd6fa2d547945ac71256339c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Sun, 16 Feb 2014 00:54:08 +0100
Subject: Misc adjustments of OTP version

---
 erts/emulator/beam/erl_bif_info.c | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f25b4dbae5..2adba9b240 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -64,7 +64,7 @@ static Export *gather_gc_info_res_trap;
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
-static char otp_correction_package[] = ERLANG_OTP_CORRECTION_PACKAGE;
+static char otp_version[] = ERLANG_OTP_VERSION;
 /* Keep erts_system_version as a global variable for easy access from a core */
 static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     "%s"
@@ -312,7 +312,7 @@ erts_print_system_version(int to, void *arg, Process *c_p)
     int i, rc = -1;
     char *rc_str = "";
     char rc_buf[100];
-    char *ocp = otp_correction_package;
+    char *ov = otp_version;
 #ifdef ERTS_SMP
     Uint total, online, active;
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -323,9 +323,9 @@ erts_print_system_version(int to, void *arg, Process *c_p)
     (void) erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 0);
 #endif
 #endif
-    for (i = 0; i < sizeof(otp_correction_package)-4; i++) {
-	if (ocp[i] == '-' && ocp[i+1] == 'r' && ocp[i+2] == 'c')
-	    rc = atoi(&ocp[i+3]);
+    for (i = 0; i < sizeof(otp_version)-4; i++) {
+	if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c')
+	    rc = atoi(&ov[i+3]);
     }
     if (rc >= 0) {
 	if (rc == 0)
@@ -2448,10 +2448,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	    DECL_AM(unknown);
 	    BIF_RET(AM_unknown);
 	}
-    } else if (ERTS_IS_ATOM_STR("otp_correction_package", BIF_ARG_1)) {
-	int n = sizeof(ERLANG_OTP_CORRECTION_PACKAGE)-1;
-	hp = HAlloc(BIF_P, 2*n);
-	BIF_RET(buf_to_intlist(&hp, ERLANG_OTP_CORRECTION_PACKAGE, n, NIL));
     } else if (ERTS_IS_ATOM_STR("otp_release", BIF_ARG_1)) {
 	int n = sizeof(ERLANG_OTP_RELEASE)-1;
 	hp = HAlloc(BIF_P, 2*n);
-- 
cgit v1.2.3


From 3e206565c2032a24b03ca13c313d1879d74dfec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 12 Feb 2014 17:04:26 +0100
Subject: erts: Introduce new instructions for combined key fetches

---
 erts/emulator/beam/beam_emu.c  | 120 +++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/beam_load.c |  46 ++++++++++++++++
 erts/emulator/beam/ops.tab     |  23 ++++++--
 3 files changed, 184 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1ae413d46e..6f295530b9 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2355,6 +2355,126 @@ void process_main(void)
      Next(4+Arg(3));
  }
 
+ OpCase(i_has_map_fields_fsI): {
+    map_t* mp;
+    Eterm map;
+    Eterm field;
+    Eterm *ks;
+    BeamInstr* fs;
+    Uint sz,n;
+
+    GetArg1(1, map);
+
+    /* this instruction assumes Arg1 is a map,
+     * i.e. that it follows a test is_map if needed.
+     */
+
+    mp = (map_t *)map_val(map);
+    sz = map_get_size(mp);
+
+    if (sz == 0) {
+	SET_I((BeamInstr *) Arg(0));
+	goto has_map_fields_fail;
+    }
+
+    ks = map_get_keys(mp);
+    n  = (Uint)Arg(2);
+    fs = &Arg(3); /* pattern fields */
+
+    ASSERT(n>0);
+
+    while(sz) {
+	field = (Eterm)*fs;
+	if (EQ(field,*ks)) {
+	    n--;
+	    fs++;
+	    if (n == 0) break;
+	}
+	ks++; sz--;
+    }
+
+    if (n) {
+	SET_I((BeamInstr *) Arg(0));
+	goto has_map_fields_fail;
+    }
+
+    I += 4 + Arg(2);
+has_map_fields_fail:
+    ASSERT(VALID_INSTR(*I));
+    Goto(*I);
+ }
+
+#define PUT_TERM_REG(term, desc)				\
+do {								\
+    switch ((desc) & _TAG_IMMED1_MASK) {			\
+    case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
+	r(0) = (term);						\
+	break;							\
+    case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
+	x((desc) >> _TAG_IMMED1_SIZE) = (term);			\
+	break;							\
+    case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
+	y((desc) >> _TAG_IMMED1_SIZE) = (term);			\
+	break;							\
+    default:							\
+	ASSERT(0);						\
+	break;							\
+    }								\
+} while(0)
+
+ OpCase(i_get_map_elements_fsI): {
+    Eterm map;
+    map_t *mp;
+    Eterm field;
+    Eterm *ks;
+    Eterm *vs;
+    BeamInstr *fs;
+    Uint sz,n;
+
+    GetArg1(1, map);
+
+    /* this instruction assumes Arg1 is a map,
+     * i.e. that it follows a test is_map if needed.
+     */
+
+    mp = (map_t *)map_val(map);
+    sz = map_get_size(mp);
+
+    if (sz == 0) {
+	SET_I((BeamInstr *) Arg(0));
+	goto get_map_elements_fail;
+    }
+
+    n  = (Uint)Arg(2) / 2;
+    fs = &Arg(3); /* pattern fields and target registers */
+    ks = map_get_keys(mp);
+    vs = map_get_values(mp);
+
+    while(sz) {
+	field = (Eterm)*fs;
+	if (EQ(field,*ks)) {
+	    PUT_TERM_REG(*vs, fs[1]);
+	    n--;
+	    fs += 2;
+	    /* no more values to fetch, we are done */
+	    if (n == 0) break;
+	}
+	ks++; sz--;
+	vs++;
+    }
+
+    if (n) {
+	SET_I((BeamInstr *) Arg(0));
+	goto get_map_elements_fail;
+    }
+
+    I += 4 + Arg(2);
+get_map_elements_fail:
+    ASSERT(VALID_INSTR(*I));
+    Goto(*I);
+ }
+#undef PUT_TERM_REG
+
  OpCase(update_map_assoc_jsdII): {
      Eterm res;
      Eterm map;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index b589d1c930..5430ebb29d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -506,6 +506,9 @@ static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S,
 static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 			       GenOpArg Size, GenOpArg* Rest);
 
+static GenOp* gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
+                                  GenOpArg Size, GenOpArg* Rest);
+
 static int freeze_code(LoaderState* stp);
 
 static void final_touch(LoaderState* stp);
@@ -3951,6 +3954,49 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
     return op;
 }
 
+/*
+ * Replace a get_map_elements with one key to an instruction with one
+ * element
+ */
+
+static GenOp*
+gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
+		    GenOpArg Size, GenOpArg* Rest)
+{
+    GenOp* op;
+
+    ASSERT(Size.type == TAG_u);
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    op->op = genop_get_map_element_4;
+    op->arity = 4;
+
+    op->a[0] = Fail;
+    op->a[1] = Src;
+    op->a[2] = Rest[0];
+    op->a[3] = Rest[1];
+    return op;
+}
+
+static GenOp*
+gen_has_map_field(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
+		    GenOpArg Size, GenOpArg* Rest)
+{
+    GenOp* op;
+
+    ASSERT(Size.type == TAG_u);
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    op->op = genop_has_map_field_3;
+    op->arity = 4;
+
+    op->a[0] = Fail;
+    op->a[1] = Src;
+    op->a[2] = Rest[0];
+    return op;
+}
 
 /*
  * Freeze the code in memory, move the string table into place,
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 9ea3a3a8ea..8406f5344a 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1469,11 +1469,6 @@ apply_last I P
 # Map instructions in R17.
 #
 
-# put_map Fail Src Dst Live Size Rest=* => jump Fail
-# is_map Fail Src => jump Fail
-# has_map_field Fail Src Key => jump Fail
-# get_map_element Fail Src Key Dst => jump Fail
-
 put_map_assoc F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
 put_map_exact F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
 put_map_assoc F Src Dst Live Size Rest=* => \
@@ -1492,6 +1487,15 @@ is_map f r
 is_map f x
 is_map f y
 
+## Transform has_map_field(s) #{ K1 := _, K2 := _ }
+
+has_map_field/3
+
+has_map_fields Fail Src Size=u==1 Rest=* => gen_has_map_field(Fail,Src,Size,Rest)
+has_map_fields Fail Src Size Rest=* => i_has_map_fields Fail Src Size Rest
+
+i_has_map_fields f s I
+
 has_map_field Fail Src=rxy Key=arxy => i_has_map_field Fail Src Key
 has_map_field Fail Src Key => move Key x | i_has_map_field Fail Src x
 
@@ -1509,6 +1513,15 @@ i_has_map_field f r y
 i_has_map_field f x y
 i_has_map_field f y y
 
+## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
+
+get_map_element/4
+
+get_map_elements Fail Src=rxy Size=u==2 Rest=* => gen_get_map_element(Fail,Src,Size,Rest)
+get_map_elements Fail Src Size Rest=* => i_get_map_elements Fail Src Size Rest
+
+i_get_map_elements f s I
+
 get_map_element Fail Src=rxy Key=ax Dst => i_get_map_element Fail Src Key Dst
 get_map_element Fail Src=rxy Key=rycq Dst => \
     move Key x | i_get_map_element Fail Src x Dst
-- 
cgit v1.2.3


From 5006a54b032a82ad55f33e532e4068c34fe143a9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 6 Feb 2014 17:15:05 +0100
Subject: erts: Fix race bug in ets:all/0

causing recently created/deleted tables to not be included/excluded.
---
 erts/emulator/beam/erl_db.c | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 41e64fcd4f..a5d67571e2 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -125,6 +125,7 @@ get_meta_main_tab_lock(unsigned slot)
 static erts_smp_spinlock_t meta_main_tab_main_lock;
 static Uint meta_main_tab_first_free;   /* Index of first free slot */
 static int meta_main_tab_cnt;		/* Number of active tables */
+static int meta_main_tab_top;           /* Highest ever used slot + 1 */
 static Uint meta_main_tab_slot_mask;    /* The slot index part of an unnamed table id */
 static Uint meta_main_tab_seq_incr;
 static Uint meta_main_tab_seq_cnt = 0;  /* To give unique(-ish) table identifiers */
@@ -1469,6 +1470,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     ASSERT(slot>=0 && slot<db_max_tabs);
     meta_main_tab_first_free = GET_NEXT_FREE_SLOT(slot);
     meta_main_tab_cnt++;
+    if (slot >= meta_main_tab_top) {
+	ASSERT(slot == meta_main_tab_top);
+	meta_main_tab_top = slot + 1;
+    }
 
     if (is_named) {
 	ret = BIF_ARG_1;
@@ -2058,27 +2063,31 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0)
 {
     DbTable* tb;
     Eterm previous;
-    int i, j;
+    int i;
     Eterm* hp;
     Eterm* hendp;
     int t_tabs_cnt;
-    int t_max_tabs;
+    int t_top;
 
     erts_smp_spin_lock(&meta_main_tab_main_lock);
     t_tabs_cnt = meta_main_tab_cnt;
-    t_max_tabs = db_max_tabs;
+    t_top = meta_main_tab_top;
     erts_smp_spin_unlock(&meta_main_tab_main_lock);
 
     hp = HAlloc(BIF_P, 2*t_tabs_cnt);
     hendp = hp + 2*t_tabs_cnt;
 
     previous = NIL;
-    j = 0;
-    for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) {
+    for(i = 0; i < t_top; i++) {
 	erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i);
 	erts_smp_rwmtx_rlock(mmtl);
 	if (IS_SLOT_ALIVE(i)) {
-	    j++;
+	    if (hp == hendp) {
+		/* Racing table creator, grab some more heap space */
+		t_tabs_cnt = 10;
+		hp = HAlloc(BIF_P, 2*t_tabs_cnt);
+		hendp = hp + 2*t_tabs_cnt;
+	    }
 	    tb = meta_main_tab[i].u.tb;
 	    previous = CONS(hp, tb->common.id, previous);
 	    hp += 2;
@@ -2849,6 +2858,7 @@ void init_db(void)
     ERTS_ETS_MISC_MEM_ADD(size);
 
     meta_main_tab_cnt = 0;
+    meta_main_tab_top = 0;
     for (i=1; i<db_max_tabs; i++) {
 	SET_NEXT_FREE_SLOT(i-1,i);
     }
-- 
cgit v1.2.3


From 32d4c07508eeffac64fd901431366d2c34416684 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 10 Feb 2014 17:00:34 +0100
Subject: erts: Fix harmless (?) typo in beam_load.c

---
 erts/emulator/beam/beam_load.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index b589d1c930..915af7fbb1 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -409,7 +409,7 @@ typedef struct LoaderState {
           __result = __result << 8 | *Stp->file_p++; \
        } \
        Dest = __result; \
-    } while (0)
+    }
 
 #define GetByte(Stp, Dest) \
     if ((Stp)->file_left < 1) { \
-- 
cgit v1.2.3


From 8899c393163f9c95a81971e013b57788b33db09c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 19 Feb 2014 18:30:35 +0100
Subject: erts: Fix erts_debug:disassemble/1

Now handles map instructions correctly.
---
 erts/emulator/beam/beam_debug.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index e36ec2a93e..a3cd08834f 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -635,6 +635,11 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_put_tuple_rI:
     case op_i_put_tuple_xI:
     case op_i_put_tuple_yI:
+    case op_new_map_jdII:
+    case op_update_map_assoc_jsdII:
+    case op_update_map_exact_jsdII:
+    case op_i_has_map_fields_fsI:
+    case op_i_get_map_elements_fsI:
 	{
 	    int n = unpacked[-1];
 
-- 
cgit v1.2.3


From 880239f529bbdefecc39cc179a24d9ea89c3736a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 20 Feb 2014 11:14:42 +0100
Subject: ASSERT that GC is not tried with "need" when GC is disabled

---
 erts/emulator/beam/erl_gc.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 2022f70cbb..aa15d2cc57 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -406,8 +406,10 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
-    if (p->flags & F_DISABLE_GC)
+    if (p->flags & F_DISABLE_GC) {
+	ASSERT(need == 0);
 	return 1;
+    }
 
     esdp = erts_get_scheduler_data();
 
-- 
cgit v1.2.3


From 0dd1ae604a093a475c3af716958092427820a64e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 21 Feb 2014 17:14:43 +0100
Subject: erts: Maps src instructions can't be literals

Move src to a register if it is a literal.
---
 erts/emulator/beam/ops.tab | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 8406f5344a..73630fda8e 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1470,11 +1470,15 @@ apply_last I P
 #
 
 put_map_assoc F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
-put_map_exact F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
-put_map_assoc F Src Dst Live Size Rest=* => \
+put_map_assoc F Src=s Dst Live Size Rest=* => \
 	       update_map_assoc F Src Dst Live Size Rest
-put_map_exact F Src Dst Live Size Rest=* => \
+put_map_assoc F Src Dst Live Size Rest=* => \
+	       move Src x | update_map_assoc F x Dst Live Size Rest
+put_map_exact F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
+put_map_exact F Src=s Dst Live Size Rest=* => \
 	      update_map_exact F Src Dst Live Size Rest
+put_map_exact F Src Dst Live Size Rest=* => \
+	      move Src x | update_map_exact F x Dst Live Size Rest
 
 new_map j d I I
 update_map_assoc j s d I I
-- 
cgit v1.2.3


From b2017b91a5572f7bc2caf15082b4b105f3a3e21d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Sun, 23 Feb 2014 20:04:30 +0100
Subject: Introduce configure option --with-assumed-cache-line-size=SIZE

---
 erts/emulator/beam/erl_alloc.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index f83f6b39cf..942eaa47d0 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -209,8 +209,8 @@ int erts_is_allctr_wrapper_prelocked(void);
 void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size);
 
 #ifndef ERTS_CACHE_LINE_SIZE
-/* Assume a cache line size of 64 bytes */
-#  define ERTS_CACHE_LINE_SIZE ((UWord) 64)
+/* Assumed cache line size */
+#  define ERTS_CACHE_LINE_SIZE ((UWord) ASSUMED_CACHE_LINE_SIZE)
 #  define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1)
 #endif
 
-- 
cgit v1.2.3


From a3af5f4a5c4568225ef91ee4493da6bf659f7161 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 2 Sep 2013 17:27:58 +0200
Subject: erts: Set default external enc to use new float scheme

This change was triggered by the OSE float printing function
not working exactly the same way as linux/win32. But it is
also a good one in general as it cuts size in more than half
for floats.
---
 erts/emulator/beam/dist.h     | 1 +
 erts/emulator/beam/external.c | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index ff8f5e106f..0519a9225e 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -44,6 +44,7 @@
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
 			       | DFLAG_NEW_FUN_TAGS		\
+			       | DFLAG_NEW_FLOATS		\
 			       | DFLAG_EXTENDED_PIDS_PORTS	\
 			       | DFLAG_EXPORT_PTR_TAG		\
 			       | DFLAG_BIT_BINARIES)
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9fb2dbd8bf..b8e6b3b072 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -529,7 +529,7 @@ Uint erts_encode_ext_size(Eterm term)
 
 Uint erts_encode_ext_size_2(Eterm term, unsigned dflags)
 {
-    return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|dflags)
+    return encode_size_struct2(NULL, term, dflags)
         + 1 /* VERSION_MAGIC */;
 }
 
@@ -1099,10 +1099,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 	    if (tp[1] == am_minor_version && is_small(tp[2])) {
 		switch (signed_val(tp[2])) {
 		case 0:
-		    flags = TERM_TO_BINARY_DFLAGS;
+		    flags = TERM_TO_BINARY_DFLAGS & ~DFLAG_NEW_FLOATS;
 		    break;
 		case 1:
-		    flags = TERM_TO_BINARY_DFLAGS|DFLAG_NEW_FLOATS;
+		    flags = TERM_TO_BINARY_DFLAGS;
 		    break;
 		default:
 		    goto error;
@@ -1605,9 +1605,9 @@ external_size_2(BIF_ALIST_2)
             if (tp[1] == am_minor_version && is_small(tp[2])) {
                 switch (signed_val(tp[2])) {
                 case 0:
+                    flags &= ~DFLAG_NEW_FLOATS;
                     break;
                 case 1:
-                    flags |= DFLAG_NEW_FLOATS;
                     break;
                 default:
                     goto error;
-- 
cgit v1.2.3


From 3e8b423a2cb11f819f3cede7ef817f4012f18944 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Sun, 26 Jan 2014 20:34:12 -0500
Subject: further enhancements for dirty schedulers

Add support for setting the number of dirty CPU schedulers online via
erlang:system_flag/2. Assuming the emulator is built with dirty schedulers
enabled, the number of dirty CPU schedulers online may not be less than 1,
nor greater than the number of dirty CPU schedulers available, nor greater
than the number of normal schedulers online. Dirty CPU scheduler threads
that are taken offline via system_flag/2 are suspended. The number of dirty
CPU schedulers online may be adjusted independently of the number of normal
schedulers online, but if system_flag/2 is used to set the number of normal
schedulers online to a value less than the current number of normal
schedulers online, the number of dirty CPU schedulers online is decreased
proportionally. Likewise, if the number of normal schedulers online is
increased, the number of dirty CPU schedulers online is increased
proportionally. For example, if 8 normal schedulers and 4 dirty CPU
schedulers are online, and system_flag/2 is called to set the number of
normal schedulers online to 4, the number of dirty CPU schedulers online is
also decreased by half, to 2. Subsequently setting the number of normal
schedulers online back to 8 also sets the number of dirty CPU schedulers
online back to 4. Augment the system_flag/2 documentation in the erlang man
page to explain this relationship between schedulers_online and
dirty_cpu_schedulers_online.

Also ensure that all dirty CPU and I/O schedulers are suspended when
multi-scheduling is blocked via system_flag/2, and brought back online when
multi-scheduling is unblocked.

Add Rickard Green's rewritten check_enqueue_in_prio_queue() function that
inspects process state more thoroughly to determine if to enqueue it and if
so on what queue, including dirty queues when appropriate.

Make sure dirty NIF jobs do not trigger erlang:system_monitor long_schedule
messages.

Add more dirty scheduler testing to the scheduler test suite.

Remove the erts_no_dirty_cpu_schedulers_online global variable, since it's
no longer needed.

Execute dirty NIFs on a normal scheduler thread while multi-scheduling
blocking is in effect. Evacuate any dirty jobs residing in the dirty run
queues over to a normal run queue when multi-scheduling is blocked.

Allow dirty schedulers to execute aux work.

Set the dirty run queues halt_in_progress flag when halting the normal
schedulers.

Change dirty scheduler numbers to a structure including both scheduler
number and type, either dirty CPU or dirty I/O. Add some assertions to
ensure that dirty CPU schedulers operate only on dirty CPU scheduler
process flags, and the same for dirty I/O schedulers.
---
 erts/emulator/beam/atom.names    |    1 +
 erts/emulator/beam/beam_emu.c    |    6 +-
 erts/emulator/beam/bif.c         |   33 +-
 erts/emulator/beam/erl_drv_nif.h |    7 +
 erts/emulator/beam/erl_init.c    |   19 +-
 erts/emulator/beam/erl_nif.c     |   24 +-
 erts/emulator/beam/erl_nif.h     |    4 +-
 erts/emulator/beam/erl_process.c | 1433 ++++++++++++++++++++++++++++++++------
 erts/emulator/beam/erl_process.h |   46 +-
 9 files changed, 1336 insertions(+), 237 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 96547ba743..d28e519ae1 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -179,6 +179,7 @@ atom dexit
 atom depth
 atom dgroup_leader
 atom dictionary
+atom dirty_cpu_schedulers_online
 atom disable_trace
 atom disabled
 atom display_items
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1ae413d46e..1dfff7cb6a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1203,7 +1203,11 @@ void process_main(void)
 
     if (start_time != 0) {
         Sint64 diff = erts_timestamp_millis() - start_time;
-	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule) {
+	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    && !ERTS_SCHEDULER_IS_DIRTY(c_p->scheduler_data)
+#endif
+	    ) {
 	    BeamInstr *inptr = find_function_from_pc(start_time_i);
 	    BeamInstr *outptr = find_function_from_pc(c_p->i);
 	    monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 9c4801041f..f34045a589 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4333,7 +4333,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	switch (erts_set_schedulers_online(BIF_P,
 					   ERTS_PROC_LOCK_MAIN,
 					   signed_val(BIF_ARG_2),
-					   &old_no)) {
+					   &old_no
+#ifdef ERTS_DIRTY_SCHEDULERS
+					   , 0
+#endif
+					   )) {
 	case ERTS_SCHDLR_SSPND_DONE:
 	    BIF_RET(make_small(old_no));
 	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
@@ -4465,6 +4469,33 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 		      ref,
 		      old ? am_true : am_false);
 	}
+#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
+    } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) {
+	Sint old_no;
+	if (!is_small(BIF_ARG_2))
+	    goto error;
+	switch (erts_set_schedulers_online(BIF_P,
+					   ERTS_PROC_LOCK_MAIN,
+					   signed_val(BIF_ARG_2),
+					   &old_no,
+					   1)) {
+	case ERTS_SCHDLR_SSPND_DONE:
+	    BIF_RET(make_small(old_no));
+	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
+	    ERTS_VBUMP_ALL_REDS(BIF_P);
+	    BIF_TRAP2(bif_export[BIF_system_flag_2],
+		      BIF_P, BIF_ARG_1, BIF_ARG_2);
+	case ERTS_SCHDLR_SSPND_YIELD_DONE:
+	    ERTS_BIF_YIELD_RETURN_X(BIF_P, make_small(old_no),
+				    am_dirty_cpu_schedulers_online);
+	case ERTS_SCHDLR_SSPND_EINVAL:
+	    goto error;
+	default:
+	    ASSERT(0);
+	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+	    break;
+	}
+#endif
     } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) {
 	int what;
 	if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2))
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index ea013a49a3..3f829ea7ea 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -41,6 +41,13 @@ typedef struct {
     int suggested_stack_size;
 } ErlDrvThreadOpts;
 
+#if defined(ERL_DRV_DIRTY_SCHEDULER_SUPPORT) || defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT)
+typedef enum {
+    ERL_DRV_DIRTY_JOB_CPU_BOUND = 1,
+    ERL_DRV_DIRTY_JOB_IO_BOUND  = 2
+} ErlDrvDirtyJobFlags;
+#endif
+
 #endif  /* __ERL_DRV_NIF_H__ */
 
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index e6c05adec1..5bdebc493c 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -178,6 +178,11 @@ int erts_compat_rel;
 
 static int no_schedulers;
 static int no_schedulers_online;
+#ifdef ERTS_DIRTY_SCHEDULERS
+static int no_dirty_cpu_schedulers;
+static int no_dirty_cpu_schedulers_online;
+static int no_dirty_io_schedulers;
+#endif
 
 #ifdef DEBUG
 Uint32 verbose;             /* See erl_debug.h for information about verbose */
@@ -304,7 +309,13 @@ erl_init(int ncpu,
     erts_init_sys_common_misc();
     erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab);
     erts_init_scheduling(no_schedulers,
-			 no_schedulers_online);
+			 no_schedulers_online
+#ifdef ERTS_DIRTY_SCHEDULERS
+			 , no_dirty_cpu_schedulers,
+			 no_dirty_cpu_schedulers_online,
+			 no_dirty_io_schedulers
+#endif
+			 );
     erts_init_cpu_topology(); /* Must be after init_scheduling */
     erts_init_gc(); /* Must be after init_scheduling */
     erts_alloc_late_init();
@@ -1055,9 +1066,9 @@ early_init(int *argc, char **argv) /*
     erts_no_schedulers = (Uint) no_schedulers;
 #endif
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_no_dirty_cpu_schedulers = dirty_cpu_scheds;
-    erts_no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online;
-    erts_no_dirty_io_schedulers = dirty_io_scheds;
+    erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds;
+    no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online;
+    erts_no_dirty_io_schedulers = no_dirty_io_schedulers = dirty_io_scheds;
 #endif
     erts_early_init_scheduling(no_schedulers);
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c35f1fc2c6..1ddfb2f7bd 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1510,6 +1510,13 @@ enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
     a = erts_smp_atomic32_read_acqb(&proc->state);
     while (1) {
 	n = state = a;
+	/*
+	 * clear any current dirty flags and dirty queue indicators,
+	 * in case the application is shifting a job from one type
+	 * of dirty scheduler to the other
+	 */
+	n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC
+	       |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q);
 	if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND)
 	    n |= ERTS_PSFLG_DIRTY_CPU_PROC;
 	else
@@ -1540,22 +1547,15 @@ enif_schedule_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result,
 				  ERL_NIF_TERM (*fp)(ErlNifEnv*, ERL_NIF_TERM))
 {
 #ifdef USE_THREADS
-    erts_aint32_t state, n, a;
     Process* proc = env->proc;
     Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
     Export* ep;
 
-    a = erts_smp_atomic32_read_acqb(&proc->state);
-    while (1) {
-	n = state = a;
-	if (!(n & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)))
-	    break;
-	n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC
-	       |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q);
-	a = erts_smp_atomic32_cmpxchg_mb(&proc->state, n, state);
-	if (a == state)
-	    break;
-    }
+    erts_smp_atomic32_read_band_mb(&proc->state,
+				   ~(ERTS_PSFLG_DIRTY_CPU_PROC
+				     |ERTS_PSFLG_DIRTY_IO_PROC
+				     |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q
+				     |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
     if (!(ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
 	alloc_proc_psd(proc, &ep);
     ERTS_VBUMP_ALL_REDS(proc);
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 7613446f64..c12ba4d554 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -172,8 +172,8 @@ typedef ErlDrvThreadOpts ErlNifThreadOpts;
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 typedef enum
 {
-    ERL_NIF_DIRTY_JOB_CPU_BOUND = 1,
-    ERL_NIF_DIRTY_JOB_IO_BOUND  = 2
+    ERL_NIF_DIRTY_JOB_CPU_BOUND = ERL_DRV_DIRTY_JOB_CPU_BOUND,
+    ERL_NIF_DIRTY_JOB_IO_BOUND  = ERL_DRV_DIRTY_JOB_IO_BOUND
 }ErlNifDirtyTaskFlags;
 #endif
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 937881212a..1d0275c524 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -148,7 +148,6 @@ int erts_sched_balance_util = 0;
 Uint erts_no_schedulers;
 #ifdef ERTS_DIRTY_SCHEDULERS
 Uint erts_no_dirty_cpu_schedulers;
-Uint erts_no_dirty_cpu_schedulers_online;
 Uint erts_no_dirty_io_schedulers;
 #endif
 
@@ -188,6 +187,13 @@ static ErtsAuxWorkData *aux_thread_aux_work_data;
 #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
   erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL))
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \
+  erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_cpu_changing, (VAL))
+#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \
+  erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_io_changing, (VAL))
+#endif
+
 #else
 
 #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL)			\
@@ -198,6 +204,23 @@ do {									\
     ASSERT(old_val__ == (OLD_VAL));					\
 } while (0)
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL)		\
+do {									\
+    erts_aint32_t old_val__;						\
+    old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_cpu_changing, \
+					   (VAL));			\
+    ASSERT(old_val__ == (OLD_VAL));					\
+} while (0)
+#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL)		\
+do {									\
+    erts_aint32_t old_val__;						\
+    old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_io_changing, \
+					   (VAL));			\
+    ASSERT(old_val__ == (OLD_VAL));					\
+} while (0)
+#endif
+
 #endif
 
 
@@ -207,11 +230,29 @@ static struct {
     int online;
     int curr_online;
     int wait_curr_online;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    int dirty_cpu_online;
+    int dirty_cpu_curr_online;
+    int dirty_cpu_wait_curr_online;
+    int dirty_io_online;
+    int dirty_io_curr_online;
+    int dirty_io_wait_curr_online;
+#endif
     erts_smp_atomic32_t changing;
     erts_smp_atomic32_t active;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_atomic32_t dirty_cpu_changing;
+    erts_smp_atomic32_t dirty_cpu_active;
+    erts_smp_atomic32_t dirty_io_changing;
+    erts_smp_atomic32_t dirty_io_active;
+#endif
     struct {
 	int ongoing;
 	long wait_active;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	long dirty_cpu_wait_active;
+	long dirty_io_wait_active;
+#endif
 	ErtsProcList *procs;
     } msb; /* Multi Scheduling Block */
 } schdlr_sspnd;
@@ -1306,6 +1347,9 @@ static ERTS_INLINE void
 haw_thr_prgr_current_check_progress(ErtsAuxWorkData *awdp)
 {
     ErtsThrPrgrVal current = awdp->current_thr_prgr;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
     if (current != ERTS_THR_PRGR_INVALID
 	&& !erts_thr_progress_equal(current, erts_thr_progress_current())) {
 	/*
@@ -1322,6 +1366,10 @@ handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, in
 {
     int jix, max_jix;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+
     ASSERT(awdp->delayed_wakeup.next != ERTS_DELAYED_WAKEUP_INFINITY);
 
     if (!waiting && awdp->delayed_wakeup.next > awdp->esdp->reductions)
@@ -1477,6 +1525,9 @@ handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 			      erts_aint32_t aux_work,
 			      int waiting)
 {
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
     if (!erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp),
 					    awdp->misc.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
@@ -1561,6 +1612,9 @@ handle_async_ready(ErtsAuxWorkData *awdp,
 		   int waiting)
 {
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY);
     if (erts_check_async_ready(awdp->async_ready.queue)) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY)
@@ -1585,6 +1639,9 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 {
     void *thr_prgr_p;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
 #ifdef ERTS_SMP
     if (awdp->async_ready.need_thr_prgr
 	&& !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp),
@@ -1622,6 +1679,9 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
     erts_aint32_t res;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
     unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
 			       | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC));
     aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
@@ -1655,11 +1715,7 @@ erts_alloc_ensure_handle_delayed_dealloc_call(int ix)
 {
 #ifdef DEBUG
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
-	return;
-#endif
-    ASSERT(!esdp || ix == (int) esdp->no);
+    ASSERT(!esdp || (ERTS_SCHEDULER_IS_DIRTY(esdp) || ix == (int) esdp->no));
 #endif
     set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1),
 				  ERTS_SSI_AUX_WORK_DD);
@@ -1673,6 +1729,9 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
     ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     int more_work = 0;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
@@ -1712,6 +1771,9 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
     if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 
@@ -1759,6 +1821,9 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait
     int lops;
     ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
     for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) {
 	ErtsThrPrgrLaterOp *lop = awdp->later_op.first;
 	if (!erts_thr_progress_has_reached_this(current, lop->later))
@@ -1917,6 +1982,14 @@ erts_smp_notify_check_children_needed(void)
     for (i = 0; i < erts_no_schedulers; i++)
 	set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i),
 				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    for (i = 0; i < erts_no_dirty_cpu_schedulers; i++)
+	set_aux_work_flags_wakeup_nob(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(i),
+				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+    for (i = 0; i < erts_no_dirty_io_schedulers; i++)
+	set_aux_work_flags_wakeup_nob(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(i),
+				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+#endif
 }
 
 static ERTS_INLINE erts_aint32_t
@@ -2697,15 +2770,15 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	while (1) {
 
-	    aux_work = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
-		erts_atomic32_read_acqb(&ssi->aux_work);
+	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
-		if (!thr_prgr_active) {
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 		    sched_wall_time_change(esdp, 1);
 		}
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
-		if (aux_work && erts_thr_progress_update(esdp))
+		if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)
+		    && erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
 	    }
 
@@ -2990,7 +3063,7 @@ wake_scheduler(ErtsRunQueue *rq)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 static void
-wake_dirty_scheduler(ErtsRunQueue *rq)
+wake_dirty_schedulers(ErtsRunQueue *rq, int one)
 {
     ErtsSchedulerSleepInfo *ssi;
     ErtsSchedulerSleepList *sl;
@@ -3000,9 +3073,27 @@ wake_dirty_scheduler(ErtsRunQueue *rq)
     sl = &rq->sleepers;
     erts_smp_spin_lock(&sl->lock);
     ssi = sl->list;
-    if (!ssi)
+    if (!ssi) {
 	erts_smp_spin_unlock(&sl->lock);
-    else {
+	if (one)
+	    wake_scheduler(rq);
+    } else if (one) {
+	erts_aint32_t flgs;
+	if (ssi->prev)
+	    ssi->prev->next = ssi->next;
+	else {
+	    ASSERT(sl->list == ssi);
+	    sl->list = ssi->next;
+	}
+	if (ssi->next)
+	    ssi->next->prev = ssi->prev;
+
+	erts_smp_spin_unlock(&sl->lock);
+
+	ERTS_THR_MEMORY_BARRIER;
+	flgs = ssi_flags_set_wake(ssi);
+	erts_sched_finish_poke(ssi, flgs);
+    } else {
 	sl->list = NULL;
 	erts_smp_spin_unlock(&sl->lock);
 
@@ -3154,7 +3245,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq)
     if (runq) {
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
-	    wake_dirty_scheduler(runq);
+	    wake_dirty_schedulers(runq, 1);
 	else
 #endif
 	    wake_scheduler(runq);
@@ -3452,6 +3543,7 @@ suspend_run_queue(ErtsRunQueue *rq)
 }
 
 static void scheduler_ix_resume_wake(Uint ix);
+static void scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi);
 
 static ERTS_INLINE void
 resume_run_queue(ErtsRunQueue *rq)
@@ -3478,7 +3570,10 @@ resume_run_queue(ErtsRunQueue *rq)
 
     erts_smp_runq_unlock(rq);
 
-    scheduler_ix_resume_wake(rq->ix);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+#endif
+	scheduler_ix_resume_wake(rq->ix);
 }
 
 typedef struct {
@@ -3509,20 +3604,28 @@ evacuate_run_queue(ErtsRunQueue *rq,
     int prio_q;
     ErtsRunQueue *to_rq;
     ErtsMigrationPaths *mps;
-    ErtsMigrationPath *mp;
+    ErtsMigrationPath *mp = NULL;
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
     (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
 
-    mps = erts_get_migration_paths_managed();
-    mp = &mps->mpath[rq->ix];
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+#endif
+    {
+	mps = erts_get_migration_paths_managed();
+	mp = &mps->mpath[rq->ix];
+    }
 
     /* Evacuate scheduled misc ops */
 
     if (rq->misc.start) {
 	ErtsMiscOpList *start, *end;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+#endif
 	to_rq = mp->misc_evac_runq;
 	if (!to_rq)
 	    return;
@@ -3551,6 +3654,9 @@ evacuate_run_queue(ErtsRunQueue *rq,
     if (rq->ports.start) {
 	Port *prt;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+#endif
 	to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq;
 	if (!to_rq)
 	    return;
@@ -3586,15 +3692,26 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	erts_aint32_t state;
 	Process *proc;
 	int notify = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	int requeue;
+#endif
 	to_rq = NULL;
 
-	if (!mp->prio[prio_q].runq)
-	    return;
-	if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq)
-	    return;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+#endif
+	{
+	    if (!mp->prio[prio_q].runq)
+		return;
+	    if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq)
+		return;
+	}
 
 	proc = dequeue_process(rq, prio_q, &state);
 	while (proc) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    requeue = 1;
+#endif
 	    if (ERTS_PSFLG_BOUND & state) {
 		/* Bound processes get stuck here... */
 		proc->next = NULL;
@@ -3603,12 +3720,43 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		else
 		    sbpp->first = proc;
 		sbpp->last = proc;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		requeue = 0;
+#endif
+	    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    else if (state & ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q) {
+		erts_aint32_t old;
+		old = erts_smp_atomic32_read_band_nob(&proc->state,
+						      ~(ERTS_PSFLG_DIRTY_CPU_PROC
+							| ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q));
+		/* assert that no other dirty flags are set */
+		ASSERT(!(old & (ERTS_PSFLG_DIRTY_IO_PROC|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)));
+	    } else if (state & ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) {
+		erts_aint32_t old;
+		old = erts_smp_atomic32_read_band_nob(&proc->state,
+						      ~(ERTS_PSFLG_DIRTY_IO_PROC
+							| ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
+		/* assert that no other dirty flags are set */
+		ASSERT(!(old & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q)));
 	    }
+	    if (requeue) {
+#else
 	    else {
+#endif
 		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		erts_smp_runq_unlock(rq);
 
-		to_rq = mp->prio[prio].runq;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+		    /*
+		     * dirty run queues evacuate only to run
+		     * queue 0 during multi-scheduling blocking
+		     */
+		    to_rq = ERTS_RUNQ_IX(0);
+		else
+#endif
+		    to_rq = mp->prio[prio].runq;
 		RUNQ_SET_RQ(&proc->run_queue, to_rq);
 
 		erts_smp_runq_lock(to_rq);
@@ -4722,13 +4870,20 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 	    rq->wakeup_other += (left_len*wo_reds
 				 + ERTS_WAKEUP_OTHER_FIXED_INC);
 	    if (rq->wakeup_other > wakeup_other.limit) {
-		int empty_rqs =
-		    erts_smp_atomic32_read_acqb(&no_empty_run_queues);
-		if (flags & ERTS_RUNQ_FLG_PROTECTED)
-		    (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
-		if (empty_rqs != 0)
-		    wake_scheduler_on_empty_runq(rq);
-		rq->wakeup_other = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->waiting)
+		    wake_dirty_schedulers(rq, 1);
+		else
+#endif
+		{
+		    int empty_rqs =
+			erts_smp_atomic32_read_acqb(&no_empty_run_queues);
+		    if (flags & ERTS_RUNQ_FLG_PROTECTED)
+			(void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
+		    if (empty_rqs != 0)
+			wake_scheduler_on_empty_runq(rq);
+		    rq->wakeup_other = 0;
+		}
 	    }
 	}
 	rq->wakeup_other_reds = 0;
@@ -5009,8 +5164,12 @@ erts_sched_set_wake_cleanup_threshold(char *str)
 static void
 init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 {
-    if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp))
+    if (!esdp)
 	awdp->sched_id = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    else if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+	awdp->sched_id = (int) ERTS_DIRTY_SCHEDULER_NO(esdp);
+#endif
     else
 	awdp->sched_id = (int) esdp->no;
     awdp->esdp = esdp;
@@ -5076,11 +5235,11 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
 	esdp->no = 0;
-	esdp->dirty_no = (Uint) num;
+	ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num;
     }
     else {
 	esdp->no = (Uint) num;
-	esdp->dirty_no = 0;
+	ERTS_DIRTY_SCHEDULER_NO(esdp) = 0;
     }
 #else
     esdp->no = (Uint) num;
@@ -5111,7 +5270,12 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 }
 
 void
-erts_init_scheduling(int no_schedulers, int no_schedulers_online)
+erts_init_scheduling(int no_schedulers, int no_schedulers_online
+#ifdef ERTS_DIRTY_SCHEDULERS
+		     , int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online,
+		     int no_dirty_io_schedulers
+#endif
+		     )
 {
     int ix, n, no_ssi;
     char *daww_ptr;
@@ -5133,6 +5297,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     ASSERT(no_schedulers_online <= no_schedulers);
     ASSERT(no_schedulers_online >= 1);
     ASSERT(no_schedulers >= 1);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(no_dirty_cpu_schedulers <= no_schedulers);
+    ASSERT(no_dirty_cpu_schedulers >= 1);
+    ASSERT(no_dirty_cpu_schedulers_online <= no_schedulers_online);
+    ASSERT(no_dirty_cpu_schedulers_online >= 1);
+#endif
 
     /* Create and initialize run queues */
 
@@ -5169,10 +5339,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 #ifdef ERTS_SMP
-	if (ERTS_RUNQ_IX_IS_DIRTY(ix)) {
+	if (ERTS_RUNQ_IX_IS_DIRTY(ix))
 	    erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list");
-	    rq->sleepers.list = NULL;
-	}
+	rq->sleepers.list = NULL;
 #endif
 #endif
 
@@ -5236,6 +5405,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 
     n = (int) no_schedulers;
     erts_no_schedulers = n;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers;
+    erts_no_dirty_io_schedulers = no_dirty_io_schedulers;
+#endif
 
     /* Create and initialize scheduler sleep info */
 #ifdef ERTS_SMP
@@ -5267,21 +5440,21 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     aligned_dirty_cpu_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
 	    ERTS_ALC_T_SCHDLR_SLP_INFO,
-	    erts_no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
-    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+	    no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) {
 	ErtsSchedulerSleepInfo *ssi = &aligned_dirty_cpu_sched_sleep_info[ix].ssi;
 	erts_smp_atomic32_init_nob(&ssi->flags, 0);
-	ssi->event = NULL; /* initialized in sched_thread_func */
+	ssi->event = NULL; /* initialized in sched_dirty_cpu_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
     aligned_dirty_io_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
 	    ERTS_ALC_T_SCHDLR_SLP_INFO,
-	    erts_no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
-    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+	    no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
 	ErtsSchedulerSleepInfo *ssi = &aligned_dirty_io_sched_sleep_info[ix].ssi;
 	erts_smp_atomic32_init_nob(&ssi->flags, 0);
-	ssi->event = NULL; /* initialized in sched_thread_func */
+	ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
 #endif
@@ -5314,8 +5487,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     erts_aligned_dirty_cpu_scheduler_data =
 	erts_alloc_permanent_cache_aligned(
 					   ERTS_ALC_T_SCHDLR_DATA,
-					   erts_no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerData));
-    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+					   no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerData));
+    for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) {
 	ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
 	init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix),
 			    ERTS_DIRTY_CPU_RUNQ, NULL, 0);
@@ -5323,8 +5496,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     erts_aligned_dirty_io_scheduler_data =
 	erts_alloc_permanent_cache_aligned(
 					   ERTS_ALC_T_SCHDLR_DATA,
-					   erts_no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerData));
-    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+					   no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerData));
+    for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
 	ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
 	init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
 			    ERTS_DIRTY_IO_RUNQ, NULL, 0);
@@ -5354,6 +5527,16 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     schdlr_sspnd.curr_online = no_schedulers;
     schdlr_sspnd.msb.ongoing = 0;
     erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_changing, 0);
+    schdlr_sspnd.dirty_cpu_online = no_dirty_cpu_schedulers_online;
+    schdlr_sspnd.dirty_cpu_curr_online = no_dirty_cpu_schedulers;
+    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_active, no_dirty_cpu_schedulers);
+    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_changing, 0);
+    schdlr_sspnd.dirty_io_online = no_dirty_io_schedulers;
+    schdlr_sspnd.dirty_io_curr_online = no_dirty_io_schedulers;
+    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_active, no_dirty_io_schedulers);
+#endif
     schdlr_sspnd.msb.procs = NULL;
     init_no_runqs(no_schedulers_online, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
@@ -5379,6 +5562,21 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
     schdlr_sspnd.curr_online *= 2; /* Boot strapping... */
     ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
 				| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    schdlr_sspnd.dirty_cpu_wait_curr_online = no_dirty_cpu_schedulers_online;
+    schdlr_sspnd.dirty_cpu_curr_online *= 2;
+    ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+					  | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+    for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) {
+	ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+	erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+    }
+
+    schdlr_sspnd.dirty_io_wait_curr_online = no_dirty_io_schedulers;
+    schdlr_sspnd.dirty_io_curr_online *= 2;
+    ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+					 | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+#endif
 
     erts_smp_atomic32_init_nob(&doing_sys_schedule, 0);
 
@@ -5497,9 +5695,16 @@ free_proxy_proc(Process *proxy)
     erts_free(ERTS_ALC_T_PROC, proxy);
 }
 
+#define ERTS_ENQUEUE_NOT 0
+#define ERTS_ENQUEUE_NORMAL_QUEUE 1
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2
+#define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3
+#endif
 
 static ERTS_INLINE int
-check_enqueue_in_prio_queue(erts_aint32_t *prq_prio_p,
+check_enqueue_in_prio_queue(Process *c_p,
+			    erts_aint32_t *prq_prio_p,
 			    erts_aint32_t *newp,
 			    erts_aint32_t actual)
 {
@@ -5511,56 +5716,105 @@ check_enqueue_in_prio_queue(erts_aint32_t *prq_prio_p,
     *prq_prio_p = aprio;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-    if (!(actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))) {
+    if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) {
+        /*
+         * If we have system tasks of a priority higher
+         * or equal to the user priority, we enqueue
+         * on ordinary run-queue and take care of
+         * those system tasks first.
+         */
+        if (actual & ERTS_PSFLG_ACTIVE_SYS) {
+            erts_aint32_t uprio, stprio, qmask;
+            uprio = (actual >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK;
+            if (aprio < uprio)
+                goto enqueue_normal_runq; /* system tasks with higher prio */
+            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+            qmask = c_p->sys_task_qs->qmask;
+            erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+            switch (qmask & -qmask) {
+            case MAX_BIT:
+                stprio = PRIORITY_MAX;
+                break;
+            case HIGH_BIT:
+                stprio = PRIORITY_HIGH;
+                break;
+            case NORMAL_BIT:
+                stprio = PRIORITY_NORMAL;
+                break;
+            case LOW_BIT:
+                stprio = PRIORITY_LOW;
+                break;
+            default:
+                stprio = PRIORITY_LOW+1;
+                break;
+            }
+            if (stprio <= uprio)
+                goto enqueue_normal_runq; /* system tasks with higher prio */
+        }
+
+        /* Enqueue in dirty run queue if not already enqueued */
+	if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))
+	    return ERTS_ENQUEUE_NOT; /* already in queue */
+        if (actual & ERTS_PSFLG_DIRTY_CPU_PROC) {
+	    *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q;
+            if (actual & ERTS_PSFLG_IN_RUNQ)
+                return -ERTS_ENQUEUE_DIRTY_CPU_QUEUE; /* use proxy */
+	    *newp |= ERTS_PSFLG_IN_RUNQ;
+            return ERTS_ENQUEUE_DIRTY_CPU_QUEUE;
+        }
+	*newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q;
+        if (actual & ERTS_PSFLG_IN_RUNQ)
+            return -ERTS_ENQUEUE_DIRTY_IO_QUEUE; /* use proxy */
+	*newp |= ERTS_PSFLG_IN_RUNQ;
+        return ERTS_ENQUEUE_DIRTY_IO_QUEUE;
+    }
+
+ enqueue_normal_runq:
 #endif
-	max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK;
-	max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
-	max_qbit &= -max_qbit;
-	/*
-	 * max_qbit now either contain bit set for highest prio queue or a bit
-	 * out of range (which will have a value larger than valid range).
-	 */
+    max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK;
+    max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
+    max_qbit &= -max_qbit;
+    /*
+     * max_qbit now either contain bit set for highest prio queue or a bit
+     * out of range (which will have a value larger than valid range).
+     */
 
-	if (qbit >= max_qbit)
-	    return 0; /* Already queued in higher or equal prio */
+    if (qbit >= max_qbit)
+	return ERTS_ENQUEUE_NOT; /* Already queued in higher or equal prio */
 
-	/* Need to enqueue (if already enqueued, it is in lower prio) */
-	*newp |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET;
+    /* Need to enqueue (if already enqueued, it is in lower prio) */
+    *newp |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET;
 
-	if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK))
-	    != (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) {
-	    /*
-	     * Process struct already enqueued, or actual prio not
-	     * equal to user prio, i.e., enqueue using proxy.
-	     */
-	    return -1;
-	}
-#ifdef ERTS_DIRTY_SCHEDULERS
-    } else {
-	if (actual & ERTS_PSFLG_DIRTY_CPU_PROC)
-	    *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q;
-	else
-	    *newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q;
+    if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK))
+	!= (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) {
+	/*
+	 * Process struct already enqueued, or actual prio not
+	 * equal to user prio, i.e., enqueue using proxy.
+	 */
+	return -ERTS_ENQUEUE_NORMAL_QUEUE;
     }
-#endif
 
     /*
      * Enqueue using process struct.
      */
     *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK;
     *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET);
-    return 1;
+    return ERTS_ENQUEUE_NORMAL_QUEUE;
 }
 
 /*
- * scheduler_out_process() return with c_rq locked.
+ * schedule_out_process() return with c_rq locked.
  */
 static ERTS_INLINE int
 schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Process *proxy)
 {
     erts_aint32_t a, e, n, enq_prio = -1;
-    int res = 0;
     int enqueue; /* < 0 -> use proxy */
+    Process* sched_p;
+    ErtsRunQueue* runq;
+#ifdef ERTS_SMP
+    int check_emigration_need;
+#endif
 
     a = state;
 
@@ -5569,20 +5823,20 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 
 	ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
 
-	enqueue = 0;
+	enqueue = ERTS_ENQUEUE_NOT;
 
 	n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS);
 	if (a & ERTS_PSFLG_ACTIVE_SYS
 	    || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
-	    enqueue = check_enqueue_in_prio_queue(&enq_prio, &n, a);
+	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
 	}
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
     }
 
-    if (!enqueue) {
-
+    switch (enqueue) {
+    case ERTS_ENQUEUE_NOT:
 	if (erts_system_profile_flags.runnable_procs) {
 
 	    if (!(a & ERTS_PSFLG_ACTIVE_SYS)
@@ -5595,60 +5849,76 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 
 	if (proxy)
 	    free_proxy_proc(proxy);
-    }
-    else {
-	Process *sched_p;
-	ErtsRunQueue *runq;
 
-	ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS));
+	erts_smp_runq_lock(c_rq);
+	return 0;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 #ifdef ERTS_SMP
-	if (ERTS_PSFLG_DIRTY_CPU_PROC & a)
-	    runq = ERTS_DIRTY_CPU_RUNQ;
-	else if (ERTS_PSFLG_DIRTY_IO_PROC & a)
-	    runq = ERTS_DIRTY_IO_RUNQ;
-	else
+    case ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
+    case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
+	runq = ERTS_DIRTY_CPU_RUNQ;
+	ASSERT(ERTS_SCHEDULER_IS_DIRTY_CPU(runq->scheduler));
+#ifdef ERTS_SMP
+	check_emigration_need = 0;
+#endif
+	break;
+
+    case ERTS_ENQUEUE_DIRTY_IO_QUEUE:
+    case -ERTS_ENQUEUE_DIRTY_IO_QUEUE:
+	runq = ERTS_DIRTY_IO_RUNQ;
+	ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(runq->scheduler));
+#ifdef ERTS_SMP
+	check_emigration_need = 0;
+#endif
+	break;
 #endif
 #endif
-	    runq = erts_get_runq_proc(p);
 
-	if (enqueue < 0)
-	    sched_p = make_proxy_proc(proxy, p, enq_prio);
-	else {
-	    sched_p = p;
-	    if (proxy)
-		free_proxy_proc(proxy);
-	}
+    default:
+	ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE
+	       || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE);
 
+	runq = erts_get_runq_proc(p);
 #ifdef ERTS_SMP
-	if (!(ERTS_PSFLG_BOUND & n)
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    && !(n & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))
+	check_emigration_need = !(ERTS_PSFLG_BOUND & n);
 #endif
-	    ) {
-	    ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
-	    if (new_runq) {
-		RUNQ_SET_RQ(&sched_p->run_queue, new_runq);
-		runq = new_runq;
-	    }
+	break;
+    }
+
+    ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS));
+
+    if (enqueue < 0)
+	sched_p = make_proxy_proc(proxy, p, enq_prio);
+    else {
+	sched_p = p;
+	if (proxy)
+	    free_proxy_proc(proxy);
+    }
+
+#ifdef ERTS_SMP
+    if (check_emigration_need) {
+	ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
+	if (new_runq) {
+	    RUNQ_SET_RQ(&sched_p->run_queue, new_runq);
+	    runq = new_runq;
 	}
+    }
 #endif
-	ASSERT(runq);
-	res = 1;
 
-	erts_smp_runq_lock(runq);
+    ASSERT(runq);
 
-	/* Enqueue the process */
-	enqueue_process(runq, (int) enq_prio, sched_p);
+    erts_smp_runq_lock(runq);
 
-	if (runq == c_rq)
-	    return res;
-	erts_smp_runq_unlock(runq);
-	smp_notify_inc_runq(runq);
-    }
+    /* Enqueue the process */
+    enqueue_process(runq, (int) enq_prio, sched_p);
+
+    if (runq == c_rq)
+	return 1;
+    erts_smp_runq_unlock(runq);
+    smp_notify_inc_runq(runq);
     erts_smp_runq_lock(c_rq);
-    return res;
+    return 1;
 }
 
 static ERTS_INLINE void
@@ -5704,7 +5974,7 @@ change_proc_schedule_state(Process *p,
 	erts_aint32_t e;
 	n = e = a;
 
-	enqueue = 0;
+	enqueue = ERTS_ENQUEUE_NOT;
 
 	if (a & ERTS_PSFLG_FREE)
 	    break; /* We don't want to schedule free processes... */
@@ -5725,13 +5995,13 @@ change_proc_schedule_state(Process *p,
 	     * process may be in a run queue via proxy, need
 	     * further inspection...
 	     */
-	    enqueue = check_enqueue_in_prio_queue(enq_prio_p, &n, a);
+	    enqueue = check_enqueue_in_prio_queue(p, enq_prio_p, &n, a);
 	}
 
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
-	if (enqueue == 0 && n == a)
+	if (enqueue == ERTS_ENQUEUE_NOT && n == a)
 	    break;
     }
 
@@ -5765,7 +6035,7 @@ schedule_process(Process *p, erts_aint32_t in_state)
 					     ERTS_PSFLG_ACTIVE,
 					     &state,
 					     &enq_prio);
-    if (enqueue)
+    if (enqueue != ERTS_ENQUEUE_NOT)
 	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
 		 state,
 		 enq_prio);
@@ -5792,14 +6062,14 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 	if (a & ERTS_PSFLG_FREE)
 	    return; /* We don't want to schedule free processes... */
 
-	enqueue = 0;
+	enqueue = ERTS_ENQUEUE_NOT;
 	n |= ERTS_PSFLG_ACTIVE_SYS;
 	if (!(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)))
-	    enqueue = check_enqueue_in_prio_queue(&enq_prio, &n, a);
+	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
-	if (a == n && !enqueue)
+	if (a == n && enqueue == ERTS_ENQUEUE_NOT)
 	    goto cleanup;
     }
 
@@ -5815,7 +6085,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 
     }
 
-    if (enqueue) {
+    if (enqueue != ERTS_ENQUEUE_NOT) {
 	Process *sched_p;
 	if (enqueue > 0)
 	    sched_p = p;
@@ -5938,6 +6208,12 @@ static void
 scheduler_ix_resume_wake(Uint ix)
 {
     ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
+    scheduler_ssi_resume_wake(ssi);
+}
+
+static void
+scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi)
+{
     erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
 			   | ERTS_SSI_FLG_TSE_SLEEPING
 			   | ERTS_SSI_FLG_WAITING
@@ -6028,12 +6304,20 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
     }
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
 static void
 suspend_scheduler(ErtsSchedulerData *esdp)
 {
     erts_aint32_t flgs;
     erts_aint32_t changing;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    long no = (long) (ERTS_SCHEDULER_IS_DIRTY(esdp)
+		      ? ERTS_DIRTY_SCHEDULER_NO(esdp)
+		      : esdp->no);
+#else
     long no = (long) esdp->no;
+#endif
     ErtsSchedulerSleepInfo *ssi = esdp->ssi;
     long active_schedulers;
     int curr_online = 1;
@@ -6041,47 +6325,106 @@ suspend_scheduler(ErtsSchedulerData *esdp)
     erts_aint32_t aux_work;
     int thr_prgr_active = 1;
     ErtsStuckBoundProcesses sbp = {NULL, NULL};
+    int* ss_onlinep;
+    int* ss_curr_onlinep;
+    int* ss_wait_curr_onlinep;
+    long* ss_wait_activep;
+    long ss_wait_active_target;
+    erts_smp_atomic32_t* ss_changingp;
+    erts_smp_atomic32_t* ss_activep;
 
     /*
      * Schedulers may be suspended in two different ways:
      * - A scheduler may be suspended since it is not online.
      *   All schedulers with scheduler ids greater than
-     *   schdlr_sspnd.online are suspended.
+     *   schdlr_sspnd.online are suspended; same for dirty
+     *   schedulers and schdlr_sspnd.dirty_cpu_online and
+     *   schdlr_sspnd.dirty_io_online.
      * - Multi scheduling is blocked. All schedulers except the
-     *   scheduler with scheduler id 1 are suspended.
+     *   scheduler with scheduler id 1 are suspended, and all
+     *   dirty CPU and dirty I/O schedulers are suspended.
      *
      * Regardless of why a scheduler is suspended, it ends up here.
      */
 
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) || no != 1);
+
 #ifdef ERTS_DIRTY_SCHEDULERS
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) {
+	    erts_smp_runq_unlock(esdp->run_queue);
+	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	    erts_smp_runq_lock(esdp->run_queue);
+	}
+	if (ongoing_multi_scheduling_block())
+	    evacuate_run_queue(esdp->run_queue, &sbp);
+    } else
 #endif
-    ASSERT(no != 1);
-
-    evacuate_run_queue(esdp->run_queue, &sbp);
+	evacuate_run_queue(esdp->run_queue, &sbp);
 
     erts_smp_runq_unlock(esdp->run_queue);
 
-    erts_sched_check_cpu_bind_prep_suspend(esdp);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+#endif
+    {
+	erts_sched_check_cpu_bind_prep_suspend(esdp);
 
-    if (erts_system_profile_flags.scheduler)
-    	profile_scheduler(make_small(esdp->no), am_inactive);
+	if (erts_system_profile_flags.scheduler)
+	    profile_scheduler(make_small(esdp->no), am_inactive);
 
-    sched_wall_time_change(esdp, 0);
+	sched_wall_time_change(esdp, 0);
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    }
 
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 
-	active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
-	ASSERT(active_schedulers >= 1);
-	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	    if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
+		active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_cpu_active);
+		ASSERT(active_schedulers >= 0);
+		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing);
+		ss_onlinep = &schdlr_sspnd.dirty_cpu_online;
+		ss_curr_onlinep = &schdlr_sspnd.dirty_cpu_curr_online;
+		ss_wait_curr_onlinep = &schdlr_sspnd.dirty_cpu_wait_curr_online;
+		ss_changingp = &schdlr_sspnd.dirty_cpu_changing;
+		ss_wait_activep = &schdlr_sspnd.msb.dirty_cpu_wait_active;
+		ss_activep = &schdlr_sspnd.dirty_cpu_active;
+	    } else {
+		active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_io_active);
+		ASSERT(active_schedulers >= 0);
+		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing);
+		ss_onlinep = &schdlr_sspnd.dirty_io_online;
+		ss_curr_onlinep = &schdlr_sspnd.dirty_io_curr_online;
+		ss_wait_curr_onlinep = &schdlr_sspnd.dirty_io_wait_curr_online;
+		ss_changingp = &schdlr_sspnd.dirty_io_changing;
+		ss_wait_activep = &schdlr_sspnd.msb.dirty_io_wait_active;
+		ss_activep = &schdlr_sspnd.dirty_io_active;
+	    }
+	    ss_wait_active_target = 0;
+	}
+	else
+#endif
+	{
+	    active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
+	    ASSERT(active_schedulers >= 1);
+	    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	    ss_onlinep = &schdlr_sspnd.online;
+	    ss_curr_onlinep = &schdlr_sspnd.curr_online;
+	    ss_wait_curr_onlinep = &schdlr_sspnd.wait_curr_online;
+	    ss_changingp = &schdlr_sspnd.changing;
+	    ss_wait_activep = &schdlr_sspnd.msb.wait_active;
+	    ss_activep = &schdlr_sspnd.active;
+	    ss_wait_active_target = 1;
+	}
 	if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
-	    if (active_schedulers == schdlr_sspnd.msb.wait_active)
+	    if (active_schedulers == *ss_wait_activep)
 		wake = 1;
-	    if (active_schedulers == 1) {
-		changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+	    if (active_schedulers == ss_wait_active_target) {
+		changing = erts_smp_atomic32_read_band_nob(ss_changingp,
 							   ~ERTS_SCHDLR_SSPND_CHNG_MSB);
 		changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
 	    }
@@ -6090,21 +6433,21 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	while (1) {
 	    if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
 		int changed = 0;
-		if (no > schdlr_sspnd.online && curr_online) {
-		    schdlr_sspnd.curr_online--;
+		if (no > *ss_onlinep && curr_online) {
+		    (*ss_curr_onlinep)--;
 		    curr_online = 0;
 		    changed = 1;
 		}
-		else if (no <= schdlr_sspnd.online && !curr_online) {
-		    schdlr_sspnd.curr_online++;
+		else if (no <= *ss_onlinep && !curr_online) {
+		    (*ss_curr_onlinep)++;
 		    curr_online = 1;
 		    changed = 1;
 		}
 		if (changed
-		    && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
+		    && *ss_curr_onlinep == *ss_wait_curr_onlinep)
 		    wake = 1;
-		if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
-		    changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+		if (*ss_onlinep == *ss_curr_onlinep) {
+		    changing = erts_smp_atomic32_read_band_nob(ss_changingp,
 							       ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
 		    changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
 		}
@@ -6130,7 +6473,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			 & ERTS_RUNQ_FLGS_QMASK);
 		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 		if (aux_work|qmask) {
-		    if (!thr_prgr_active) {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			sched_wall_time_change(esdp, 1);
 		    }
@@ -6138,21 +6481,40 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			aux_work = handle_aux_work(&esdp->aux_work_data,
 						   aux_work,
 						   1);
-		    if (aux_work && erts_thr_progress_update(esdp))
+
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
+			(aux_work && erts_thr_progress_update(esdp)))
 			erts_thr_progress_leader_update(esdp);
 		    if (qmask) {
-			erts_smp_runq_lock(esdp->run_queue);
-			evacuate_run_queue(esdp->run_queue, &sbp);
-			erts_smp_runq_unlock(esdp->run_queue);
+#ifdef ERTS_DIRTY_SCHEDULERS
+			if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+			    erts_smp_runq_lock(esdp->run_queue);
+			    if (ongoing_multi_scheduling_block())
+				evacuate_run_queue(esdp->run_queue, &sbp);
+			    erts_smp_runq_unlock(esdp->run_queue);
+			    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+			} else
+#endif
+			{
+			    erts_smp_runq_lock(esdp->run_queue);
+			    evacuate_run_queue(esdp->run_queue, &sbp);
+			    erts_smp_runq_unlock(esdp->run_queue);
+			}
 		    }
 		}
 
 		if (!aux_work) {
-		    if (thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			sched_wall_time_change(esdp, 0);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+#endif
+		    {
+			if (thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			    sched_wall_time_change(esdp, 0);
+			}
+			erts_thr_progress_prepare_wait(esdp);
 		    }
-		    erts_thr_progress_prepare_wait(esdp);
 		    flgs = sched_spin_suspended(ssi,
 						ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
 		    if (flgs == (ERTS_SSI_FLG_SLEEPING
@@ -6170,24 +6532,230 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			    } while (res == EINTR);
 			}
 		    }
-		    erts_thr_progress_finalize_wait(esdp);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+#endif
+			erts_thr_progress_finalize_wait(esdp);
 		}
 
 		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
 						       | ERTS_SSI_FLG_SUSPENDED));
 		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		    break;
-		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+		changing = erts_smp_atomic32_read_nob(ss_changingp);
 		if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
 		    break;
 	    }
 
 	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	    changing = erts_smp_atomic32_read_nob(ss_changingp);
 	}
 
-	active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active);
-	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	active_schedulers = erts_smp_atomic32_inc_read_nob(ss_activep);
+	changing = erts_smp_atomic32_read_nob(ss_changingp);
+	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
+	    && *ss_onlinep == active_schedulers) {
+	    erts_smp_atomic32_read_band_nob(ss_changingp,
+					    ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+	}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+#endif
+	    ASSERT(no <= *ss_onlinep);
+	ASSERT(!ongoing_multi_scheduling_block());
+
+    }
+
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
+    ASSERT(curr_online);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+#endif
+    {
+	if (erts_system_profile_flags.scheduler)
+	    profile_scheduler(make_small(esdp->no), am_active);
+
+	if (!thr_prgr_active) {
+	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	    sched_wall_time_change(esdp, 1);
+	}
+    }
+
+    erts_smp_runq_lock(esdp->run_queue);
+    non_empty_runq(esdp->run_queue);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+#endif
+    {
+	schedule_bound_processes(esdp->run_queue, &sbp);
+
+	erts_sched_check_cpu_bind_post_suspend(esdp);
+    }
+}
+
+#else /* !ERTS_DIRTY_SCHEDULERS */
+
+static void
+suspend_scheduler(ErtsSchedulerData *esdp)
+{
+    erts_aint32_t flgs;
+    erts_aint32_t changing;
+    long no = (long) esdp->no;
+    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
+    long active_schedulers;
+    int curr_online = 1;
+    int wake = 0;
+    erts_aint32_t aux_work;
+    int thr_prgr_active = 1;
+    ErtsStuckBoundProcesses sbp = {NULL, NULL};
+
+    /*
+     * Schedulers may be suspended in two different ways:
+     * - A scheduler may be suspended since it is not online.
+     *   All schedulers with scheduler ids greater than
+     *   schdlr_sspnd.online are suspended.
+     * - Multi scheduling is blocked. All schedulers except the
+     *   scheduler with scheduler id 1 are suspended.
+     *
+     * Regardless of why a scheduler is suspended, it ends up here.
+     */
+
+    ASSERT(no != 1);
+
+    evacuate_run_queue(esdp->run_queue, &sbp);
+
+    erts_smp_runq_unlock(esdp->run_queue);
+
+    erts_sched_check_cpu_bind_prep_suspend(esdp);
+
+    if (erts_system_profile_flags.scheduler)
+    	profile_scheduler(make_small(esdp->no), am_inactive);
+
+    sched_wall_time_change(esdp, 0);
+
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+
+    flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
+    if (flgs & ERTS_SSI_FLG_SUSPENDED) {
+
+	active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
+	ASSERT(active_schedulers >= 1);
+	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
+	    if (active_schedulers == schdlr_sspnd.msb.wait_active)
+		wake = 1;
+	    if (active_schedulers == 1) {
+		changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+							   ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+		changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
+	    }
+	}
+
+	while (1) {
+	    if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+		int changed = 0;
+		if (no > schdlr_sspnd.online && curr_online) {
+		    schdlr_sspnd.curr_online--;
+		    curr_online = 0;
+		    changed = 1;
+		}
+		else if (no <= schdlr_sspnd.online && !curr_online) {
+		    schdlr_sspnd.curr_online++;
+		    curr_online = 1;
+		    changed = 1;
+		}
+		if (changed
+		    && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
+		    wake = 1;
+		if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
+		    changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+							       ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+		    changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
+		}
+	    }
+
+	    if (wake) {
+		erts_smp_cnd_signal(&schdlr_sspnd.cnd);
+		wake = 0;
+	    }
+
+	    if (curr_online && !ongoing_multi_scheduling_block()) {
+		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
+		    break;
+	    }
+	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
+	    while (1) {
+		erts_aint32_t qmask;
+		erts_aint32_t flgs;
+
+		qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
+			 & ERTS_RUNQ_FLGS_QMASK);
+		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+		if (aux_work|qmask) {
+		    if (!thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			sched_wall_time_change(esdp, 1);
+		    }
+		    if (aux_work)
+			aux_work = handle_aux_work(&esdp->aux_work_data,
+						   aux_work,
+						   1);
+		    if (aux_work && erts_thr_progress_update(esdp))
+			erts_thr_progress_leader_update(esdp);
+		    if (qmask) {
+			erts_smp_runq_lock(esdp->run_queue);
+			evacuate_run_queue(esdp->run_queue, &sbp);
+			erts_smp_runq_unlock(esdp->run_queue);
+		    }
+		}
+
+		if (!aux_work) {
+		    if (thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			sched_wall_time_change(esdp, 0);
+		    }
+		    erts_thr_progress_prepare_wait(esdp);
+		    flgs = sched_spin_suspended(ssi,
+						ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
+		    if (flgs == (ERTS_SSI_FLG_SLEEPING
+				 | ERTS_SSI_FLG_WAITING
+				 | ERTS_SSI_FLG_SUSPENDED)) {
+			flgs = sched_set_suspended_sleeptype(ssi);
+			if (flgs == (ERTS_SSI_FLG_SLEEPING
+				     | ERTS_SSI_FLG_TSE_SLEEPING
+				     | ERTS_SSI_FLG_WAITING
+				     | ERTS_SSI_FLG_SUSPENDED)) {
+			    int res;
+
+			    do {
+				res = erts_tse_wait(ssi->event);
+			    } while (res == EINTR);
+			}
+		    }
+		    erts_thr_progress_finalize_wait(esdp);
+		}
+
+		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
+						       | ERTS_SSI_FLG_SUSPENDED));
+		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
+		    break;
+		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+		if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
+		    break;
+	    }
+
+	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	}
+
+	active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active);
+	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
 	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
 	    && schdlr_sspnd.online == active_schedulers) {
 	    erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
@@ -6219,6 +6787,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
     erts_sched_check_cpu_bind_post_suspend(esdp);
 }
 
+#endif
+
 ErtsSchedSuspendResult
 erts_schedulers_state(Uint *total,
 		      Uint *online,
@@ -6230,40 +6800,315 @@ erts_schedulers_state(Uint *total,
 {
     int res = ERTS_SCHDLR_SSPND_EINVAL;
     erts_aint32_t changing;
-    if (total) {
-	ASSERT(online);
-	ASSERT(active);
-	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-	if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
-	    res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
-	else {
-	    *active = *online = schdlr_sspnd.online;
-	    if (ongoing_multi_scheduling_block())
-		*active = 1;
-	    res = ERTS_SCHDLR_SSPND_DONE;
-	}
-	erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-	*total = erts_no_schedulers;
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
+		 | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing));
+#endif
+    if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
+	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
+    else {
+	if (active)
+	    *active = schdlr_sspnd.online;
+	if (online)
+	    *online = schdlr_sspnd.online;
+	if (ongoing_multi_scheduling_block() && active)
+	    *active = 1;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (dirty_cpu_online)
+	    *dirty_cpu_online = schdlr_sspnd.dirty_cpu_online;
+#endif
+	res = ERTS_SCHDLR_SSPND_DONE;
     }
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    if (total)
+	*total = erts_no_schedulers;
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (dirty_cpu)
 	*dirty_cpu = erts_no_dirty_cpu_schedulers;
-    if (dirty_cpu_online)
-	*dirty_cpu_online = erts_no_dirty_cpu_schedulers_online;
     if (dirty_io)
 	*dirty_io = erts_no_dirty_io_schedulers;
-#else
-    if (dirty_cpu)
-	*dirty_cpu = 0;
-    if (dirty_cpu_online)
-	*dirty_cpu_online = 0;
-    if (dirty_io)
-	*dirty_io = 0;
 #endif
     return res;
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+ErtsSchedSuspendResult
+erts_set_schedulers_online(Process *p,
+			   ErtsProcLocks plocks,
+			   Sint new_no,
+			   Sint *old_no
+#ifdef ERTS_DIRTY_SCHEDULERS
+			   , int dirty_only
+#endif
+			   )
+{
+    ErtsSchedulerData *esdp;
+    int ix, res = -1, no, have_unlocked_plocks, end_wait;
+    erts_aint32_t changing = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ErtsSchedulerSleepInfo* ssi;
+    int dirty_no, change_dirty;
+#endif
+
+    if (new_no < 1)
+	return ERTS_SCHDLR_SSPND_EINVAL;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no)
+	return ERTS_SCHDLR_SSPND_EINVAL;
+#endif
+    else if (erts_no_schedulers < new_no)
+	return ERTS_SCHDLR_SSPND_EINVAL;
+
+    esdp = ERTS_PROC_GET_SCHDATA(p);
+    end_wait = 0;
+
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+
+    have_unlocked_plocks = 0;
+    no = (int) new_no;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(schdlr_sspnd.dirty_cpu_online <= erts_no_dirty_cpu_schedulers);
+    if (dirty_only) {
+	if (no > schdlr_sspnd.online) {
+	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+	    return ERTS_SCHDLR_SSPND_EINVAL;
+	}
+	dirty_no = no;
+    } else {
+	/*
+	 * Adjust the number of dirty CPU schedulers online relative to the
+	 * adjustment made to the number of normal schedulers online.
+	 */
+	int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers;
+	int onln_pct = no*total_pct/schdlr_sspnd.online;
+	dirty_no = schdlr_sspnd.dirty_cpu_online*onln_pct/100;
+	if (dirty_no == 0)
+	    dirty_no = 1;
+	ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers);
+    }
+#endif
+    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    changing |= erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing);
+#endif
+    if (changing) {
+	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
+    }
+    else {
+	int online = *old_no = schdlr_sspnd.online;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	int dirty_online = schdlr_sspnd.dirty_cpu_online;
+
+	if (dirty_only) {
+	    *old_no = schdlr_sspnd.dirty_cpu_online;
+	    if (dirty_no == schdlr_sspnd.dirty_cpu_online) {
+		res = ERTS_SCHDLR_SSPND_DONE;
+	    }
+	    change_dirty = 1;
+	} else {
+#endif
+	    if (no == schdlr_sspnd.online) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+		dirty_only = 1;
+		if (dirty_no == schdlr_sspnd.dirty_cpu_online)
+#endif
+		    res = ERTS_SCHDLR_SSPND_DONE;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		else
+		    change_dirty = 1;
+#endif
+	    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    else
+		change_dirty = (dirty_no != schdlr_sspnd.dirty_cpu_online);
+	}
+#endif
+	if (res == -1)
+	{
+	    int increase = (no > online);
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    if (!dirty_only) {
+#endif
+		ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+					    | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+		schdlr_sspnd.online = no;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    } else
+		increase = (dirty_no > dirty_online);
+	    if (change_dirty) {
+		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+						      | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+		schdlr_sspnd.dirty_cpu_online = dirty_no;
+	    }
+#endif
+	    if (increase) {
+		int ix;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (!dirty_only) {
+#endif
+		    schdlr_sspnd.wait_curr_online = no;
+		    if (ongoing_multi_scheduling_block()) {
+			for (ix = online; ix < no; ix++)
+			    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+		    }
+		    else {
+			if (plocks) {
+			    have_unlocked_plocks = 1;
+			    erts_smp_proc_unlock(p, plocks);
+			}
+			change_no_used_runqs(no);
+
+			for (ix = online; ix < no; ix++)
+			    resume_run_queue(ERTS_RUNQ_IX(ix));
+
+			for (ix = no; ix < erts_no_run_queues; ix++)
+			    suspend_run_queue(ERTS_RUNQ_IX(ix));
+		    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+		}
+		if (change_dirty) {
+		    schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no;
+		    ASSERT(schdlr_sspnd.dirty_cpu_curr_online !=
+			   schdlr_sspnd.dirty_cpu_wait_curr_online);
+		    if (ongoing_multi_scheduling_block()) {
+			for (ix = dirty_online; ix < dirty_no; ix++) {
+			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+			    erts_sched_poke(ssi);
+			}
+		    } else {
+			for (ix = dirty_online; ix < dirty_no; ix++) {
+			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+			    scheduler_ssi_resume_wake(ssi);
+			    erts_smp_atomic32_read_band_nob(&ssi->flags,
+							    ~ERTS_SSI_FLG_SUSPENDED);
+			}
+			wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+		    }
+		}
+#endif
+		res = ERTS_SCHDLR_SSPND_DONE;
+	    }
+	    else /* if (no < online) */ {
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (change_dirty) {
+		    schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no;
+		    ASSERT(schdlr_sspnd.dirty_cpu_curr_online !=
+			   schdlr_sspnd.dirty_cpu_wait_curr_online);
+		    if (ongoing_multi_scheduling_block()) {
+			for (ix = dirty_no; ix < dirty_online; ix++) {
+			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+			    erts_sched_poke(ssi);
+			}
+		    } else {
+			for (ix = dirty_no; ix < dirty_online; ix++) {
+			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+			    erts_smp_atomic32_read_bor_nob(&ssi->flags,
+							   ERTS_SSI_FLG_SUSPENDED);
+			}
+			wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+		    }
+		}
+		if (dirty_only) {
+		    res = ERTS_SCHDLR_SSPND_DONE;
+		}
+		else
+#endif
+		{
+		    if (p->scheduler_data->no <= no) {
+			res = ERTS_SCHDLR_SSPND_DONE;
+			schdlr_sspnd.wait_curr_online = no;
+		    }
+		    else {
+			/*
+			 * Yield! Current process needs to migrate
+			 * before bif returns.
+			 */
+			res = ERTS_SCHDLR_SSPND_YIELD_DONE;
+			schdlr_sspnd.wait_curr_online = no+1;
+		    }
+
+		    if (ongoing_multi_scheduling_block()) {
+			for (ix = no; ix < online; ix++)
+			    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+		    }
+		    else {
+			if (plocks) {
+			    have_unlocked_plocks = 1;
+			    erts_smp_proc_unlock(p, plocks);
+			}
+
+			change_no_used_runqs(no);
+			for (ix = no; ix < erts_no_run_queues; ix++)
+			    suspend_run_queue(ERTS_RUNQ_IX(ix));
+
+			for (ix = no; ix < online; ix++) {
+			    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+			    wake_scheduler(rq);
+			}
+		    }
+		}
+	    }
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    if (change_dirty) {
+		while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online)
+		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
+		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing,
+						~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+	    }
+	    if (!dirty_only)
+#endif
+	    {
+		if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) {
+		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+		    if (plocks && !have_unlocked_plocks) {
+			have_unlocked_plocks = 1;
+			erts_smp_proc_unlock(p, plocks);
+		    }
+		    erts_thr_progress_active(esdp, 0);
+		    erts_thr_progress_prepare_wait(esdp);
+		    end_wait = 1;
+		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+		}
+
+		while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
+		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+
+		ASSERT(res != ERTS_SCHDLR_SSPND_DONE
+		       ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
+			  & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
+		       : (ERTS_SCHDLR_SSPND_CHNG_WAITER
+			  == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
+		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+						~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+	    }
+	}
+    }
+
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(schdlr_sspnd.dirty_cpu_online <= schdlr_sspnd.online);
+    if (!dirty_only)
+#endif
+    {
+	if (end_wait) {
+	    erts_thr_progress_finalize_wait(esdp);
+	    erts_thr_progress_active(esdp, 1);
+	}
+	if (have_unlocked_plocks)
+	    erts_smp_proc_lock(p, plocks);
+    }
+
+    return res;
+}
+
+#else /* !ERTS_DIRTY_SCHEDULERS */
+
 ErtsSchedSuspendResult
 erts_set_schedulers_online(Process *p,
 			   ErtsProcLocks plocks,
@@ -6352,10 +7197,6 @@ erts_set_schedulers_online(Process *p,
 			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 			wake_scheduler(rq);
 		    }
-#ifdef ERTS_DIRTY_SCHEDULERS
-		    wake_dirty_scheduler(ERTS_DIRTY_CPU_RUNQ);
-		    wake_dirty_scheduler(ERTS_DIRTY_IO_RUNQ);
-#endif
 		}
 	    }
 
@@ -6396,15 +7237,24 @@ erts_set_schedulers_online(Process *p,
     return res;
 }
 
+#endif
+
 ErtsSchedSuspendResult
 erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 {
-    int ix, res, have_unlocked_plocks = 0;
+    int ix, res, have_unlocked_plocks = 0, online;
     erts_aint32_t changing;
     ErtsProcList *plp;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ErtsSchedulerSleepInfo* ssi;
+#endif
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
     changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
+		 | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing));
+#endif
     if (changing) {
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */
     }
@@ -6414,10 +7264,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	    erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp);
 	    p->flags |= F_HAVE_BLCKD_MSCHED;
 	    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 0);
+	    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) == 0);
+#endif
 	    ASSERT(p->scheduler_data->no == 1);
 	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-	}
-	else {
+	} else {
 	    int online = schdlr_sspnd.online;
 	    p->flags |= F_HAVE_BLCKD_MSCHED;
 	    if (plocks) {
@@ -6429,6 +7282,35 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	    if (online == 1) {
 		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 1);
+		ASSERT(!(erts_smp_atomic32_read_nob(&ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0)->flags)
+			 & ERTS_SSI_FLG_SUSPENDED));
+		schdlr_sspnd.msb.dirty_cpu_wait_active = 0;
+		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
+						      | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+		ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0);
+		erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+		wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active)
+		       != schdlr_sspnd.msb.dirty_cpu_wait_active)
+		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
+
+		schdlr_sspnd.msb.dirty_io_wait_active = 0;
+		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
+						     | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+		    ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
+						   ERTS_SSI_FLG_SUSPENDED);
+		}
+		wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
+		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active)
+		       != schdlr_sspnd.msb.dirty_io_wait_active)
+		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
+#endif
 		ASSERT(p->scheduler_data->no == 1);
 	    }
 	    else {
@@ -6447,6 +7329,37 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    schdlr_sspnd.msb.wait_active = 2;
 		}
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+		schdlr_sspnd.msb.dirty_cpu_wait_active = 0;
+		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
+						      | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+		for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
+						   ERTS_SSI_FLG_SUSPENDED);
+		}
+		wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active)
+		       != schdlr_sspnd.msb.dirty_cpu_wait_active)
+		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
+		ASSERT(schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_online);
+
+		schdlr_sspnd.msb.dirty_io_wait_active = 0;
+		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
+						     | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+		    ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
+						   ERTS_SSI_FLG_SUSPENDED);
+		}
+		wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
+		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active)
+		       != schdlr_sspnd.msb.dirty_io_wait_active)
+		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
+		ASSERT(schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_online);
+#endif
 		change_no_used_runqs(1);
 		for (ix = 1; ix < erts_no_run_queues; ix++)
 		    suspend_run_queue(ERTS_RUNQ_IX(ix));
@@ -6487,6 +7400,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
 		}
+
 		ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED
 		       ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
 			  & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
@@ -6527,12 +7441,12 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	    p->flags &= ~F_HAVE_BLCKD_MSCHED;
 	    schdlr_sspnd.msb.ongoing = 0;
 	    if (schdlr_sspnd.online == 1) {
-		/* No schedulers to resume */
+		/* No normal schedulers to resume */
 		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
 		ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB);
 	    }
 	    else {
-		int online = schdlr_sspnd.online;
+		online = schdlr_sspnd.online;
 		if (plocks) {
 		    have_unlocked_plocks = 1;
 		    erts_smp_proc_unlock(p, plocks);
@@ -6547,6 +7461,27 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		for (ix = online; ix < erts_no_run_queues; ix++)
 		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 	    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
+	    schdlr_sspnd.msb.dirty_cpu_wait_active = schdlr_sspnd.dirty_cpu_online;
+	    for (ix = 0; ix < schdlr_sspnd.dirty_cpu_online; ix++) {
+		ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+		scheduler_ssi_resume_wake(ssi);
+		erts_smp_atomic32_read_band_nob(&ssi->flags,
+						~ERTS_SSI_FLG_SUSPENDED);
+	    }
+	    wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+
+	    ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
+	    schdlr_sspnd.msb.dirty_io_wait_active = erts_no_dirty_io_schedulers;
+	    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+		ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+		scheduler_ssi_resume_wake(ssi);
+		erts_smp_atomic32_read_band_nob(&ssi->flags,
+						~ERTS_SSI_FLG_SUSPENDED);
+	    }
+	    wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
+#endif
 	    res = ERTS_SCHDLR_SSPND_DONE;
 	}
     }
@@ -6673,7 +7608,11 @@ sched_thread_func(void *vesdp)
 	erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
 					~ERTS_SCHDLR_SSPND_CHNG_ONLN);
 	if (no != 1)
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
+#else
 	    erts_smp_cnd_signal(&schdlr_sspnd.cnd);
+#endif
     }
 
     if (no == 1) {
@@ -6710,7 +7649,9 @@ sched_dirty_cpu_thread_func(void *vesdp)
 {
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
-    Uint no = esdp->dirty_no;
+    Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+    ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_CPU_SCHEDULER;
+    ASSERT(no != 0);
     ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
     callbacks.wakeup = thr_prgr_wakeup;
@@ -6738,6 +7679,24 @@ sched_dirty_cpu_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
+	   & ERTS_SCHDLR_SSPND_CHNG_ONLN);
+
+    if (--schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_wait_curr_online) {
+	erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing,
+					~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+	if (no != 1)
+	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
+    }
+
+    if (no == 1) {
+	while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online)
+	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+	ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
+    }
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
     process_main();
     /* No schedulers should *ever* terminate */
     erl_exit(ERTS_ABORT_EXIT,
@@ -6751,7 +7710,9 @@ sched_dirty_io_thread_func(void *vesdp)
 {
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
-    Uint no = esdp->dirty_no;
+    Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+    ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_IO_SCHEDULER;
+    ASSERT(no != 0);
     ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
     callbacks.wakeup = thr_prgr_wakeup;
@@ -6779,6 +7740,24 @@ sched_dirty_io_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)
+	   & ERTS_SCHDLR_SSPND_CHNG_ONLN);
+
+    if (--schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_wait_curr_online) {
+	erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_io_changing,
+					~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+	if (no != 1)
+	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
+    }
+
+    if (no == 1) {
+	while (schdlr_sspnd.dirty_io_curr_online != schdlr_sspnd.dirty_io_wait_curr_online)
+	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+	ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
+    }
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
     process_main();
     /* No schedulers should *ever* terminate */
     erl_exit(ERTS_ABORT_EXIT,
@@ -7995,7 +8974,6 @@ Process *schedule(Process *p, int calls)
 	       || flags & ERTS_RUNQ_FLG_NONEMPTY);
 
 	if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
-	
 	    if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
 		suspend_scheduler(esdp);
 		flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
@@ -8006,10 +8984,17 @@ Process *schedule(Process *p, int calls)
 		erts_sched_check_cpu_bind(esdp);
 	    }
 	}
+#ifdef ERTS_DIRTY_SCHEDULERS
+	else if (ERTS_SCHEDULER_IS_DIRTY(esdp)
+		 && (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+		     & ERTS_SSI_FLG_SUSPENDED))
+	    suspend_scheduler(esdp);
+#endif
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	{
 	    erts_aint32_t aux_work;
-	    int leader_update = erts_thr_progress_update(esdp);
+	    int leader_update = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0
+		: erts_thr_progress_update(esdp);
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
 	    if (aux_work | leader_update) {
 		erts_smp_runq_unlock(rq);
@@ -8020,7 +9005,8 @@ Process *schedule(Process *p, int calls)
 		erts_smp_runq_lock(rq);
 	    }
 
-	    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+	    ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)
+			       || !erts_thr_progress_is_blocking());
 	}
 	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
@@ -8035,6 +9021,17 @@ Process *schedule(Process *p, int calls)
 
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->halt_in_progress) {
+	    /*
+	     * TODO: if halt in progress, need to put the dirty scheduler
+	     * to sleep somewhere around here to prevent it from picking up
+	     * new work
+	     */
+	}
+	else
+#endif
+
 	if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start)
 	    || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) {
 	    /* Prepare for scheduler wait */
@@ -8182,11 +9179,15 @@ Process *schedule(Process *p, int calls)
 							+ ERTS_PSFLGS_IN_PRQ_MASK_OFFSET));
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-	    /* if a non-dirty scheduler picks up a process marked as already being
-	       in a dirty run queue, just drop it and go get another process */
-	    if (state & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) &&
-		!ERTS_SCHEDULER_IS_DIRTY(esdp))
-		goto pick_next_process;
+	    ASSERT((state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) !=
+		   (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC));
+	    if (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) {
+		ASSERT((ERTS_SCHEDULER_IS_DIRTY_CPU(esdp) && (state & ERTS_PSFLG_DIRTY_CPU_PROC)) ||
+		       (ERTS_SCHEDULER_IS_DIRTY_IO(esdp) && (state & ERTS_PSFLG_DIRTY_IO_PROC)));
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !(state & ERTS_PSFLG_ACTIVE_SYS))
+		    goto pick_next_process;
+		state &= ~(ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q);
+	    }
 #endif
 
 	    if (!(state & ERTS_PSFLG_PROXY))
@@ -8322,7 +9323,11 @@ Process *schedule(Process *p, int calls)
 
 	if (state & ERTS_PSFLG_RUNNING_SYS) {
 	    reds -= execute_sys_tasks(p, &state, reds);
-	    if (reds <= 0) {
+	    if (reds <= 0
+#ifdef ERTS_DIRTY_SCHEDULERS
+		|| (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))
+#endif
+		) {
 		p->fcalls = reds;
 		goto sched_out_proc;
 	    }
@@ -10087,7 +11092,7 @@ save_pending_exiter(Process *p)
     erts_smp_runq_unlock(rq);
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-	wake_dirty_scheduler(rq);
+	wake_dirty_schedulers(rq, 0);
     else
 #endif
 	wake_scheduler(rq);
@@ -11148,6 +12153,10 @@ void erl_halt(int code)
     if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress,
 					     erts_no_schedulers,
 					     -1)) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+	ERTS_DIRTY_CPU_RUNQ->halt_in_progress = 1;
+	ERTS_DIRTY_IO_RUNQ->halt_in_progress = 1;
+#endif
 	erts_halt_code = code;
 	notify_reap_ports_relb();
     }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index dcb9251d0d..f6b185c0ad 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -109,7 +109,6 @@ extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
 #ifdef ERTS_DIRTY_SCHEDULERS
 extern Uint erts_no_dirty_cpu_schedulers;
-extern Uint erts_no_dirty_cpu_schedulers_online;
 extern Uint erts_no_dirty_io_schedulers;
 #endif
 extern Uint erts_no_run_queues;
@@ -544,6 +543,21 @@ typedef struct {
 #endif
 } ErtsAuxWorkData;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+typedef enum {
+    ERTS_DIRTY_CPU_SCHEDULER,
+    ERTS_DIRTY_IO_SCHEDULER
+} ErtsDirtySchedulerType;
+
+typedef union {
+    struct {
+	ErtsDirtySchedulerType type: 1;
+	unsigned num: 31;
+    } s;
+    Uint no;
+} ErtsDirtySchedId;
+#endif
+
 struct ErtsSchedulerData_ {
     /*
      * Keep X registers first (so we get as many low
@@ -570,7 +584,7 @@ struct ErtsSchedulerData_ {
     Process *current_process;
     Uint no;			/* Scheduler number for normal schedulers */
 #ifdef ERTS_DIRTY_SCHEDULERS
-    Uint dirty_no;		/* Scheduler number for dirty schedulers */
+    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
 #endif
     Port *current_port;
     ErtsRunQueue *run_queue;
@@ -1292,6 +1306,8 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
    &erts_aligned_run_queues[(IX)].runq)
 #define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
 #define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
+#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
+#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
 #else
 #define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
 #endif
@@ -1305,21 +1321,37 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
   (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
    &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
+#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
+  ((ESDP)->dirty_no.s.num)
+#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
+  ((ESDP)->dirty_no.s.type)
 #ifdef ERTS_SMP
 #define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
-  ((ESDP)->dirty_no != 0)
+  ((ESDP)->dirty_no.s.num != 0)
+#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
+    ((ESDP)->dirty_no.s.type == 0)
+#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
+    ((ESDP)->dirty_no.s.type == 1)
 #else
 #define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
+#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
+#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
 #endif
 #else
 #define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
 #define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
+#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
+#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
 #endif
 
 void erts_pre_init_process(void);
 void erts_late_init_process(void);
 void erts_early_init_scheduling(int);
-void erts_init_scheduling(int, int);
+void erts_init_scheduling(int, int
+#ifdef ERTS_DIRTY_SCHEDULERS
+			  , int, int, int
+#endif
+			  );
 
 int erts_set_gc_state(Process *c_p, int enable);
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
@@ -1526,7 +1558,11 @@ ErtsSchedSuspendResult
 erts_set_schedulers_online(Process *p,
 			   ErtsProcLocks plocks,
 			   Sint new_no,
-			   Sint *old_no);
+			   Sint *old_no
+#ifdef ERTS_DIRTY_SCHEDULERS
+			   , int dirty_only
+#endif
+			   );
 ErtsSchedSuspendResult
 erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
 int erts_is_multi_scheduling_blocked(void);
-- 
cgit v1.2.3


From 200fbe924466720bd2a8c5eb05b05d67b0a2414c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 14 Mar 2013 15:42:19 +0100
Subject: Added support for ENEA OSE

This port has support for both non-smp and smp.

It contains a new way to do io checking in which erts_poll_wait
receives the payload of the polled entity. This has implications
for all linked-in drivers.
---
 erts/emulator/beam/beam_bif_load.c    |  2 +-
 erts/emulator/beam/beam_emu.c         |  5 +++
 erts/emulator/beam/beam_load.c        |  2 +-
 erts/emulator/beam/bif.c              |  8 ++--
 erts/emulator/beam/erl_alloc.types    | 15 +++++++
 erts/emulator/beam/erl_async.c        |  7 +++
 erts/emulator/beam/erl_bif_lists.c    |  6 +--
 erts/emulator/beam/erl_bif_port.c     |  5 ++-
 erts/emulator/beam/erl_cpu_topology.c |  2 +-
 erts/emulator/beam/erl_driver.h       | 18 ++++++++
 erts/emulator/beam/erl_drv_thread.c   |  3 +-
 erts/emulator/beam/erl_nif.c          |  2 +-
 erts/emulator/beam/erl_port_task.c    |  3 +-
 erts/emulator/beam/erl_process.c      | 82 ++++++++++++++++++++++++++++++-----
 erts/emulator/beam/erl_process_dict.c |  2 +-
 erts/emulator/beam/erl_process_lock.c |  2 +-
 erts/emulator/beam/erl_trace.c        |  9 ++++
 erts/emulator/beam/erl_utils.h        |  2 +-
 erts/emulator/beam/global.h           |  3 ++
 erts/emulator/beam/io.c               |  7 +++
 erts/emulator/beam/sys.h              |  2 +
 erts/emulator/beam/utils.c            |  2 +-
 22 files changed, 159 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 3f92c5b025..df1983a83d 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -201,7 +201,7 @@ finish_loading_1(BIF_ALIST_1)
      * to keep the elements in.
      */
 
-    n = list_length(BIF_ARG_1);
+    n = erts_list_length(BIF_ARG_1);
     if (n == -1) {
 	ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
 	goto done;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 89d9442526..ceff43b24f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1190,11 +1190,16 @@ void process_main(void)
      * c_p->arg_reg before calling the scheduler.
      */
     if (!init_done) {
+       /* This should only be reached during the init phase when only the main
+        * process is running. I.e. there is no race for init_done.
+        */
 	init_done = 1;
 	goto init_emulator;
     }
+
     c_p = NULL;
     reds_used = 0;
+
     goto do_schedule1;
 
  do_schedule:
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index b589d1c930..c983615827 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5824,7 +5824,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     Funcs = tp[1];
     Patchlist = tp[2];        
    
-    if ((n = list_length(Funcs)) < 0) {
+    if ((n = erts_list_length(Funcs)) < 0) {
 	goto error;
     }
     if ((bytes = erts_get_aligned_binary_bytes(Beam, &temp_alloc)) == NULL) {
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 9c4801041f..1f568726a6 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2675,7 +2675,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
 
     if (i < 0) {
 	erts_free(ERTS_ALC_T_TMP, (void *) buf);
-	i = list_length(BIF_ARG_1);
+	i = erts_list_length(BIF_ARG_1);
 	if (i > MAX_ATOM_CHARACTERS) {
 	    BIF_ERROR(BIF_P, SYSTEM_LIMIT);
 	}
@@ -2953,7 +2953,7 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2)
     char *buf = NULL;
     int base;
 
-    i = list_length(BIF_ARG_1);
+    i = erts_list_length(BIF_ARG_1);
     if (i < 0)
       BIF_ERROR(BIF_P, BADARG);
     
@@ -3292,7 +3292,7 @@ BIF_RETTYPE list_to_float_1(BIF_ALIST_1)
     Eterm res;
     char *buf = NULL;
 
-    i = list_length(BIF_ARG_1);
+    i = erts_list_length(BIF_ARG_1);
     if (i < 0)
       BIF_ERROR(BIF_P, BADARG);
     
@@ -3407,7 +3407,7 @@ BIF_RETTYPE list_to_tuple_1(BIF_ALIST_1)
     Eterm* hp;
     int len;
 
-    if ((len = list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) {
+    if ((len = erts_list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) {
 	BIF_ERROR(BIF_P, BADARG);
     }
 
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index b4e52770e3..17ac6316b7 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -414,6 +414,21 @@ type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
 
 +endif
 
++if ose
+
+type	SYS_READ_BUF	TEMPORARY	SYSTEM		sys_read_buf
+type	FD_TAB		LONG_LIVED	SYSTEM		fd_tab
+type	FD_ENTRY_BUF	STANDARD	SYSTEM		fd_entry_buf
+type	FD_SIG_LIST	SHORT_LIVED	SYSTEM		fd_sig_list
+type    DRV_EV		STANDARD	SYSTEM		driver_event
+type	CS_PROG_PATH	LONG_LIVED	SYSTEM		cs_prog_path
+type	ENVIRONMENT	TEMPORARY	SYSTEM		environment
+type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
+type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
+
++endif
+
+
 +if win32
 
 type	DRV_DATA_BUF	SYSTEM		SYSTEM		drv_data_buf
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index f0cec1c53c..afb5bc302f 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -226,6 +226,13 @@ erts_init_async(void)
 	thr_opts.suggested_stack_size
 	    = erts_async_thread_suggested_stack_size;
 
+#ifdef ETHR_HAVE_THREAD_NICENESS
+	thr_opts.prio += 2;
+#endif
+#ifdef ETHR_HAVE_THREAD_NAMES
+	thr_opts.name = "async_thread";
+#endif
+
 	for (i = 0; i < erts_async_max_threads; i++) {
 	    ErtsAsyncQ *aq = async_q(i);
 	    erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts);
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 1805366cfe..820ed2385d 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -43,7 +43,7 @@ static BIF_RETTYPE append(Process* p, Eterm A, Eterm B)
     Eterm* hp;
     int i;
 
-    if ((i = list_length(A)) < 0) {
+    if ((i = erts_list_length(A)) < 0) {
 	BIF_ERROR(p, BADARG);
     }
     if (i == 0) {
@@ -102,10 +102,10 @@ static Eterm subtract(Process* p, Eterm A, Eterm B)
     int     n;
     int     m;
     
-    if ((n = list_length(A)) < 0) {
+    if ((n = erts_list_length(A)) < 0) {
 	BIF_ERROR(p, BADARG);
     }
-    if ((m = list_length(B)) < 0) {
+    if ((m = erts_list_length(B)) < 0) {
 	BIF_ERROR(p, BADARG);
     }
     
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index f298422267..77627a6897 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -941,7 +941,8 @@ static char **convert_args(Eterm l)
     if (is_not_list(l) && is_not_nil(l)) {
 	return NULL;
     }
-    n = list_length(l);
+
+    n = erts_list_length(l);
     /* We require at least one element in argv[0] + NULL at end */
     pp = erts_alloc(ERTS_ALC_T_TMP, (n + 2) * sizeof(char **));
     pp[i++] = erts_default_arg0;
@@ -986,7 +987,7 @@ static byte* convert_environment(Process* p, Eterm env)
     byte* bytes;
     int encoding = erts_get_native_filename_encoding();
 
-    if ((n = list_length(env)) < 0) {
+    if ((n = erts_list_length(env)) < 0) {
 	return NULL;
     }
     heap_size = 2*(5*n+1);
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 88c6c34881..f594cb9392 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -1699,7 +1699,7 @@ erts_early_init_cpu_topology(int no_schedulers,
     }
 
     max_main_threads = erts_get_cpu_configured(cpuinfo);
-    if (max_main_threads > no_schedulers)
+    if (max_main_threads > no_schedulers || max_main_threads < 0)
 	max_main_threads = no_schedulers;
     *max_main_threads_p = max_main_threads;
 
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index ab9ee63104..9ede2982de 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -271,6 +271,13 @@ typedef struct ErlDrvCond_ ErlDrvCond;
 typedef struct ErlDrvRWLock_ ErlDrvRWLock;
 typedef int ErlDrvTSDKey;
 
+/*
+ * Potential OSE signals
+ */
+#ifdef __OSE__
+typedef union SIGNAL OseSignal;
+#endif
+
 
 /*
  * 
@@ -346,6 +353,9 @@ typedef struct erl_drv_entry {
     	                        /* Called on behalf of driver_select when
 				   it is safe to release 'event'. A typical
 				   unix driver would call close(event) */
+#ifdef __OSE__
+    int (*resolve_signal)(OseSignal* sig, int* mode);
+#endif
     /* When adding entries here, dont forget to pad in obsolete/driver.h */
 } ErlDrvEntry;
 
@@ -680,6 +690,14 @@ EXTERN char *driver_dl_error(void);
 EXTERN int erl_drv_putenv(char *key, char *value);
 EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 
+#ifdef __OSE__
+EXTERN OseSignal *erl_drv_ose_get_output_signal(ErlDrvEvent ev);
+EXTERN OseSignal *erl_drv_ose_get_input_signal(ErlDrvEvent ev);
+EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig,int id);
+EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev);
+EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig,int *id);
+#endif
+
 #endif /* !ERL_DRIVER_TYPES_ONLY */
 
 #ifdef WIN32_DYNAMIC_ERL_DRIVER
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 4f1bba8657..beaeed03a2 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -78,8 +78,6 @@ struct ErlDrvTid_ {
 
 static ethr_tsd_key tid_key;
 
-static ethr_thr_opts def_ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
-
 #else /* USE_THREADS */
 static Uint tsd_len;
 static void **tsd;
@@ -605,6 +603,7 @@ erl_drv_thread_create(char *name,
     struct ErlDrvTid_ *dtid;
     ethr_thr_opts ethr_opts;
     ethr_thr_opts *use_opts;
+    ethr_thr_opts def_ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
 
     if (!opts)
 	use_opts = NULL;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c35f1fc2c6..d467d129e8 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -876,7 +876,7 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail)
 int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len)
 {
     if (is_not_list(term) && is_not_nil(term)) return 0;
-    *len = list_length(term);
+    *len = erts_list_length(term);
     return 1;
 }
 
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index d4108067d0..fb6048b41f 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1681,7 +1681,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    reds = ERTS_PORT_REDS_INPUT;
 	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
             DTRACE_DRIVER(driver_ready_input, pp);
-	    /* NOTE some windows drivers use ->ready_input for input and output */
+	    /* NOTE some windows/ose drivers use ->ready_input
+	       for input and output */
 	    (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data,
 					ptp->u.alive.td.io.event);
 	    io_tasks_executed++;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 937881212a..7e5d88cb24 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -44,6 +44,7 @@
 #include "dtrace-wrapper.h"
 #include "erl_ptab.h"
 
+
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
 #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2)
 
@@ -53,7 +54,11 @@
 
 #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10)
 
+#ifndef ERTS_SCHED_MIN_SPIN
 #define ERTS_SCHED_SPIN_UNTIL_YIELD 100
+#else
+#define ERTS_SCHED_SPIN_UNTIL_YIELD 1
+#endif
 
 #define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40
 #define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000
@@ -2682,7 +2687,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
      * be waiting in erl_sys_schedule()
      */
 
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+    if (esdp->no != 1) {
+#else
     if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule()) {
+#endif
 
 	sched_waiting(esdp->no, rq);
 
@@ -2690,7 +2699,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	spincount = sched_busy_wait.tse;
 
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+	ASSERT(esdp->no != 1);
+#else
     tse_wait:
+#endif
 
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working)
 	    sched_wall_time_change(esdp, thr_prgr_active);
@@ -2782,6 +2795,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	ASSERT(working);
 	sched_wall_time_change(esdp, working = 0);
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+	sys_poll_try:
+	while(!prepare_for_sys_schedule()) {
+		delay(1);
+	}
+#endif
 
 	spincount = sched_busy_wait.sys_schedule;
 	if (spincount == 0)
@@ -2795,7 +2814,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		sched_wall_time_change(esdp, working = 0);
 
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
-
 	    erl_sys_schedule(1); /* Might give us something to do */
 
 	    dt = erts_do_time_read_and_reset();
@@ -2843,7 +2861,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		 */
 		if (!prepare_for_sys_schedule()) {
 		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+		    goto sys_poll_try;
+#else
 		    goto tse_wait;
+#endif
 		}
 	    }
 #endif
@@ -2871,7 +2893,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		sched_change_waiting_sys_to_waiting(esdp->no, rq);
 		erts_smp_runq_unlock(rq);
 		spincount = 0;
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+		goto sys_poll_try;
+#else
 		goto tse_wait;
+#endif
 	    }
 	}
 #endif
@@ -4889,11 +4915,17 @@ erts_early_init_scheduling(int no_schedulers)
     wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
     wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
 #endif
+#ifndef ERTS_SCHED_MIN_SPIN
     sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
     sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
 			   * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT);
     sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
 				* ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM);
+#else
+    sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
+    sched_busy_wait.tse = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
+    sched_busy_wait.aux_work = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
+#endif
 }
 
 int
@@ -6795,7 +6827,7 @@ void
 erts_start_schedulers(void)
 {
     int res = 0;
-    Uint actual = 0;
+    Uint actual;
     Uint wanted = erts_no_schedulers;
     Uint wanted_no_schedulers = erts_no_schedulers;
     ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER;
@@ -6826,15 +6858,31 @@ erts_start_schedulers(void)
 	res = ENOTSUP;
     }
 
-    while (actual < wanted) {
+#ifdef ETHR_HAVE_THREAD_NAMES
+    opts.name = malloc(sizeof(char)*(strlen("scheduler_XXXX")+1));
+#endif
+
+    for (actual = 0; actual < wanted; actual++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual);
-	actual++;
-	ASSERT(actual == esdp->no);
-	res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts);
+
+	ASSERT(actual == esdp->no - 1);
+
+#ifdef ETHR_HAVE_THREAD_NAMES
+	sprintf(opts.name,"scheduler_%d", actual+1);
+#endif
+
+#ifdef __OSE__
+        /* This should be done in the bind strategy */
+	opts.coreNo = (actual+1) % ose_num_cpus();
+#endif
+
+	res = ethr_thr_create(&esdp->tid, sched_thread_func,(void*)esdp,&opts);
+
 	if (res != 0) {
-	    actual--;
-	    break;
+           //actual--;
+           break;
 	}
+
     }
     
     erts_no_schedulers = actual;
@@ -6860,6 +6908,16 @@ erts_start_schedulers(void)
 #endif
 
     ERTS_THR_MEMORY_BARRIER;
+#ifdef ETHR_HAVE_THREAD_NAMES
+    free(opts.name);
+    opts.name   = "aux_thread";
+#endif
+#ifdef __OSE__
+    opts.coreNo = 0;
+#endif
+#ifdef ETHR_HAVE_THREAD_NICENESS
+    opts.prio++;
+#endif
 
     res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
     if (res != 0)
@@ -8076,7 +8134,11 @@ Process *schedule(Process *p, int calls)
 
 	    goto check_activities_to_run;
 	}
-	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
+	else if (
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+		 esdp->no == 1 &&
+#endif
+		 !ERTS_SCHEDULER_IS_DIRTY(esdp) &&
 		 (fcalls > input_reductions && prepare_for_sys_schedule())) {
 	    /*
 	     * Schedule system-level activities.
@@ -9379,7 +9441,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
      * Check for errors.
      */
 
-    if (is_not_atom(mod) || is_not_atom(func) || ((arity = list_length(args)) < 0)) {
+    if (is_not_atom(mod) || is_not_atom(func) || ((arity = erts_list_length(args)) < 0)) {
 	so->error_code = BADARG;
 	goto error;
     }
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index a611b52af2..23e5bf737f 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -659,7 +659,7 @@ static void shrink(Process *p, Eterm* ret)
 	    } else {
 		int needed = 4;
 		if (is_list(hi) && is_list(lo)) {
-		    needed = 2*list_length(hi);
+		    needed = 2*erts_list_length(hi);
 		}
 		if (HeapWordsLeft(p) < needed) {
 		    BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1));
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 2db5df06b4..bb17593297 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1001,8 +1001,8 @@ erts_pid2proc_opt(Process *c_p,
 void
 erts_proc_lock_init(Process *p)
 {
-#if ERTS_PROC_LOCK_OWN_IMPL
     int i;
+#if ERTS_PROC_LOCK_OWN_IMPL
     /* We always start with all locks locked */
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
     erts_smp_atomic32_init_nob(&p->lock.flags,
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index ff7fdfcfca..62a0b8ccea 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3467,12 +3467,21 @@ static void
 init_sys_msg_dispatcher(void)
 {
     erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+#ifdef __OSE__
+    thr_opts.coreNo   = 0;
+#endif
     thr_opts.detached = 1;
     init_smq_element_alloc();
     sys_message_queue = NULL;
     sys_message_queue_end = NULL;
     erts_smp_cnd_init(&smq_cnd);
     erts_smp_mtx_init(&smq_mtx, "sys_msg_q");
+#ifdef ETHR_HAVE_THREAD_NAMES
+    thr_opts.name = "sys_msg_dispatcher";
+#endif
+#ifdef ETHR_HAVE_THREAD_NICENESS
+    thr_opts.prio++;
+#endif
     erts_smp_thr_create(&sys_msg_dispatcher_tid,
 			sys_msg_dispatcher_func,
 			NULL,
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 5b81d814c6..4e61429cd1 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -150,7 +150,7 @@ void erts_silence_warn_unused_result(long unused);
 
 int erts_fit_in_bits_int64(Sint64);
 int erts_fit_in_bits_int32(Sint32);
-int list_length(Eterm);
+int erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
 Uint32 make_broken_hash(Eterm);
 Uint32 block_hash(byte *, unsigned, Uint32);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 8fcb95d0e2..5a97ac5892 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -160,6 +160,9 @@ struct erts_driver_t_ {
     void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data); /* Might be NULL */ 
     void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
     void (*stop_select)(ErlDrvEvent event, void*); /* Might be NULL */
+#ifdef __OSE__
+    int (*resolve_signal)(OseSignal* sig, int* mode);
+#endif
 };
 
 extern erts_driver_t *driver_list;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3b16cdeb4a..f4d905552d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -49,7 +49,9 @@
 #include "erl_map.h"
 
 extern ErlDrvEntry fd_driver_entry;
+#ifndef __OSE__
 extern ErlDrvEntry vanilla_driver_entry;
+#endif
 extern ErlDrvEntry spawn_driver_entry;
 extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */
 
@@ -2764,7 +2766,9 @@ void erts_init_io(int port_tab_size,
     erts_smp_rwmtx_rwlock(&erts_driver_list_lock);
 
     init_driver(&fd_driver, &fd_driver_entry, NULL);
+#ifndef __OSE__
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
+#endif
     init_driver(&spawn_driver, &spawn_driver_entry, NULL);
     erts_init_static_drivers();
     for (dp = driver_tab; *dp != NULL; dp++)
@@ -7333,6 +7337,9 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 	drv->stop_select = de->stop_select;
     else
 	drv->stop_select = no_stop_select_callback;
+#ifdef __OSE__
+    drv->resolve_signal = de->resolve_signal;
+#endif
 
     if (!de->init)
 	return 0;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 189d9ebac8..e273056a2b 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -38,6 +38,8 @@
 
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
+#elif defined (__OSE__)
+#  include "erl_ose_sys.h"
 #else 
 #  include "erl_unix_sys.h"
 #ifndef UNIX
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index bc4a05d385..d92909c673 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -258,7 +258,7 @@ erl_grow_wstack(ErtsWStack* s, UWord* default_wstack)
  * Returns -1 if not a proper list (i.e. not terminated with NIL)
  */
 int
-list_length(Eterm list)
+erts_list_length(Eterm list)
 {
     int i = 0;
 
-- 
cgit v1.2.3


From 48201abc7961b44be9e5cf0feb74d18a399d6099 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 1 Aug 2013 15:50:46 +0200
Subject: erts: Make source file info available in lc

---
 erts/emulator/beam/beam_bp.c          |   3 +-
 erts/emulator/beam/beam_emu.c         |   3 +-
 erts/emulator/beam/erl_alloc_util.c   |   4 +-
 erts/emulator/beam/erl_lock_check.c   |  74 ++++++++++++---------
 erts/emulator/beam/erl_lock_check.h   |  26 ++++++--
 erts/emulator/beam/erl_lock_count.h   |   6 ++
 erts/emulator/beam/erl_port_task.h    |   8 ++-
 erts/emulator/beam/erl_process.h      |  10 ++-
 erts/emulator/beam/erl_process_lock.c |  62 +++++++++--------
 erts/emulator/beam/erl_process_lock.h |  25 ++++---
 erts/emulator/beam/erl_smp.h          |  78 ++++++++++++++--------
 erts/emulator/beam/erl_threads.h      | 122 +++++++++++++++++++++++++++-------
 erts/emulator/beam/io.c               |  13 ++--
 13 files changed, 289 insertions(+), 145 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 68907a771a..49a34ab4ad 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -46,7 +46,8 @@
 
 #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
 #  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
-      if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
+      if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
+					 __FILE__, __LINE__)
 #  define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
       if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
 #else
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ceff43b24f..c82d2fb2be 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -71,7 +71,8 @@ do {									\
     	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());		\
 } while (0)
 #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
-        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
+        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
+					   __FILE__, __LINE__)
 #    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
 #  else
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index c6cea0185f..45f0cc4312 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -5561,11 +5561,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	erts_mtx_init_x_opt(&allctr->mutex,
 			    "alcu_allocator",
 			    make_small(allctr->alloc_no),
-			    ERTS_LCNT_LT_ALLOC);
+			    ERTS_LCNT_LT_ALLOC,1);
 #else
 	erts_mtx_init_x(&allctr->mutex,
 			"alcu_allocator",
-			make_small(allctr->alloc_no));
+			make_small(allctr->alloc_no),1);
 #endif /*ERTS_ENABLE_LOCK_COUNT*/
 	
 #ifdef DEBUG
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index a8ff94ac89..706d284a00 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -241,6 +241,8 @@ struct erts_lc_locked_lock_t_ {
     erts_lc_locked_lock_t *prev;
     UWord extra;
     Sint16 id;
+    char *file;
+    unsigned int line;
     Uint16 flags;
 };
 
@@ -430,47 +432,51 @@ make_my_locked_locks(void)
 }
 
 static ERTS_INLINE erts_lc_locked_lock_t *
-new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags)
+new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags,
+		char *file, unsigned int line)
 {
     erts_lc_locked_lock_t *l_lck = (erts_lc_locked_lock_t *) lc_alloc();
     l_lck->next = NULL;
     l_lck->prev = NULL;
     l_lck->id = lck->id;
     l_lck->extra = lck->extra;
+    l_lck->file = file;
+    l_lck->line = line;
     l_lck->flags = lck->flags | op_flags;
     return l_lck;
 }
 
 static void
-print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix)
+raw_print_lock(char *prefix, Sint16 id, Wterm extra, Uint16 flags,
+	       char* file, unsigned int line, char *suffix)
 {
     char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
 		   ? erts_lock_order[id].name
 		   : "unknown");
+    erts_fprintf(stderr,"%s'%s:",prefix,lname);
+
     if (is_not_immed(extra))
-	erts_fprintf(stderr,
-		     "%s'%s:%p%s'%s%s",
-		     prefix,
-		     lname,
-		     _unchecked_boxed_val(extra),
-		     lock_type(flags),
-		     rw_op_str(flags),
-		     suffix);
+      erts_fprintf(stderr,"%p",_unchecked_boxed_val(extra));
     else
-	erts_fprintf(stderr,
-		     "%s'%s:%T%s'%s%s",
-		     prefix,
-		     lname,
-		     extra,
-		     lock_type(flags),
-		     rw_op_str(flags),
-		     suffix);
+      erts_fprintf(stderr,"%T",extra);
+    erts_fprintf(stderr,"%s",lock_type(flags));
+
+    if (file)
+      erts_fprintf(stderr,"(%s:%d)",file,line);
+
+    erts_fprintf(stderr,"'%s%s",rw_op_str(flags),suffix);
+}
+
+static void
+print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix)
+{
+  raw_print_lock(prefix, id, extra, flags, NULL, 0, suffix);
 }
 
 static void
 print_lock(char *prefix, erts_lc_lock_t *lck, char *suffix)
 {
-    print_lock2(prefix, lck->id, lck->extra, lck->flags, suffix);
+  raw_print_lock(prefix, lck->id, lck->extra, lck->flags, NULL, 0, suffix);
 }
 
 static void
@@ -486,7 +492,8 @@ print_curr_locks(erts_lc_locked_locks_t *l_lcks)
 		     "Currently these locks are locked by the %s thread:\n",
 		     l_lcks->thread_name);
 	for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next)
-	    print_lock2("  ", l_lck->id, l_lck->extra, l_lck->flags, "\n");
+	  raw_print_lock("  ", l_lck->id, l_lck->extra, l_lck->flags,
+			 l_lck->file, l_lck->line, "\n");
     }
 }
 
@@ -1002,7 +1009,8 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 #endif
 }
 
-void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags)
+void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
+			   char *file, unsigned int line)
 {
     erts_lc_locked_locks_t *l_lcks;
     erts_lc_locked_lock_t *l_lck;
@@ -1014,7 +1022,7 @@ void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags)
 	return;
 
     l_lcks = make_my_locked_locks();
-    l_lck = locked ? new_locked_lock(lck, op_flags) : NULL;
+    l_lck = locked ? new_locked_lock(lck, op_flags, file, line) : NULL;
 
     if (!l_lcks->locked.last) {
 	ASSERT(!l_lcks->locked.first);
@@ -1055,13 +1063,14 @@ void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags)
 
 }
 
-void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
+void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags,
+			      char *file, unsigned int line)
 {
     erts_lc_locked_locks_t *l_lcks = make_my_locked_locks();
     erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
     if (!find_lock(&l_lck, lck))
 	required_not_locked(l_lcks, lck);
-    l_lck = new_locked_lock(lck, op_flags);
+    l_lck = new_locked_lock(lck, op_flags, file, line);
     if (!l_lcks->required.last) {
 	ASSERT(!l_lcks->required.first);
 	l_lck->next = l_lck->prev = NULL;
@@ -1129,7 +1138,8 @@ void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
     lc_free((void *) l_lck);
 }
 
-void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
+void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
+			char *file, unsigned int line)
 {
     erts_lc_locked_locks_t *l_lcks;
     erts_lc_locked_lock_t *l_lck;
@@ -1141,7 +1151,7 @@ void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 	return;
 
     l_lcks = make_my_locked_locks();
-    l_lck = new_locked_lock(lck, op_flags);
+    l_lck = new_locked_lock(lck, op_flags, file, line);
 
     if (!l_lcks->locked.last) {
 	ASSERT(!l_lcks->locked.first);
@@ -1232,15 +1242,15 @@ erts_lc_trylock_force_busy(erts_lc_lock_t *lck)
 }
 
 void
-erts_lc_trylock(int locked, erts_lc_lock_t *lck)
+erts_lc_trylock_x(int locked, erts_lc_lock_t *lck, char *file, unsigned int line)
 {
-    erts_lc_trylock_flg(locked, lck, 0);
+    erts_lc_trylock_flg_x(locked, lck, 0, file, line);
 }
 
 void
-erts_lc_lock(erts_lc_lock_t *lck)
+erts_lc_lock_x(erts_lc_lock_t *lck, char *file, unsigned int line)
 {
-    erts_lc_lock_flg(lck, 0);
+    erts_lc_lock_flg_x(lck, 0, file, line);
 }
 
 void
@@ -1254,9 +1264,9 @@ void erts_lc_might_unlock(erts_lc_lock_t *lck)
     erts_lc_might_unlock_flg(lck, 0);
 }
 
-void erts_lc_require_lock(erts_lc_lock_t *lck)
+void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line)
 {
-    erts_lc_require_lock_flg(lck, 0);
+    erts_lc_require_lock_flg(lck, 0, file, line);
 }
 
 void erts_lc_unrequire_lock(erts_lc_lock_t *lck)
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 068340abe7..3f7f417e61 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -35,6 +35,11 @@
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 
+#ifndef ERTS_ENABLE_LOCK_POSITION
+/* Enable in order for _x variants of mtx functions to be used. */
+#define ERTS_ENABLE_LOCK_POSITION 1
+#endif
+
 typedef struct {
     int inited;
     Sint16 id;
@@ -79,13 +84,16 @@ void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len);
 void erts_lc_have_lock_ids(int *resv, int *ids, int len);
 void erts_lc_check_no_locked_of_type(Uint16 flags);
 int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags);
-void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags);
-void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
+void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
+			   char *file, unsigned int line);
+void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
+			char *file, unsigned int line);
 void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
 void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
 int erts_lc_trylock_force_busy(erts_lc_lock_t *lck);
-void erts_lc_trylock(int locked, erts_lc_lock_t *lck);
-void erts_lc_lock(erts_lc_lock_t *lck);
+void erts_lc_trylock_x(int locked, erts_lc_lock_t *lck,
+		     char* file, unsigned int line);
+void erts_lc_lock_x(erts_lc_lock_t *lck, char* file, unsigned int line);
 void erts_lc_unlock(erts_lc_lock_t *lck);
 void erts_lc_might_unlock(erts_lc_lock_t *lck);
 void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags);
@@ -96,10 +104,11 @@ int erts_lc_assert_failed(char *file, int line, char *assertion);
 void erts_lc_set_thread_name(char *thread_name);
 void erts_lc_pll(void);
 
-void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
+void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags,
+			      char *file, unsigned int line);
 void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
 
-void erts_lc_require_lock(erts_lc_lock_t *lck);
+void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line);
 void erts_lc_unrequire_lock(erts_lc_lock_t *lck);
 
 int erts_lc_is_emu_thr(void);
@@ -116,4 +125,9 @@ int erts_lc_is_emu_thr(void);
 #define ERTS_LC_ASSERT(A) ((void) 1)
 #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
 
+#define erts_lc_lock(lck) erts_lc_lock_x(lck,__FILE__,__LINE__)
+#define erts_lc_trylock(res,lck) erts_lc_trylock_x(res,lck,__FILE__,__LINE__)
+#define erts_lc_lock_flg(lck) erts_lc_lock_flg_x(lck,__FILE__,__LINE__)
+#define erts_lc_trylock_flg(res,lck) erts_lc_trylock_flg_x(res,lck,__FILE__,__LINE__)
+
 #endif /* #ifndef ERTS_LOCK_CHECK_H__ */
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index a4fc91b510..75f7cd028b 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -61,8 +61,14 @@
 #define ERTS_LOCK_COUNT_H__
 
 #ifdef  ERTS_ENABLE_LOCK_COUNT
+#ifndef ERTS_ENABLE_LOCK_POSITION
+/* Enable in order for _x variants of mtx functions to be used. */
+#define ERTS_ENABLE_LOCK_POSITION 1
+#endif
+
 #include "ethread.h"
 
+
 #define ERTS_LCNT_MAX_LOCK_LOCATIONS (10)
 
 #define ERTS_LCNT_LT_SPINLOCK   (((Uint16) 1) << 0)
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 123253a057..1d30465ec9 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -185,11 +185,13 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
     ptsp->taskq.in.last = NULL;
     erts_smp_atomic32_init_nob(&ptsp->flags, 0);
 #ifdef ERTS_SMP
+    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id,
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
-	lock_str = NULL;
+		    (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
+#else
+		    1
 #endif
-    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id);
+		    );
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index dcb9251d0d..cf20ed5acf 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -2009,12 +2009,6 @@ erts_get_runq_current(ErtsSchedulerData *esdp)
 #endif
 }
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-
-#define erts_smp_runq_lock(rq)	erts_smp_mtx_lock_x(&(rq)->mtx, __FILE__, __LINE__)
-
-#else
-
 ERTS_GLB_INLINE void
 erts_smp_runq_lock(ErtsRunQueue *rq)
 {
@@ -2023,6 +2017,10 @@ erts_smp_runq_lock(ErtsRunQueue *rq)
 #endif
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+
+#define erts_smp_runq_lock(rq)	erts_smp_mtx_lock_x(&(rq)->mtx, __FILE__, __LINE__)
+
 #endif
 
 ERTS_GLB_INLINE int
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index bb17593297..82cc68222d 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -117,7 +117,7 @@ erts_init_proc_lock(int cpus)
     for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_mtx_init_x(&erts_pix_locks[i].u.mtx,
-			"pix_lock", make_small(i));
+			"pix_lock", make_small(i), 1);
 #else
 	erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
 #endif
@@ -901,7 +901,7 @@ erts_pid2proc_opt(Process *c_p,
 		busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
-		erts_proc_lc_trylock(proc, need_locks, !busy);
+		erts_proc_lc_trylock(proc, need_locks, !busy, __FILE__,__LINE__);
 #endif
 #ifdef ERTS_PROC_LOCK_DEBUG
 		if (!busy)
@@ -1013,25 +1013,33 @@ erts_proc_lock_init(Process *p)
     for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
 	p->lock.queue[i] = NULL;
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1);
+    erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1,__FILE__,__LINE__);
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id);
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    int do_lock_count = 1;
+#else
+    int do_lock_count = 0;
+#endif
+
+    erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id, do_lock_count);
     ethr_mutex_lock(&p->lock.main.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.main.lc);
 #endif
-    erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id);
+    erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id, do_lock_count);
     ethr_mutex_lock(&p->lock.link.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.link.lc);
 #endif
-    erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id);
+    erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id, do_lock_count);
     ethr_mutex_lock(&p->lock.msgq.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.msgq.lc);
 #endif
-    erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id);
+    erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id,
+		    do_lock_count);
     ethr_mutex_lock(&p->lock.status.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.status.lc);
@@ -1210,50 +1218,51 @@ void erts_lcnt_enable_proc_lock_count(int enable)
 #if ERTS_PROC_LOCK_OWN_IMPL
 
 void
-erts_proc_lc_lock(Process *p, ErtsProcLocks locks)
+erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line)
 {
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
-	erts_lc_lock(&lck);
+	erts_lc_lock_x(&lck,file,line);
     }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	lck.id = lc_id.proc_lock_link;
-	erts_lc_lock(&lck);
+	erts_lc_lock_x(&lck,file,line);
     }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
-	erts_lc_lock(&lck);
+	erts_lc_lock_x(&lck,file,line);
     }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
-	erts_lc_lock(&lck);
+	erts_lc_lock_x(&lck,file,line);
     }
 }
 
 void
-erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked)
+erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
+		     char* file, unsigned int line)
 {
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
-	erts_lc_trylock(locked, &lck);
+	erts_lc_trylock_x(locked, &lck, file, line);
     }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	lck.id = lc_id.proc_lock_link;
-	erts_lc_trylock(locked, &lck);
+	erts_lc_trylock_x(locked, &lck, file, line);
     }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
-	erts_lc_trylock(locked, &lck);
+	erts_lc_trylock_x(locked, &lck, file, line);
     }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
-	erts_lc_trylock(locked, &lck);
+	erts_lc_trylock_x(locked, &lck, file, line);
     }
 }
 
@@ -1319,7 +1328,8 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 }
 
 void
-erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks)
+erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
+			  unsigned int line)
 {
 #if ERTS_PROC_LOCK_OWN_IMPL
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
@@ -1327,29 +1337,29 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks)
 					   ERTS_LC_FLG_LT_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
-	erts_lc_require_lock(&lck);
+	erts_lc_require_lock(&lck, file, line);
     }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	lck.id = lc_id.proc_lock_link;
-	erts_lc_require_lock(&lck);
+	erts_lc_require_lock(&lck, file, line);
     }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
-	erts_lc_require_lock(&lck);
+	erts_lc_require_lock(&lck, file, line);
     }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
-	erts_lc_require_lock(&lck);
+	erts_lc_require_lock(&lck, file, line);
     }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     if (locks & ERTS_PROC_LOCK_MAIN)
-	erts_lc_require_lock(&p->lock.main.lc);
+	erts_lc_require_lock(&p->lock.main.lc, file, line);
     if (locks & ERTS_PROC_LOCK_LINK)
-	erts_lc_require_lock(&p->lock.link.lc);
+	erts_lc_require_lock(&p->lock.link.lc, file, line);
     if (locks & ERTS_PROC_LOCK_MSGQ)
-	erts_lc_require_lock(&p->lock.msgq.lc);
+	erts_lc_require_lock(&p->lock.msgq.lc, file, line);
     if (locks & ERTS_PROC_LOCK_STATUS)
-	erts_lc_require_lock(&p->lock.status.lc);
+	erts_lc_require_lock(&p->lock.status.lc, file, line);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 9dd503f3cb..052d992d3f 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -215,7 +215,7 @@ typedef struct erts_proc_lock_t_ {
 
 /* Lock counter implemetation */
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 #define erts_smp_proc_lock__(P,I,L) erts_smp_proc_lock_x__(P,I,L,__FILE__,__LINE__)
 #define erts_smp_proc_lock(P,L) erts_smp_proc_lock_x(P,L,__FILE__,__LINE__)
 #endif
@@ -243,8 +243,10 @@ void erts_lcnt_enable_proc_lock_count(int enable);
   erts_proc_lc_chk_no_proc_locks(__FILE__, __LINE__)
 #define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \
   erts_proc_lc_chk_only_proc_main((P))
-void erts_proc_lc_lock(Process *p, ErtsProcLocks locks);
-void erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked);
+void erts_proc_lc_lock(Process *p, ErtsProcLocks locks,
+		       char *file, unsigned int line);
+void erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
+			  char *file, unsigned int line);
 void erts_proc_lc_unlock(Process *p, ErtsProcLocks locks);
 void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks);
 void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks);
@@ -253,7 +255,8 @@ void erts_proc_lc_chk_only_proc_main(Process *p);
 void erts_proc_lc_chk_no_proc_locks(char *file, int line);
 ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p);
 int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks);
-void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks);
+void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks,
+			       char* file, unsigned int line);
 void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks);
 #else
 #define ERTS_SMP_CHK_NO_PROC_LOCKS
@@ -372,7 +375,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *);
 
 ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p,
 							  ErtsProcLocks locks);
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *,
 					    erts_pix_lock_t *,
 					    ErtsProcLocks,
@@ -482,7 +485,7 @@ busy_main:
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_smp_proc_lock_x__(Process *p,
 		     erts_pix_lock_t *pix_lck,
 		     ErtsProcLocks locks,
@@ -528,7 +531,7 @@ erts_smp_proc_lock__(Process *p,
     erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line);
 #endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_proc_lc_lock(p, locks);
+    erts_proc_lc_lock(p, locks, file, line);
 #endif
 
 #ifdef ERTS_PROC_LOCK_DEBUG
@@ -695,7 +698,7 @@ erts_smp_proc_trylock__(Process *p,
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_proc_lc_trylock(p, locks, res == 0);
+    erts_proc_lc_trylock(p, locks, res == 0, __FILE__, __LINE__);
 #endif
 
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
@@ -741,7 +744,7 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
 
 #endif /* ERTS_SMP */
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE void erts_smp_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line);
 #else
 ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks);
@@ -756,13 +759,13 @@ ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *, Sint32);
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_smp_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line)
 #else
 erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
 #endif 
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_smp_proc_lock_x__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 			 NULL,
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index ecb5525022..971d68be75 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -26,10 +26,13 @@
 #define ERL_SMP_H
 #include "erl_threads.h"
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 #define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
+#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
 #define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
+#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
 #define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
+#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
 #define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
 #define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
 #define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
@@ -131,10 +134,11 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
 ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line);
+#ifdef ERTS_ENABLE_LOCK_POSITION
+ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
+ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
 #else
+ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
 ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
 #endif
 ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
@@ -159,16 +163,18 @@ ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
 ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
 					 char *name);
 ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
+ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
+ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
 #else
+ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
+ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
 #endif
 ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
@@ -179,7 +185,7 @@ ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
 					    char *name);
 ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
 ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
 #else
 ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
@@ -192,7 +198,7 @@ ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
 					  char *name);
 ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
 ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
 #else
@@ -835,7 +841,7 @@ ERTS_GLB_INLINE void
 erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_x(mtx, name, extra);
+    erts_mtx_init_x(mtx, name, extra, 1);
 #endif
 }
 
@@ -843,7 +849,7 @@ ERTS_GLB_INLINE void
 erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_locked_x(mtx, name, extra);
+    erts_mtx_init_locked_x(mtx, name, extra, 1);
 #endif
 }
 
@@ -872,9 +878,15 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
 }
 
 ERTS_GLB_INLINE int
+#ifdef ERTS_ENABLE_LOCK_POSITION
+erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
+#else
 erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
+#endif
 {
-#ifdef ERTS_SMP
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+    return erts_mtx_trylock_x(mtx,file,line);
+#elif defined(ERTS_SMP)
     return erts_mtx_trylock(mtx);
 #else
     return 0;
@@ -884,13 +896,13 @@ erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
 
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
-erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line)
+#ifdef ERTS_ENABLE_LOCK_POSITION
+erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
 #else
 erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_mtx_lock_x(mtx, file, line);
 #elif defined(ERTS_SMP)
     erts_mtx_lock(mtx);
@@ -1020,9 +1032,15 @@ erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
 }
 
 ERTS_GLB_INLINE int
+#ifdef ERTS_ENABLE_LOCK_POSITION
+erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
+#else
 erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
+#endif
 {
-#ifdef ERTS_SMP
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
+#elif defined(ERTS_SMP)
     return erts_rwmtx_tryrlock(rwmtx);
 #else
     return 0;
@@ -1030,13 +1048,13 @@ erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
 #else
 erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_rwmtx_rlock_x(rwmtx, file, line);
 #elif defined(ERTS_SMP)
     erts_rwmtx_rlock(rwmtx);
@@ -1053,9 +1071,15 @@ erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
 
 
 ERTS_GLB_INLINE int
+#ifdef ERTS_ENABLE_LOCK_POSITION
+erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
+#else
 erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
+#endif
 {
-#ifdef ERTS_SMP
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
+#elif defined(ERTS_SMP)
     return erts_rwmtx_tryrwlock(rwmtx);
 #else
     return 0;
@@ -1063,13 +1087,13 @@ erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
 #else
 erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_rwmtx_rwlock_x(rwmtx, file, line);
 #elif defined(ERTS_SMP)
     erts_rwmtx_rwlock(rwmtx);
@@ -1171,13 +1195,13 @@ erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
 #else
 erts_smp_spin_lock(erts_smp_spinlock_t *lock)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_spin_lock_x(lock, file, line);
 #elif defined(ERTS_SMP)
     erts_spin_lock(lock);
@@ -1237,13 +1261,13 @@ erts_smp_read_unlock(erts_smp_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
 #else
 erts_smp_read_lock(erts_smp_rwlock_t *lock)
 #endif 
 {
-#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
     erts_read_lock_x(lock, file, line);
 #elif defined(ERTS_SMP)
     erts_read_lock(lock);
@@ -1263,13 +1287,13 @@ erts_smp_write_unlock(erts_smp_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
 #else
 erts_smp_write_lock(erts_smp_rwlock_t *lock)
 #endif 
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_write_lock_x(lock, file, line);
 #elif defined(ERTS_SMP)
     erts_write_lock(lock);
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 759c8f4c33..b2f68462a6 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -281,10 +281,13 @@
 #define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
 #define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
+#define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__)
 #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
+#define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
 #define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__)
+#define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
 #define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__)
 #define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__)
 #define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__)
@@ -461,18 +464,24 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res);
 ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
-ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra);
-ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
+ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
+				     int enable_lcnt);
+ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra,
+					 Uint16 opt, int enable_lcnt);
 ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
 					    char *name,
-					    Eterm extra);
+					    Eterm extra,
+					    int enable_lcnt);
 ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
-ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line);
+#ifdef ERTS_ENABLE_LOCK_POSITION
+ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file,
+				       unsigned int line);
+ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file,
+				     unsigned int line);
 #else
+ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
 ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx);
 #endif
 ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx);
@@ -496,16 +505,18 @@ ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
 ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
 					 char *name);
 ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx);
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
+ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
 ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
 ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
+ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
 #else
+ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx);
+ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
 #endif
 ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
@@ -571,7 +582,7 @@ ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
 					char *name);
 ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock);
 ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock);
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line);
 #else
 ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock);
@@ -584,7 +595,7 @@ ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock,
 				      char *name);
 ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock);
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
 ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
 #else
@@ -1549,7 +1560,7 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y)
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
+erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -1559,13 +1570,17 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
+    if (enable_lcnt)
+      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
+    else
+      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
 #endif
 #endif
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
+erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
+		    int enable_lcnt)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -1575,14 +1590,17 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
+    if (enable_lcnt)
+      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
+    else
+      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra);
 #endif
 #endif
 }
 
 
 ERTS_GLB_INLINE void
-erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
+erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -1592,7 +1610,10 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
+    if (enable_lcnt)
+      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
+    else
+      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
 #endif
     ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -1670,7 +1691,11 @@ erts_mtx_destroy(erts_mtx_t *mtx)
 }
 
 ERTS_GLB_INLINE int
+#ifdef ERTS_ENABLE_LOCK_POSITION
+erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line)
+#else
 erts_mtx_trylock(erts_mtx_t *mtx)
+#endif
 {
 #ifdef USE_THREADS
     int res;
@@ -1684,8 +1709,12 @@ erts_mtx_trylock(erts_mtx_t *mtx)
     res = ethr_mutex_trylock(&mtx->mtx);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_trylock_x(res == 0, &mtx->lc,file,line);
+#else
     erts_lc_trylock(res == 0, &mtx->lc);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock(&mtx->lcnt, res);
 #endif    
@@ -1697,7 +1726,7 @@ erts_mtx_trylock(erts_mtx_t *mtx)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line)
 #else
 erts_mtx_lock(erts_mtx_t *mtx)
@@ -1705,8 +1734,12 @@ erts_mtx_lock(erts_mtx_t *mtx)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_lock_x(&mtx->lc, file, line);
+#else
     erts_lc_lock(&mtx->lc);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock(&mtx->lcnt);
 #endif
@@ -1857,7 +1890,10 @@ erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
     erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
+    if (name && name[0] == '\0')
+      erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
+    else
+      erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
 #endif
 #endif
 }
@@ -1921,7 +1957,11 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 }
 
 ERTS_GLB_INLINE int
+#ifdef ERTS_ENABLE_LOCK_POSITION
+erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
+#else
 erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
+#endif
 {
 #ifdef USE_THREADS
     int res;
@@ -1935,8 +1975,12 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
     res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
+#else
     erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
 #endif
@@ -1948,7 +1992,7 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
 #else
 erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
@@ -1956,8 +2000,12 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
+#else
     erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
 #endif
@@ -1984,7 +2032,11 @@ erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
 
 
 ERTS_GLB_INLINE int
+#ifdef ERTS_ENABLE_LOCK_POSITION
+erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
+#else
 erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
+#endif
 {
 #ifdef USE_THREADS
     int res;
@@ -1998,8 +2050,12 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
     res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
+#else
     erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
 #endif
@@ -2011,7 +2067,7 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
 #else
 erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
@@ -2019,8 +2075,12 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
+#else
     erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
@@ -2426,7 +2486,7 @@ erts_spin_unlock(erts_spinlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line)
 #else
 erts_spin_lock(erts_spinlock_t *lock)
@@ -2434,8 +2494,12 @@ erts_spin_lock(erts_spinlock_t *lock)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_lock_x(&lock->lc,file,line);
+#else
     erts_lc_lock(&lock->lc);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock(&lock->lcnt);
 #endif
@@ -2545,7 +2609,7 @@ erts_read_unlock(erts_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
 #else
 erts_read_lock(erts_rwlock_t *lock)
@@ -2553,8 +2617,12 @@ erts_read_lock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line);
+#else
     erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
 #endif
@@ -2584,7 +2652,7 @@ erts_write_unlock(erts_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_COUNT
+#ifdef ERTS_ENABLE_LOCK_POSITION
 erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
 #else
 erts_write_lock(erts_rwlock_t *lock)
@@ -2592,8 +2660,12 @@ erts_write_lock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_POSITION
+    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
+#else
     erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
+#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index f4d905552d..c7c6e5a5d7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -247,11 +247,13 @@ static ERTS_INLINE void port_init_instr(Port *prt
     ASSERT(prt->drv_ptr && prt->lock);
     if (!prt->drv_ptr->lock) {
 	char *lock_str = "port_lock";
+	erts_mtx_init_locked_x(prt->lock, lock_str, id,
 #ifdef ERTS_ENABLE_LOCK_COUNT
-	if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
-	    lock_str = NULL;
+			       (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
+#else
+			       0
 #endif
-	erts_mtx_init_locked_x(prt->lock, lock_str, id);
+			       );
     }
 #endif
     erts_port_task_init_sched(&prt->sched, id);
@@ -7310,10 +7312,11 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 			    erts_atom_put((byte *) drv->name,
 					  sys_strlen(drv->name),
 					  ERTS_ATOM_ENC_LATIN1,
-					  1)
+					  1),
 #else
-			NIL
+			NIL,
 #endif
+			1
 	    );
     }
 #endif
-- 
cgit v1.2.3


From c743ed359f16f791dd15b58b86af7f77db4799aa Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Sun, 23 Feb 2014 17:00:41 +0100
Subject: ose: Force atleast one async thread for ose

This is needed because a file has to be opened and operated on
in the same process at all times. Using async threads guarantee this.
---
 erts/emulator/beam/erl_init.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index c17256f466..e2227e8de4 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -780,7 +780,7 @@ early_init(int *argc, char **argv) /*
 		case 'A': {
 		    /* set number of threads in thread pool */
 		    char *arg = get_arg(argv[i]+2, argv[i+1], &i);
-		    if (((erts_async_max_threads = atoi(arg)) < 0) ||
+		    if (((erts_async_max_threads = atoi(arg)) < ERTS_MIN_NO_OF_ASYNC_THREADS) ||
 			(erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) {
 			erts_fprintf(stderr,
 				     "bad number of async threads %s\n",
-- 
cgit v1.2.3


From d932131754c2bfb2e0539b6419e3d09533fe84e8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 28 Oct 2013 16:10:07 +0100
Subject: erts: configure number of write_concurrency locks

Make it possible to change the number of write_concurrency locks to use. This
is usefull to change when you for some reason want to use more/less locks
per write_concurrency ets table. eg. OSs with a limit on how many mutexes
can exist at once.
---
 erts/emulator/beam/erl_db_hash.h | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index d17bd9f693..908cec11d4 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -33,7 +33,12 @@ typedef struct hash_db_term {
     DbTerm dbterm;         /* The actual term */
 } HashDbTerm;
 
+#ifdef ERTS_DB_HASH_LOCK_CNT
+#define DB_HASH_LOCK_CNT ERTS_DB_HASH_LOCK_CNT
+#else
 #define DB_HASH_LOCK_CNT 64
+#endif
+
 typedef struct db_table_hash_fine_locks {
     union {
 	erts_smp_rwmtx_t lck;
-- 
cgit v1.2.3


From 8103fed4090e9f1457fb1705e9a4df0821f5db9b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 31 Oct 2013 14:40:33 +0100
Subject: ose,erts: Specify name for tsd keys

This simplified debugging on OSE and also limits the number of ppdata
keys that are created when beam is restarted.
---
 erts/emulator/beam/code_ix.c          | 3 ++-
 erts/emulator/beam/erl_alloc.c        | 3 ++-
 erts/emulator/beam/erl_db_util.c      | 3 ++-
 erts/emulator/beam/erl_drv_thread.c   | 2 +-
 erts/emulator/beam/erl_init.c         | 2 +-
 erts/emulator/beam/erl_lock_check.c   | 2 +-
 erts/emulator/beam/erl_lock_count.c   | 2 +-
 erts/emulator/beam/erl_process.c      | 2 +-
 erts/emulator/beam/erl_smp.h          | 7 ++++---
 erts/emulator/beam/erl_thr_progress.c | 3 ++-
 erts/emulator/beam/erl_threads.h      | 6 +++---
 erts/emulator/beam/io.c               | 6 ++++--
 12 files changed, 24 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index c66d5a2f05..4344558348 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -58,7 +58,8 @@ void erts_code_ix_init(void)
     erts_smp_atomic32_init_nob(&the_staging_code_index, 0);
     erts_smp_mtx_init(&code_write_permission_mtx, "code_write_permission");
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_tsd_key_create(&has_code_write_permission);
+    erts_tsd_key_create(&has_code_write_permission,
+			"erts_has_code_write_permission");
 #endif
     CIX_TRACE("init");
 }
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index ca7e39041b..c2bca9e54f 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -585,7 +585,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     if (ncpu < 1)
 	ncpu = 1;
 
-    erts_tsd_key_create(&erts_allctr_prelock_tsd_key);
+    erts_tsd_key_create(&erts_allctr_prelock_tsd_key,
+			"erts_allctr_prelock_tsd_key");
 
     erts_sys_alloc_init();
     erts_init_utils_mem();
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 3986ccd4d3..3927615e04 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -483,7 +483,8 @@ void
 match_pseudo_process_init(void)
 {
 #ifdef ERTS_SMP
-    erts_smp_tsd_key_create(&match_pseudo_process_key);
+    erts_smp_tsd_key_create(&match_pseudo_process_key,
+			    "erts_match_pseudo_process_key");
     erts_smp_install_exit_handler(destroy_match_pseudo_process);
 #else
     match_pseudo_process = create_match_pseudo_process();
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index beaeed03a2..147249f751 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -121,7 +121,7 @@ void erl_drv_thr_init(void)
 {
     int i;
 #ifdef USE_THREADS
-    int res = ethr_tsd_key_create(&tid_key);
+    int res = ethr_tsd_key_create(&tid_key,"erts_tid_key");
     if (res == 0)
 	res = ethr_install_exit_handler(thread_exit_handler);
     if (res != 0)
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index e2227e8de4..6b190326aa 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -709,7 +709,7 @@ early_init(int *argc, char **argv) /*
 #endif
 #ifdef ERTS_SMP
     erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
-    erts_tsd_key_create(&erts_is_crash_dumping_key);
+    erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key");
 #else
     erts_writing_erl_crash_dump = 0;
 #endif
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 706d284a00..7e3a90779d 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1332,7 +1332,7 @@ erts_lc_init(void)
     if (ethr_spinlock_init(&free_blocks_lock) != 0)
 	lc_abort();
 
-    erts_tsd_key_create(&locks_key);
+    erts_tsd_key_create(&locks_key,"erts_lock_check_key");
 }
 
 void
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 5f75b0ac0b..6f44bf097b 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -236,7 +236,7 @@ void erts_lcnt_init() {
     /* init tsd */    
     lcnt_n_thr = 0;
 
-    ethr_tsd_key_create(&lcnt_thr_data_key);
+    ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data");
 
     lcnt_lock();
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7e5d88cb24..929dbfd16b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -512,7 +512,7 @@ void
 erts_pre_init_process(void)
 {
 #ifdef USE_THREADS
-    erts_tsd_key_create(&sched_data_key);
+    erts_tsd_key_create(&sched_data_key, "erts_sched_data_key");
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 971d68be75..c38ef47d87 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -208,7 +208,8 @@ ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp);
+ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
+					     char *keyname);
 ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
 ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
 ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);
@@ -1323,10 +1324,10 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp)
+erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
 {
 #ifdef ERTS_SMP
-    erts_tsd_key_create(keyp);
+    erts_tsd_key_create(keyp,keyname);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index cf5e3dc012..545a0343d0 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -417,7 +417,8 @@ void
 erts_thr_progress_pre_init(void)
 {
     intrnl = NULL;
-    erts_tsd_key_create(&erts_thr_prgr_data_key__);
+    erts_tsd_key_create(&erts_thr_prgr_data_key__,
+			"erts_thr_prgr_data_key");
     init_nob(&erts_thr_prgr__.current, ERTS_THR_PRGR_VAL_FIRST);
 }
 
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index b2f68462a6..80026104db 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -605,7 +605,7 @@ ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp);
+ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname);
 ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
 ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
 ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
@@ -2707,10 +2707,10 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-erts_tsd_key_create(erts_tsd_key_t *keyp)
+erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname)
 {
 #ifdef USE_THREADS
-    int res = ethr_tsd_key_create(keyp);
+    int res = ethr_tsd_key_create(keyp, keyname);
     if (res)
 	erts_thr_fatal_error(res, "create thread specific data key");
 #endif
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c7c6e5a5d7..09681f167d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2747,8 +2747,10 @@ void erts_init_io(int port_tab_size,
 			    &drv_list_rwmtx_opts,
 			    "driver_list");
     driver_list = NULL;
-    erts_smp_tsd_key_create(&driver_list_lock_status_key);
-    erts_smp_tsd_key_create(&driver_list_last_error_key);
+    erts_smp_tsd_key_create(&driver_list_lock_status_key,
+			    "erts_driver_list_lock_status_key");
+    erts_smp_tsd_key_create(&driver_list_last_error_key,
+			    "erts_driver_list_last_error_key");
 
     erts_ptab_init_table(&erts_port,
 			 ERTS_ALC_T_PORT_TABLE,
-- 
cgit v1.2.3


From c64851d619fb916362abc8db9c28534eff39f53c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Fri, 10 Jan 2014 17:17:58 +0100
Subject: ose: Rewrite resolve_signal API for ose drivers

This new API has less impact on the check_io code and
also removes the callback from ErlDrvEntry. The downside
is that you have to give the resolve function when creating
each event.

Also the mode if the resolve was removed as this mimics the
win32 code and decreases complexity.
---
 erts/emulator/beam/erl_driver.h | 21 ++++++---------------
 erts/emulator/beam/global.h     |  3 ---
 erts/emulator/beam/io.c         |  3 ---
 3 files changed, 6 insertions(+), 21 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 9ede2982de..a15b0e17f5 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -271,14 +271,6 @@ typedef struct ErlDrvCond_ ErlDrvCond;
 typedef struct ErlDrvRWLock_ ErlDrvRWLock;
 typedef int ErlDrvTSDKey;
 
-/*
- * Potential OSE signals
- */
-#ifdef __OSE__
-typedef union SIGNAL OseSignal;
-#endif
-
-
 /*
  * 
  */
@@ -353,9 +345,6 @@ typedef struct erl_drv_entry {
     	                        /* Called on behalf of driver_select when
 				   it is safe to release 'event'. A typical
 				   unix driver would call close(event) */
-#ifdef __OSE__
-    int (*resolve_signal)(OseSignal* sig, int* mode);
-#endif
     /* When adding entries here, dont forget to pad in obsolete/driver.h */
 } ErlDrvEntry;
 
@@ -691,11 +680,13 @@ EXTERN int erl_drv_putenv(char *key, char *value);
 EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 
 #ifdef __OSE__
-EXTERN OseSignal *erl_drv_ose_get_output_signal(ErlDrvEvent ev);
-EXTERN OseSignal *erl_drv_ose_get_input_signal(ErlDrvEvent ev);
-EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig,int id);
+typedef ErlDrvUInt ErlDrvOseEventId;
+EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev);
+EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig,ErlDrvOseEventId id,
+					   ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig));
 EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev);
-EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig,int *id);
+EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig,
+				    ErlDrvOseEventId *id);
 #endif
 
 #endif /* !ERL_DRIVER_TYPES_ONLY */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 5a97ac5892..8fcb95d0e2 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -160,9 +160,6 @@ struct erts_driver_t_ {
     void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data); /* Might be NULL */ 
     void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
     void (*stop_select)(ErlDrvEvent event, void*); /* Might be NULL */
-#ifdef __OSE__
-    int (*resolve_signal)(OseSignal* sig, int* mode);
-#endif
 };
 
 extern erts_driver_t *driver_list;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 09681f167d..cd5060ebb3 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7342,9 +7342,6 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 	drv->stop_select = de->stop_select;
     else
 	drv->stop_select = no_stop_select_callback;
-#ifdef __OSE__
-    drv->resolve_signal = de->resolve_signal;
-#endif
 
     if (!de->init)
 	return 0;
-- 
cgit v1.2.3


From ff463a10f37ca74c0b6a8c0e24ce919e56b12bb8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Wed, 29 Jan 2014 14:38:56 +0100
Subject: ose: Add fair scheduling yields

This is needed on OSs that do not do round robin scheduling
of threads.
---
 erts/emulator/beam/erl_process.c | 10 ++++++++--
 erts/emulator/beam/erl_process.h |  7 +++++++
 erts/emulator/beam/erl_trace.c   |  2 ++
 3 files changed, 17 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 929dbfd16b..48e9b63462 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2620,6 +2620,8 @@ aux_thread(void *unused)
 		erts_thr_progress_active(NULL, thr_prgr_active = 0);
 	    erts_thr_progress_prepare_wait(NULL);
 
+	    ERTS_SCHED_FAIR_YIELD();
+
 	    flgs = sched_spin_wait(ssi, 0);
 
 	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
@@ -2733,6 +2735,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    erts_thr_progress_prepare_wait(esdp);
 		}
 
+		ERTS_SCHED_FAIR_YIELD();
+
 		flgs = sched_spin_wait(ssi, spincount);
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
@@ -2798,7 +2802,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 #ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
 	sys_poll_try:
 	while(!prepare_for_sys_schedule()) {
-		delay(1);
+	  ERTS_SCHED_FAIR_YIELD();
 	}
 #endif
 
@@ -8069,10 +8073,12 @@ Process *schedule(Process *p, int calls)
 	    erts_aint32_t aux_work;
 	    int leader_update = erts_thr_progress_update(esdp);
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
-	    if (aux_work | leader_update) {
+	    if (aux_work | leader_update | ERTS_SCHED_FAIR) {
 		erts_smp_runq_unlock(rq);
 		if (leader_update)
 		    erts_thr_progress_leader_update(esdp);
+		else if (ERTS_SCHED_FAIR)
+		  ERTS_SCHED_FAIR_YIELD();
 		if (aux_work)
 		    handle_aux_work(&esdp->aux_work_data, aux_work, 0);
 		erts_smp_runq_lock(rq);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index cf20ed5acf..a86ac65aa2 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -607,6 +607,13 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
 extern ErtsSchedulerData *erts_scheduler_data;
 #endif
 
+#ifdef ERTS_SCHED_FAIR
+#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
+#else
+#define ERTS_SCHED_FAIR 0
+#define ERTS_SCHED_FAIR_YIELD()
+#endif
+
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
 #endif
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 62a0b8ccea..43e7435d2a 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3309,6 +3309,8 @@ sys_msg_dispatcher_func(void *unused)
 	    if (erts_thr_progress_update(NULL))
 		erts_thr_progress_leader_update(NULL);
 
+	    ERTS_SCHED_FAIR_YIELD();
+
 #ifdef DEBUG_PRINTOUTS
 	    print_msg_type(smqp);
 #endif
-- 
cgit v1.2.3


From 3b0eb33f899f361d5006824782e1ef1d16f57e5c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 10 Feb 2014 18:48:44 +0100
Subject: ose: Cleanup POLL_SCHED_1 code

Now schedulers 2..N make sure to wake sched 1 if they
find that all io has been consumed and sched 1 is
sleeping. Before sched 1 was spinning in sys_schedule
waiting for sched 2..N to finish consuming io jobs
---
 erts/emulator/beam/erl_process.c | 60 ++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 36 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 48e9b63462..a4241c6d43 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -472,7 +472,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 
 #ifdef ERTS_SMP
 static void handle_pending_exiters(ErtsProcList *);
-
+static void wake_scheduler(ErtsRunQueue *rq);
 #endif
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
@@ -2307,14 +2307,24 @@ try_set_sys_scheduling(void)
 #endif
 
 static ERTS_INLINE int
-prepare_for_sys_schedule(void)
+prepare_for_sys_schedule(ErtsSchedulerData *esdp)
 {
 #ifdef ERTS_SMP
     while (!erts_port_task_have_outstanding_io_tasks()
 	   && try_set_sys_scheduling()) {
-	if (!erts_port_task_have_outstanding_io_tasks())
-	    return 1;
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+      if (esdp->no != 1) {
+	/* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used
+	   then we make sure to wake scheduler 1 */
+	ErtsRunQueue *rq = ERTS_RUNQ_IX(0);
 	clear_sys_scheduling();
+	wake_scheduler(rq);
+	return 0;
+      }
+#endif
+      if (!erts_port_task_have_outstanding_io_tasks())
+	return 1;
+      clear_sys_scheduling();
     }
     return 0;
 #else
@@ -2689,11 +2699,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
      * be waiting in erl_sys_schedule()
      */
 
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-    if (esdp->no != 1) {
-#else
-    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule()) {
-#endif
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp)) {
 
 	sched_waiting(esdp->no, rq);
 
@@ -2701,11 +2707,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	spincount = sched_busy_wait.tse;
 
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-	ASSERT(esdp->no != 1);
-#else
     tse_wait:
-#endif
 
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working)
 	    sched_wall_time_change(esdp, thr_prgr_active);
@@ -2792,6 +2794,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 #ifdef ERTS_DIRTY_SCHEDULERS
 	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
 #endif
+
+#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
+	ASSERT(esdp->no == 1);
+#endif
 	sched_waiting_sys(esdp->no, rq);
 
 
@@ -2799,12 +2805,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	ASSERT(working);
 	sched_wall_time_change(esdp, working = 0);
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-	sys_poll_try:
-	while(!prepare_for_sys_schedule()) {
-	  ERTS_SCHED_FAIR_YIELD();
-	}
-#endif
 
 	spincount = sched_busy_wait.sys_schedule;
 	if (spincount == 0)
@@ -2863,13 +2863,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		 * Got to check that we still got I/O tasks; otherwise
 		 * we have to continue checking for I/O...
 		 */
-		if (!prepare_for_sys_schedule()) {
+		if (!prepare_for_sys_schedule(esdp)) {
 		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-		    goto sys_poll_try;
-#else
 		    goto tse_wait;
-#endif
 		}
 	    }
 #endif
@@ -2889,7 +2885,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	     * Got to check that we still got I/O tasks; otherwise
 	     * we have to wait in erl_sys_schedule() after all...
 	     */
-	    if (!prepare_for_sys_schedule()) {
+	    if (!prepare_for_sys_schedule(esdp)) {
 		/*
 		 * Not allowed to wait in erl_sys_schedule;
 		 * do tse wait instead...
@@ -2897,11 +2893,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		sched_change_waiting_sys_to_waiting(esdp->no, rq);
 		erts_smp_runq_unlock(rq);
 		spincount = 0;
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-		goto sys_poll_try;
-#else
 		goto tse_wait;
-#endif
 	    }
 	}
 #endif
@@ -6883,7 +6875,6 @@ erts_start_schedulers(void)
 	res = ethr_thr_create(&esdp->tid, sched_thread_func,(void*)esdp,&opts);
 
 	if (res != 0) {
-           //actual--;
            break;
 	}
 
@@ -8140,12 +8131,9 @@ Process *schedule(Process *p, int calls)
 
 	    goto check_activities_to_run;
 	}
-	else if (
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-		 esdp->no == 1 &&
-#endif
-		 !ERTS_SCHEDULER_IS_DIRTY(esdp) &&
-		 (fcalls > input_reductions && prepare_for_sys_schedule())) {
+	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
+		 (fcalls > input_reductions &&
+		  prepare_for_sys_schedule(esdp))) {
 	    /*
 	     * Schedule system-level activities.
 	     */
-- 
cgit v1.2.3


From a35d0f5f82f8152f1b953eda039807a7d4f4e9b9 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Sun, 23 Feb 2014 18:26:38 +0100
Subject: ose: Thread priorities configurable from lmconf

The pattern used for getting the priority from the lmconf
is based on the name of the process created. The pattern is:

ERTS_%%PROCESS_NAME%%_PRIO

with the %%PROCESS_NAME%% replaced by the prefix of the process
the priority applies to. eg:

ERTS_SCHEDULER_PRIO=24

applies to processes with name SCHEDULER_1, SCHEDULER_2 etc.
---
 erts/emulator/beam/erl_async.c   | 14 ++++++++++----
 erts/emulator/beam/erl_process.c | 38 ++++++++++++++++++++++++--------------
 erts/emulator/beam/erl_trace.c   |  5 ++---
 3 files changed, 36 insertions(+), 21 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index afb5bc302f..b3dc327704 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -166,6 +166,7 @@ async_ready_q(Uint sched_id)
 
 #endif
 
+
 void
 erts_init_async(void)
 {
@@ -226,18 +227,23 @@ erts_init_async(void)
 	thr_opts.suggested_stack_size
 	    = erts_async_thread_suggested_stack_size;
 
-#ifdef ETHR_HAVE_THREAD_NICENESS
-	thr_opts.prio += 2;
-#endif
 #ifdef ETHR_HAVE_THREAD_NAMES
-	thr_opts.name = "async_thread";
+	thr_opts.name = malloc(sizeof(char)*(strlen("async_XXXX")+1));
 #endif
 
 	for (i = 0; i < erts_async_max_threads; i++) {
 	    ErtsAsyncQ *aq = async_q(i);
+
+#ifdef ETHR_HAVE_THREAD_NAMES
+            sprintf(thr_opts.name, "async_%d", i+1);
+#endif
+
 	    erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts);
 	}
 
+#ifdef ETHR_HAVE_THREAD_NAMES
+	free(thr_opts.name);
+#endif
 	/* Wait for async threads to initialize... */
 
 	erts_mtx_lock(&async->init.data.mtx);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a4241c6d43..83856ab983 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6830,9 +6830,16 @@ erts_start_schedulers(void)
 
     opts.detached = 1;
 
+#ifdef ETHR_HAVE_THREAD_NAMES
+    opts.name = malloc(80);
+#endif
+
 #ifdef ERTS_SMP
     if (erts_runq_supervision_interval) {
 	opts.suggested_stack_size = 16;
+#ifdef ETHR_HAVE_THREAD_NAMES
+        sprintf(opts.name, "runq_supervisor");
+#endif
 	erts_atomic_init_nob(&runq_supervisor_sleeping, 0);
 	if (0 != ethr_event_init(&runq_supervision_event))
 	    erl_exit(1, "Failed to create run-queue supervision event\n");
@@ -6854,17 +6861,13 @@ erts_start_schedulers(void)
 	res = ENOTSUP;
     }
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-    opts.name = malloc(sizeof(char)*(strlen("scheduler_XXXX")+1));
-#endif
-
     for (actual = 0; actual < wanted; actual++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual);
 
 	ASSERT(actual == esdp->no - 1);
 
 #ifdef ETHR_HAVE_THREAD_NAMES
-	sprintf(opts.name,"scheduler_%d", actual+1);
+	sprintf(opts.name, "scheduler_%d", actual + 1);
 #endif
 
 #ifdef __OSE__
@@ -6872,14 +6875,13 @@ erts_start_schedulers(void)
 	opts.coreNo = (actual+1) % ose_num_cpus();
 #endif
 
-	res = ethr_thr_create(&esdp->tid, sched_thread_func,(void*)esdp,&opts);
+	res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts);
 
 	if (res != 0) {
            break;
 	}
-
     }
-    
+
     erts_no_schedulers = actual;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -6888,12 +6890,18 @@ erts_start_schedulers(void)
 	int ix;
 	for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+#ifdef ETHR_HAVE_THREAD_NAMES
+            sprintf(opts.name,"dirty_cpu_scheduler_%d", ix + 1);
+#endif
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
 		erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix);
 	}
 	for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
+#ifdef ETHR_HAVE_THREAD_NAMES
+            sprintf(opts.name,"dirty_io_scheduler_%d", ix + 1);
+#endif
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
 		erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix);
@@ -6903,16 +6911,14 @@ erts_start_schedulers(void)
 #endif
 
     ERTS_THR_MEMORY_BARRIER;
+
 #ifdef ETHR_HAVE_THREAD_NAMES
-    free(opts.name);
-    opts.name   = "aux_thread";
+    sprintf(opts.name, "aux");
 #endif
+
 #ifdef __OSE__
     opts.coreNo = 0;
-#endif
-#ifdef ETHR_HAVE_THREAD_NICENESS
-    opts.prio++;
-#endif
+#endif /* __OSE__ */
 
     res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
     if (res != 0)
@@ -6933,6 +6939,10 @@ erts_start_schedulers(void)
 		      actual, actual == 1 ? " was" : "s were");
 	erts_send_error_to_logger_nogl(dsbufp);
     }
+
+#ifdef ETHR_HAVE_THREAD_NAMES
+    free(opts.name);
+#endif
 }
 
 #endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 43e7435d2a..6978a5f11a 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3478,12 +3478,11 @@ init_sys_msg_dispatcher(void)
     sys_message_queue_end = NULL;
     erts_smp_cnd_init(&smq_cnd);
     erts_smp_mtx_init(&smq_mtx, "sys_msg_q");
+
 #ifdef ETHR_HAVE_THREAD_NAMES
     thr_opts.name = "sys_msg_dispatcher";
 #endif
-#ifdef ETHR_HAVE_THREAD_NICENESS
-    thr_opts.prio++;
-#endif
+
     erts_smp_thr_create(&sys_msg_dispatcher_tid,
 			sys_msg_dispatcher_func,
 			NULL,
-- 
cgit v1.2.3


From 2c20bdf2e146dbe27554be7f02982b503ecf08d5 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Wed, 19 Feb 2014 15:12:29 +0100
Subject: erts: Fix sys_msg_dispatcher assert

Schedulers is too restrictive. Managed threads should be
able to clean this up.
---
 erts/emulator/beam/erl_ptab.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index fa5482b841..eabf016081 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -756,7 +756,8 @@ erts_ptab_delete_element(ErtsPTab *ptab,
 
     pix = erts_ptab_id2pix(ptab, ptab_el->id);
 
-    ASSERT(erts_get_scheduler_id()); /* *Need* to be a scheduler */
+    /* *Need* to be an managed thread */
+    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
 
     erts_ptab_rlock(ptab);
     maybe_save = ptab->list.data.deleted.end != NULL;
-- 
cgit v1.2.3


From 4a6850e522b91eb009ddd0ed9d9f542f1baf1bee Mon Sep 17 00:00:00 2001
From: Jonas Karlsson <jonas.karlsson@enea.com>
Date: Fri, 21 Feb 2014 14:01:38 +0100
Subject: ose: Updating event and signal API for OSE

---
 erts/emulator/beam/erl_driver.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index a15b0e17f5..5517c26ba4 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -682,11 +682,11 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 #ifdef __OSE__
 typedef ErlDrvUInt ErlDrvOseEventId;
 EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev);
-EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig,ErlDrvOseEventId id,
-					   ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig));
+EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig, ErlDrvOseEventId handle,
+					   ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig), void *extra);
 EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev);
 EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig,
-				    ErlDrvOseEventId *id);
+                  ErlDrvOseEventId *handle, void **extra);
 #endif
 
 #endif /* !ERL_DRIVER_TYPES_ONLY */
-- 
cgit v1.2.3


From eb53a3f0b7a7d6c4d0a877fe71bc0b0ca11d1597 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 20 Feb 2014 14:34:41 +0100
Subject: erts: Fix heap overwrite by hipe "trap frames" when GC is disabled

by trapping BIFs like term_to_binary and binary_to_term.
---
 erts/emulator/beam/external.c | 65 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9fb2dbd8bf..2ca52c8025 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1068,7 +1068,7 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
             BIF_RET(res);
     }
 }
-    
+
 BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
 {
     Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL);
@@ -4459,3 +4459,66 @@ error:
 #undef SKIP2
 #undef CHKSIZE
 }
+
+
+#ifdef HIPE
+BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1);
+BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2);
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1);
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2);
+
+/* Hipe wrappers used by native code for BIFs that disable GC while trapping.
+ *
+ * Problem:
+ * When native code calls a BIF that traps, hipe_mode_switch will push a
+ * "trap frame" on the Erlang stack in order to find its way back from beam_emu
+ * back to native caller when finally done. If GC is disabled and stack/heap
+ * is full there is no place to push the "trap frame".
+ *
+ * Solution:
+ * We reserve space on stack for the "trap frame" here before the BIF is called.
+ * If the BIF does not trap, the space is reclaimed here before returning.
+ * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame"
+ * already is reserved and use it.
+ */
+BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1);
+    res = term_to_binary_1(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2);
+    res = term_to_binary_2(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1);
+    res = erts_internal_binary_to_term_1(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2);
+    res = erts_internal_binary_to_term_2(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+#endif /*HIPE*/
-- 
cgit v1.2.3


From 615e41b3810cecc7a30668193ce1fb0207cc42c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 27 Feb 2014 17:50:58 +0100
Subject: erts: Fix Map cmp exact equal of an empty map

The expression,

  #{} =:= M

where M was any Map, would always result in 'true'.

This commit fixes this issue by first comparing sizes for
both terms and then checking for size zero.
---
 erts/emulator/beam/utils.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 5b43d25e3c..738f793020 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2125,7 +2125,11 @@ tailrecur_ne:
 		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
 			goto not_equal;
 		    bb = map_val_rel(b,b_base);
-		    if ((sz = map_get_size((map_t*)aa)) == 0) goto pop_next;
+		    sz = map_get_size((map_t*)aa);
+
+		    if (sz != map_get_size((map_t*)bb)) goto not_equal;
+		    if (sz == 0) goto pop_next;
+
 		    aa += 2;
 		    bb += 2;
 		    sz += 1; /* increment for tuple-keys */
-- 
cgit v1.2.3


From 75937381ec52e1714cfbc5936441b15bb1240bdb Mon Sep 17 00:00:00 2001
From: Cobus Carstens <cobus.carstens@gmail.com>
Date: Wed, 5 Mar 2014 15:30:48 +0200
Subject: Fix comment that differs from code

The comment in the code state that the tree is traversed to the left,
when in fact it is traversed to the right.
---
 erts/emulator/beam/erl_db_tree.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 25029ba90f..a62a83a928 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -485,7 +485,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret)
 	*ret = am_EOT;
 	return DB_ERROR_NONE;
     }
-    /* Walk down to the tree to the left */
+    /* Walk down the tree to the left */
     if ((stack = get_static_stack(tb)) != NULL) {
 	stack->pos = stack->slot = 0;
     }
@@ -531,7 +531,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret)
 	*ret = am_EOT;
 	return DB_ERROR_NONE;
     }
-    /* Walk down to the tree to the left */
+    /* Walk down the tree to the right */
     if ((stack = get_static_stack(tb)) != NULL) {
 	stack->pos = stack->slot = 0;
     }    
-- 
cgit v1.2.3


From 38bd20f4f58e8025bd3ffc718cb7e40a4bde6396 Mon Sep 17 00:00:00 2001
From: Rick Reed <rr@whatsapp.com>
Date: Fri, 16 Mar 2012 15:46:39 -0700
Subject: Demote rare debug slogan of message discarding to debug build

---
 erts/emulator/beam/bif.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 06a1230ca0..03c436210f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1869,6 +1869,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
     } else if (is_external_pid(to)) {
 	dep = external_pid_dist_entry(to);
 	if(dep == erts_this_dist_entry) {
+#if DEBUG
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	    erts_dsprintf(dsbufp,
 			  "Discarding message %T from %T to %T in an old "
@@ -1879,6 +1880,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 			  external_pid_creation(to),
 			  erts_this_node->creation);
 	    erts_send_error_to_logger(p->group_leader, dsbufp);
+#endif
 	    return 0;
 	}
 	return remote_send(p, dep, to, to, msg, suspend);
@@ -1907,6 +1909,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
     } else if (is_external_port(to)
 	       && (external_port_dist_entry(to)
 		   == erts_this_dist_entry)) {
+#if DEBUG
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	erts_dsprintf(dsbufp,
 		      "Discarding message %T from %T to %T in an old "
@@ -1917,6 +1920,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 		      external_port_creation(to),
 		      erts_this_node->creation);
 	erts_send_error_to_logger(p->group_leader, dsbufp);
+#endif
 	return 0;
     } else if (is_internal_port(to)) {
 	int ret_val;
-- 
cgit v1.2.3


From 345af4a0c8d68b9369c3556fa6d911854c123d3f Mon Sep 17 00:00:00 2001
From: Rick Reed <rr@whatsapp.com>
Date: Wed, 13 Feb 2013 08:48:18 -0800
Subject: Add run queue index to process dump info

---
 erts/emulator/beam/break.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 7d4f52ee23..08265b590d 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -256,6 +256,7 @@ print_process_info(int to, void *to_arg, Process *p)
 		   p->current[1],
 		   p->current[2]);
     }
+    erts_print(to, to_arg, "Run queue: %d\n", erts_get_runq_proc(p)->ix);
 
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
     approx_started = (time_t) p->approx_started;
-- 
cgit v1.2.3


From 5d5f9c1857029d7e8e1de141e29d20dd3de929be Mon Sep 17 00:00:00 2001
From: Rick Reed <rr@whatsapp.com>
Date: Wed, 13 Feb 2013 08:49:20 -0800
Subject: Add thread index to allocator enomem dump slogan

---
 erts/emulator/beam/erl_alloc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 05ac24e04d..90cd227fae 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1873,8 +1873,8 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
 	size = va_arg(argp, Uint);
 	va_end(argp);
 	erl_exit(1,
-		 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
-		 allctr_str, op, size, t_str);
+		 "%s: Cannot %s %lu bytes of memory (of type \"%s\", thread %d).\n",
+		 allctr_str, op, size, t_str, ERTS_ALC_GET_THR_IX());
 	break;
     }
     case ERTS_ALC_E_NOALLCTR:
-- 
cgit v1.2.3


From 8b4b4a026041bfbe5a8b4e2535c5fd0804b18103 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Sun, 2 Mar 2014 16:48:06 -0500
Subject: enable enif_send to work from a dirty scheduler

---
 erts/emulator/beam/erl_alloc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 942eaa47d0..d3109b9432 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -492,7 +492,7 @@ static TYPE *								\
 NAME##_alloc(void)							\
 {									\
     ErtsSchedulerData *esdp = erts_get_scheduler_data();		\
-    if (!esdp)								\
+    if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp))				\
 	return NULL;							\
     return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__,		\
 				    (int) esdp->no - 1);		\
-- 
cgit v1.2.3


From effb1c0a593e71664dec5b06da72518e74254c1b Mon Sep 17 00:00:00 2001
From: Michal Ptaszek <michal@ptaszek.net>
Date: Wed, 12 Mar 2014 16:22:08 +0100
Subject: Pass full strings to DTrace probes

Whenever string is passed as an argument to a DTrace probe, its length
should be properly computed. Until now in order to get length of the
input buffer size_of(char *) was used - which evalutes to 4 or 8
(depending on the architecture). To get a proper length,
size_of(DTRACE_CHARBUF_NAME(buffer_name)) should be used.
---
 erts/emulator/beam/beam_emu.c      |  2 +-
 erts/emulator/beam/copy.c          |  3 +-
 erts/emulator/beam/dist.c          | 64 +++++++++++++++++++++++---------------
 erts/emulator/beam/erl_async.c     |  6 ++--
 erts/emulator/beam/erl_bif_port.c  |  2 +-
 erts/emulator/beam/erl_port_task.c |  4 +--
 erts/emulator/beam/erl_process.c   |  2 +-
 erts/emulator/beam/io.c            | 14 ++++-----
 8 files changed, 57 insertions(+), 40 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 78ab6fa30f..b8688608c2 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1270,7 +1270,7 @@ void process_main(void)
                                       (Eterm)fptr[1], (Uint)fptr[2],
                                       NULL, fun_buf);
                 } else {
-                    erts_snprintf(fun_buf, sizeof(fun_buf),
+                    erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
                                   "<unknown/%p>", next);
                 }
             }
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 23c0fca6aa..ff2ce975ea 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -47,7 +47,8 @@ copy_object(Eterm obj, Process* to)
     if (DTRACE_ENABLED(copy_object)) {
         DTRACE_CHARBUF(proc_name, 64);
 
-        erts_snprintf(proc_name, sizeof(proc_name), "%T", to->common.id);
+        erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)),
+                      "%T", to->common.id);
         DTRACE2(copy_object, proc_name, size);
     }
 #endif
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 6ecf3f0722..ec07ddcd9c 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -851,9 +851,12 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
 #ifdef USE_VM_PROBES
     *node_name = *sender_name = *receiver_name = '\0';
     if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) {
-        erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname);
-        erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id);
-        erts_snprintf(receiver_name, sizeof(receiver_name), "%T", remote);
+        erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)),
+                      "%T", dsdp->dep->sysname);
+        erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
+                      "%T", sender->common.id);
+        erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
+                      "%T", remote);
         msize = size_object(message);
         if (token != NIL && token != am_have_dt_utag) {
             tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
@@ -908,9 +911,11 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
 #ifdef USE_VM_PROBES
     *node_name = *sender_name = *receiver_name = '\0';
     if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) {
-        erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname);
-        erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id);
-        erts_snprintf(receiver_name, sizeof(receiver_name),
+        erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)),
+                      "%T", dsdp->dep->sysname);
+        erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
+                      "%T", sender->common.id);
+        erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
                       "{%T,%s}", remote_name, node_name);
         msize = size_object(message);
         if (token != NIL && token != am_have_dt_utag) {
@@ -971,11 +976,14 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
 #ifdef USE_VM_PROBES
     *node_name = *sender_name = *remote_name = '\0';
     if (DTRACE_ENABLED(process_exit_signal_remote)) {
-        erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname);
-        erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id);
-        erts_snprintf(remote_name, sizeof(remote_name),
+        erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)),
+                      "%T", dsdp->dep->sysname);
+        erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
+                      "%T", sender->common.id);
+        erts_snprintf(remote_name, sizeof(DTRACE_CHARBUF_NAME(remote_name)),
                       "{%T,%s}", remote, node_name);
-        erts_snprintf(reason_str, sizeof(reason), "%T", reason);
+        erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)),
+                      "%T", reason);
         if (token != NIL && token != am_have_dt_utag) {
             tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
@@ -1797,8 +1805,9 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
                     DTRACE_CHARBUF(port_str, 64);
                     DTRACE_CHARBUF(remote_str, 64);
 
-                    erts_snprintf(port_str, sizeof(port_str), "%T", cid);
-                    erts_snprintf(remote_str, sizeof(remote_str),
+                    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+                                  "%T", cid);
+                    erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
                                   "%T", dep->sysname);
                     DTRACE3(dist_port_not_busy, erts_this_node_sysname,
                             port_str, remote_str);
@@ -1855,9 +1864,11 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
             DTRACE_CHARBUF(remote_str, 64);
             DTRACE_CHARBUF(pid_str, 16);
 
-            erts_snprintf(port_str, sizeof(port_str), "%T", cid);
-            erts_snprintf(remote_str, sizeof(remote_str), "%T", dep->sysname);
-            erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->common.id);
+            erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid);
+            erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
+                          "%T", dep->sysname);
+            erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)),
+                          "%T", c_p->common.id);
             DTRACE4(dist_port_busy, erts_this_node_sysname,
                     port_str, remote_str, pid_str);
         }
@@ -1890,8 +1901,9 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
         DTRACE_CHARBUF(port_str, 64);
         DTRACE_CHARBUF(remote_str, 64);
 
-        erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
-        erts_snprintf(remote_str, sizeof(remote_str),
+        erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+                      "%T", prt->common.id);
+        erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
                       "%T", prt->dist_entry->sysname);
         DTRACE4(dist_output, erts_this_node_sysname, port_str,
                 remote_str, size);
@@ -1944,8 +1956,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
         DTRACE_CHARBUF(port_str, 64);
         DTRACE_CHARBUF(remote_str, 64);
 
-        erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
-        erts_snprintf(remote_str, sizeof(remote_str),
+        erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+                      "%T", prt->common.id);
+        erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
                       "%T", prt->dist_entry->sysname);
         DTRACE4(dist_outputv, erts_this_node_sysname, port_str,
                 remote_str, size);
@@ -2280,8 +2293,9 @@ erts_dist_port_not_busy(Port *prt)
         DTRACE_CHARBUF(port_str, 64);
         DTRACE_CHARBUF(remote_str, 64);
 
-        erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
-        erts_snprintf(remote_str, sizeof(remote_str),
+        erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+                      "%T", prt->common.id);
+        erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
                       "%T", prt->dist_entry->sysname);
         DTRACE3(dist_port_not_busy, erts_this_node_sysname,
                 port_str, remote_str);
@@ -3246,10 +3260,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
         DTRACE_CHARBUF(type_str, 12);
         DTRACE_CHARBUF(reason_str, 64);
 
-        erts_snprintf(what_str, sizeof(what_str), "%T", what);
-        erts_snprintf(node_str, sizeof(node_str), "%T", node);
-        erts_snprintf(type_str, sizeof(type_str), "%T", type);
-        erts_snprintf(reason_str, sizeof(reason_str), "%T", reason);
+        erts_snprintf(what_str, sizeof(DTRACE_CHARBUF_NAME(what_str)), "%T", what);
+        erts_snprintf(node_str, sizeof(DTRACE_CHARBUF_NAME(node_str)), "%T", node);
+        erts_snprintf(type_str, sizeof(DTRACE_CHARBUF_NAME(type_str)), "%T", type);
+        erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason);
         DTRACE5(dist_monitor, erts_this_node_sysname,
                 what_str, node_str, type_str, reason_str);
     }
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index e6d72f569b..aaeab0d56c 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -279,7 +279,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
     if (DTRACE_ENABLED(aio_pool_add)) {
         DTRACE_CHARBUF(port_str, 16);
 
-        erts_snprintf(port_str, sizeof(port_str), "%T", a->port);
+        erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+                      "%T", a->port);
         /* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */
         len = -1;
         DTRACE2(aio_pool_add, port_str, len);
@@ -314,7 +315,8 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
             if (DTRACE_ENABLED(aio_pool_get)) {
                 DTRACE_CHARBUF(port_str, 16);
 
-                erts_snprintf(port_str, sizeof(port_str), "%T", a->port);
+                erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+                              "%T", a->port);
                 /* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */
                 len = -1;
                 DTRACE2(aio_pool_get, port_str, len);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 3cd53ef65d..c014d81bb6 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -917,7 +917,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
         DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
 
         dtrace_proc_str(p, process_str);
-        erts_snprintf(port_str, sizeof(port_str), "%T", port->common.id);
+        erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", port->common.id);
         DTRACE3(port_open, process_str, name_buf, port_str);
     }
 #endif
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 547a42beb2..d6c0c9e78e 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -2029,9 +2029,9 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 	    DTRACE_CHARBUF(pid_str, 16);
 	    ErtsProcList* plp2 = plp;
 
-	    erts_snprintf(port_str, sizeof(port_str), "%T", pp->common.id);
+	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", pp->common.id);
 	    while (plp2 != NULL) {
-		erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid);
+        erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid);
 		DTRACE2(process_port_unblocked, pid_str, port_str);
 	    }
 	}
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1efd070afd..dbe54a25c3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8349,7 +8349,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 
         dtrace_pid_str(from, sender_str);
         dtrace_proc_str(rp, receiver_str);
-        erts_snprintf(reason_buf, sizeof(reason_buf) - 1, "%T", reason);
+        erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason);
         DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf);
     }
 #endif
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 9076bbe73c..920701c086 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2468,7 +2468,7 @@ set_port_connected(int bang_op,
 	    DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE);
 
 	    dtrace_pid_str(connect, process_str);
-	    erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
+	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id);
 	    dtrace_proc_str(rp, newprocess_str);
 	    DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str);
 	}
@@ -3581,9 +3581,9 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
        DTRACE_CHARBUF(rreason_str, 64);
 
-       erts_snprintf(from_str, sizeof(from_str), "%T", from);
+       erts_snprintf(from_str, sizeof(DTRACE_CHARBUF_NAME(from_str)), "%T", from);
        dtrace_port_str(p, port_str);
-       erts_snprintf(rreason_str, sizeof(rreason_str), "%T", rreason);
+       erts_snprintf(rreason_str, sizeof(DTRACE_CHARBUF_NAME(rreason_str)), "%T", rreason);
        DTRACE4(port_exit, from_str, port_str, p->name, rreason_str);
    }
 #endif
@@ -4650,7 +4650,7 @@ set_busy_port(ErlDrvPort dprt, int on)
 
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(port_busy)) {
-            erts_snprintf(port_str, sizeof(port_str),
+            erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
                           "%T", prt->common.id);
             DTRACE1(port_busy, port_str);
         }
@@ -4663,7 +4663,7 @@ set_busy_port(ErlDrvPort dprt, int on)
 
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(port_not_busy)) {
-            erts_snprintf(port_str, sizeof(port_str),
+            erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
                           "%T", prt->common.id);
             DTRACE1(port_not_busy, port_str);
         }
@@ -4715,9 +4715,9 @@ erts_port_resume_procs(Port *prt)
 	    DTRACE_CHARBUF(pid_str, 16);
 	    ErtsProcList* plp2 = plp;
 
-	    erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
+	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id);
 	    while (plp2 != NULL) {
-		erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid);
+        erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid);
 		DTRACE2(process_port_unblocked, pid_str, port_str);
 	    }
 	}
-- 
cgit v1.2.3


From c543d5bff7fb23c3f44cc4817c0654117de78919 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 12 Mar 2014 20:11:10 +0100
Subject: erts: Change external format for maps

to be: 116,Arity, K1,V1,K2,V2,...,Kn,Vn

instead of: 116,Arity, K1,K2,...,Kn, V1,V2,....,Vn

We think this will be better for future internal map structures
like HAMT. Would be bad if we need to iterate twice over HAMT
in term_to_binary, one for keys and one for values.
---
 erts/emulator/beam/external.c | 55 ++++++++++++++-----------------------------
 1 file changed, 18 insertions(+), 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9671cde228..656de7c49a 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2562,29 +2562,25 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    {
 		map_t *mp = (map_t*)map_val(obj);
 		Uint size = map_get_size(mp);
-		Eterm *mptr;
 
 		*ep++ = MAP_EXT;
 		put_int32(size, ep); ep += 4;
 
-		/* Push values first */
 		if (size > 0) {
-		    mptr = map_get_values(mp);
+		    Eterm *kptr = map_get_keys(mp);
+		    Eterm *vptr = map_get_values(mp);
+
 		    for (i = size-1; i >= 1; i--) {
 			WSTACK_PUSH(s, ENC_TERM);
-			WSTACK_PUSH(s, (UWord) mptr[i]);
+			WSTACK_PUSH(s, (UWord) vptr[i]);
+			WSTACK_PUSH(s, ENC_TERM);
+			WSTACK_PUSH(s, (UWord) kptr[i]);
 		    }
 
 		    WSTACK_PUSH(s, ENC_TERM);
-		    WSTACK_PUSH(s, (UWord) mptr[0]);
-
-		    mptr = map_get_keys(mp);
-		    for (i = size-1; i >= 1; i--) {
-			WSTACK_PUSH(s, ENC_TERM);
-			WSTACK_PUSH(s, (UWord) mptr[i]);
-		    }
+		    WSTACK_PUSH(s, (UWord) vptr[0]);
 
-		    obj = mptr[0];
+		    obj = kptr[0];
 		    goto L_jump_start;
 		}
 	    }
@@ -3518,16 +3514,16 @@ dec_term_atom_common:
 
 		keys  = make_tuple(hp);
 		*hp++ = make_arityval(size);
-		kptr  = hp;
 		hp   += size;
+		kptr = hp - 1;
 
 		mp    = (map_t*)hp;
 		hp   += MAP_HEADER_SIZE;
-		vptr  = hp;
 		hp   += size;
+		vptr = hp - 1;
 
-		/* kptr, first word for keys
-		 * vptr, first word for values
+		/* kptr, last word for keys
+		 * vptr, last word for values
 		 */
 
 		/*
@@ -3542,27 +3538,12 @@ dec_term_atom_common:
 		mp->keys       = keys;
 		*objp          = make_map(mp);
 
-		/* We assume the map is wellformed, meaning:
-		 * - ascending key order
-		 * - unique keys
-		 */
-
-		objp  = vptr + size - 1;
-		n     = size;
-
-		while (n-- > 0) {
-		    *objp = (Eterm) COMPRESS_POINTER(next);
-		    next  = objp;
-		    objp--;
-		}
-
-		objp  = kptr + size - 1;
-		n     = size;
-
-		while (n-- > 0) {
-		    *objp = (Eterm) COMPRESS_POINTER(next);
-		    next  = objp;
-		    objp--;
+		for (n = size; n; n--) {
+		    *vptr = (Eterm) COMPRESS_POINTER(next);
+		    *kptr = (Eterm) COMPRESS_POINTER(vptr);
+		    next  = kptr;
+		    vptr--;
+		    kptr--;
 		}
 	    }
 	    break;
-- 
cgit v1.2.3


From a996e168bfebe599cfe393cd132a87984d905d84 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 13 Mar 2014 15:23:44 +0100
Subject: erts: Add distribution capability flag for maps DFLAG_MAP_TAG

This is just a preparation to allow detection of older nodes
that do not understand maps (R16 and older).
---
 erts/emulator/beam/dist.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 0519a9225e..f32b999198 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -40,6 +40,7 @@
 #define DFLAG_SMALL_ATOM_TAGS     0x4000
 #define DFLAG_INTERNAL_TAGS       0x8000
 #define DFLAG_UTF8_ATOMS          0x10000
+#define DFLAG_MAP_TAG             0x20000
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
@@ -47,7 +48,8 @@
 			       | DFLAG_NEW_FLOATS		\
 			       | DFLAG_EXTENDED_PIDS_PORTS	\
 			       | DFLAG_EXPORT_PTR_TAG		\
-			       | DFLAG_BIT_BINARIES)
+			       | DFLAG_BIT_BINARIES             \
+			       | DFLAG_MAP_TAG)
 
 /* opcodes used in distribution messages */
 #define DOP_LINK		1
-- 
cgit v1.2.3


From 54a53c31e4bff25aef79d1624361b52407f572ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 6 Mar 2014 19:06:25 +0100
Subject: erts: Handle literals in is_map/1

---
 erts/emulator/beam/ops.tab | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 73630fda8e..68fcc177ae 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1484,7 +1484,8 @@ new_map j d I I
 update_map_assoc j s d I I
 update_map_exact j s d I I
 
-is_map Fail cq => jump Fail
+is_map Fail Literal=q => move Literal x | is_map Fail x
+is_map Fail c => jump Fail
 
 %macro: is_map IsMap -fail_action
 is_map f r
-- 
cgit v1.2.3


From 9327904dcbf2708823c05baa7532783fb0214792 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 7 Mar 2014 10:41:32 +0100
Subject: Teach the call_time trace to notice when the trace dies (non-SMP
 system)

The call_time trace is a special kind of tracing that requires
a tracer process just like ordinary call trace, but it never
actually sends anything to the tracer. It merely use the existence
of a trace process (and call trace flags) as an indication that
call_time tracing is active for the process.

If the tracer dies in a non-SMP run-time system, processes with
call_time tracing would not notice that the tracer had
died. Furthermore, if the set_on_spawn flag was active, the dead
tracer could be propagaged to newly spawned processes.

Before accumulating trace information in a non-SMP system, always
validate the tracer process. (In an SMP system, a reference to a
dead tracer will be cleared away each time a process is scheduled.)

While we could put all of the new code beam_bp.c, we have chosen to
make a function call from beam_bp.c to a function in erl_trace.c for
clarity's sake and to ease further maintenance. In the future, we
might want to handle tracing in more similar ways in the SMP and
non-SMP system.
---
 erts/emulator/beam/beam_bp.c   |  5 +++--
 erts/emulator/beam/erl_trace.c | 19 +++++++++++++++++++
 erts/emulator/beam/erl_trace.h |  1 +
 3 files changed, 23 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 49a34ab4ad..4e711c89e0 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -642,7 +642,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 	erts_smp_atomic_inc_nob(&bp->count->acount);
     }
 
-    if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) {
+    if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && erts_is_tracer_proc_valid(c_p)) {
 	Eterm w;
 	erts_trace_time_call(c_p, I, bp->time);
 	w = (BeamInstr) *c_p->cp;
@@ -730,7 +730,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	}
     }
     if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE &&
-	IS_TRACED_FL(p, F_TRACE_CALLS)) {
+	IS_TRACED_FL(p, F_TRACE_CALLS) &&
+	erts_is_tracer_proc_valid(p)) {
 	BeamInstr *pc = (BeamInstr *)ep->code+3;
 	erts_trace_time_call(p, pc, bp->time);
     }
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 6978a5f11a..305058ceff 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -151,6 +151,11 @@ do { \
     message dispatcher thread takes care of that). */
 #define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \
 do { (RES) = (TPID); } while(0)
+int
+erts_is_tracer_proc_valid(Process* p)
+{
+    return 1;
+}
 #else
 #define ERTS_NULL_TRACER_REF NULL
 #define ERTS_TRACER_REF_TYPE Process *
@@ -163,6 +168,20 @@ do { \
 	return; \
     } \
 } while (0)
+int
+erts_is_tracer_proc_valid(Process* p)
+{
+    Process* tracer;
+
+    tracer = erts_proc_lookup(ERTS_TRACER_PROC(p));
+    if (tracer && ERTS_TRACE_FLAGS(tracer) & F_TRACER) {
+	return 1;
+    } else {
+	ERTS_TRACER_PROC(p) = NIL;
+	ERTS_TRACE_FLAGS(p) = ~TRACEE_FLAGS;
+	return 0;
+    }
+}
 #endif
 
 static Uint active_sched;
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 853c6cb0d8..4f2c70d6e7 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -39,6 +39,7 @@ void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp);
 void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp);
 void erts_set_system_monitor(Eterm monitor);
 Eterm erts_get_system_monitor(void);
+int erts_is_tracer_proc_valid(Process* p);
 
 #ifdef ERTS_SMP
 void erts_check_my_tracer_proc(Process *);
-- 
cgit v1.2.3


From f43a1dc55d42de3097f75ca65baead9a2ff05c78 Mon Sep 17 00:00:00 2001
From: Dmitry Kolesnikov <dmitry.kolesnikov>
Date: Tue, 11 Feb 2014 00:27:11 +0200
Subject: Raspberry PI / Android a minimal cross-compile configuration

Enable a cross compile Erlang/OTP platform to Android or Raspberry PI
using Android NDK. Port emulator and core application to support target
HW platform. Exclude any add-on services required for OTP platform deployment
into target hardware due to device fragmentation and jail-break requirements.

 * fix erts/emulator/beam/sys.h
   Disable redefinition of __noreturn macro

 * port erts/emulator/sys/unix/erl_child_setup.c
   Use techniques proposed by https://code.google.com/p/erlang4android to
   access system properties

 * fix erts/emulator/sys/unix/erl_unix_sys_ddll.c
   The static linking of emulator cannot find dlerror(), dlopen() symbols

 * port erts/emulator/sys/unix/sys.c
   make path to shell configurable at build time

 * port erts/etc/common/Makefile.in
   disable librt for *-linux-androideabi

 * port erts/lib_src/pthread/ethread.c
   Use techniques proposed by https://code.google.com/p/erlang4android to
   disable emulator crash if kernel threads are on. Replace unreliable
   pthread_sigmask() by sigprocmask()

 * port lib/erl_interface/src/connect/ei_connect.c
   Disable call to undefined gethostid()

 * port lib/erl_interface/src/connect/ei_resolve.c
   Use gethostbyname_r() on Android platform
---
 erts/emulator/beam/sys.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index e273056a2b..05f07e57b2 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -154,10 +154,14 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 /* In VC++, noreturn is a declspec that has to be before the types,
  * but in GNUC it is an att ribute to be placed between return type
  * and function name, hence __decl_noreturn <types> __noreturn <function name>
+ *
+ * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h
  */
 #if __GNUC__
 #  define __decl_noreturn
-#  define __noreturn __attribute__((noreturn))
+#  ifndef __noreturn
+#     define __noreturn __attribute__((noreturn))
+#  endif
 #else
 #  if defined(__WIN32__) && defined(_MSC_VER)
 #    define __noreturn
-- 
cgit v1.2.3


From bf3222f10edbd1f6a5186c8fb35c29900ad0665f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 20 Mar 2014 19:01:11 +0100
Subject: Introduce minimum allowed major driver and nif versions on load

---
 erts/emulator/beam/erl_bif_ddll.c |  6 ++++--
 erts/emulator/beam/erl_driver.h   | 16 ++++++++++++++++
 erts/emulator/beam/erl_nif.c      |  6 ++++--
 erts/emulator/beam/erl_nif.h      | 12 ++++++++++++
 4 files changed, 36 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 1728b200f7..56cd2ba04f 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1548,8 +1548,10 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 
     switch (dp->extended_marker) {
     case ERL_DRV_EXTENDED_MARKER:
-	if (ERL_DRV_EXTENDED_MAJOR_VERSION != dp->major_version
-	    || ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version) {
+	if (dp->major_version < ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD
+	    || (ERL_DRV_EXTENDED_MAJOR_VERSION < dp->major_version
+		|| (ERL_DRV_EXTENDED_MAJOR_VERSION == dp->major_version
+		    && ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version))) {
 	    /* Incompatible driver version */
 	    res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
 	    goto error;
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 5517c26ba4..3ecb379326 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -135,6 +135,22 @@ typedef struct {
 #define ERL_DRV_EXTENDED_MAJOR_VERSION	3
 #define ERL_DRV_EXTENDED_MINOR_VERSION	0
 
+/*
+ * The emulator will refuse to load a driver with a major version
+ * lower than ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load
+ * may however fail if user have not removed use of deprecated
+ * symbols.
+ *
+ * The ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow
+ * loading of drivers built at least two major OTP releases
+ * ago.
+ *
+ * Bump of major version to 3 happened in OTP 17. That is, in
+ * OTP 19 we can increase ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD
+ * to 3.
+ */
+#define ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2
+
 /*
  * The emulator will refuse to load a driver with different major
  * version than the one used by the emulator.
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 40860e141c..063dba056e 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2049,8 +2049,10 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	      (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) {
 	ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful");
     }
-    else if (entry->major != ERL_NIF_MAJOR_VERSION
-	     || entry->minor > ERL_NIF_MINOR_VERSION
+    else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD
+	     || (ERL_NIF_MAJOR_VERSION < entry->major
+		 || (ERL_NIF_MAJOR_VERSION == entry->major
+		     && ERL_NIF_MINOR_VERSION < entry->minor))
 	     || (entry->major==2 && entry->minor == 5)) { /* experimental maps */
 	
 	ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).",
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index c12ba4d554..5b93c2398e 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -46,6 +46,18 @@
 #define ERL_NIF_MAJOR_VERSION 2
 #define ERL_NIF_MINOR_VERSION 6
 
+/*
+ * The emulator will refuse to load a nif-lib with a major version
+ * lower than ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load
+ * may however fail if user have not removed use of deprecated
+ * symbols.
+ *
+ * The ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow
+ * loading of nif-libs built at least two major OTP releases
+ * ago.
+ */
+#define ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2
+
 #include <stdlib.h>
 
 #ifdef SIZEOF_CHAR
-- 
cgit v1.2.3


From 78b118bc5f503435b1d9216b3a3279e0c9fd9ecd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 21 Mar 2014 16:38:13 +0100
Subject: erts: Fix heap overflow in maps:remove/2 when key is not found

One key-value pair too many was copied.
---
 erts/emulator/beam/erl_map.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 2fff7f9390..fdd2d0c0f6 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -647,22 +647,24 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     *mhp++ = tup;
 
     if (is_immed(key)) {
-	while(n--) {
+	while (1) {
 	    if (*ks == key) {
 		goto found_key;
-	    } else {
+	    } else if (--n) {
 		*mhp++ = *vs++;
 		*thp++ = *ks++;
-	    }
+	    } else
+		break;
 	}
     } else {
-	while(n--) {
+	while(1) {
 	    if (EQ(*ks, key)) {
 		goto found_key;
-	    } else {
+	    } else if (--n) {
 		*mhp++ = *vs++;
 		*thp++ = *ks++;
-	    }
+	    } else
+		break;
 	}
     }
 
@@ -676,7 +678,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 
 found_key:
     /* Copy rest of keys and values */
-    if (n) {
+    if (--n) {
 	sys_memcpy(mhp, vs+1, n*sizeof(Eterm));
 	sys_memcpy(thp, ks+1, n*sizeof(Eterm));
     }
-- 
cgit v1.2.3


From c8b55a755da15e766a745a948f3d8020f581fd37 Mon Sep 17 00:00:00 2001
From: Scott Lystig Fritchie <slfritchie@snookles.com>
Date: Tue, 25 Mar 2014 18:34:11 +0900
Subject: Fix DTrace/SystemTap-related formatting
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Thanks to Michał Ptaszek for bringing this embarrassing formatting error to
my attention.  Many DTrace/SystemTap trace strings are incorrectly
truncated at 4/8 bytes, depending on the CPU word size.  This patch expands
the work from commit d032e097 by zheng siyao.

Michał's report to the erlang-bugs list can be found at:
http://erlang.org/pipermail/erlang-bugs/2014-March/004250.html

Conflicts:
	erts/emulator/beam/beam_emu.c
	erts/emulator/beam/copy.c
	erts/emulator/beam/dist.c
	erts/emulator/beam/erl_async.c
	erts/emulator/beam/erl_bif_port.c
	erts/emulator/beam/erl_port_task.c
	erts/emulator/beam/erl_process.c
	erts/emulator/beam/io.c
---
 erts/emulator/beam/erl_message.c   | 6 ++++--
 erts/emulator/beam/erl_port_task.c | 2 +-
 erts/emulator/beam/io.c            | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 6a9030fd99..0eb8117980 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -896,8 +896,10 @@ erts_send_message(Process* sender,
  #ifdef USE_VM_PROBES
     *sender_name = *receiver_name = '\0';
    if (DTRACE_ENABLED(message_send)) {
-        erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id);
-        erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id);
+        erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
+		      "%T", sender->common.id);
+        erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
+		      "%T", receiver->common.id);
     }
 #endif
     if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index d6c0c9e78e..e9665c0142 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -2031,7 +2031,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 
 	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", pp->common.id);
 	    while (plp2 != NULL) {
-        erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid);
+		erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid);
 		DTRACE2(process_port_unblocked, pid_str, port_str);
 	    }
 	}
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 920701c086..6bb3ffd8ce 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4717,7 +4717,7 @@ erts_port_resume_procs(Port *prt)
 
 	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id);
 	    while (plp2 != NULL) {
-        erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid);
+		erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid);
 		DTRACE2(process_port_unblocked, pid_str, port_str);
 	    }
 	}
-- 
cgit v1.2.3


From 64ffc509c5069e29c5d172dfcad6655efbeafd9b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 25 Mar 2014 14:23:45 +0100
Subject: erts: Fix compile warning in io.c for windows

---
 erts/emulator/beam/io.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index cd5060ebb3..0480cdf823 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -914,8 +914,8 @@ int erts_port_handle_xports(Port *prt)
    (iov)->iov_base = (ptr);				\
    (iov)->iov_len = (len);				\
    if (sizeof((iov)->iov_len) < sizeof(len)				\
-       /* Check if (len) overflowed (iov)->iov_len */			\
-       && ((len) >> (sizeof((iov)->iov_len)*CHAR_BIT)) != 0) {		\
+       /* Check if (len) overflowed (iov)->iov_len */                   \
+       && (iov)->iov_len != (len)) {		                        \
        goto L_overflow;							\
    }									\
    *(bv)++ = (bin);					\
-- 
cgit v1.2.3


From 32eae452dbd6da109914af3a9003d9bbcb696d42 Mon Sep 17 00:00:00 2001
From: Kostis Sagonas <kostis@it.uu.se>
Date: Wed, 26 Mar 2014 23:51:05 +0100
Subject: Change the subtag used for maps from 0xB to 0xF

The subtag chosen for maps breaks crucial assumptions in other parts of
the system, in particular the native code generation scheme used in the
HiPE compiler. Therefore it is better to use 'the other' unused subtag.

The main change here is the use of 0xF subtag for maps instead of 0xB.
The rest of the differences are changes to and additions of comments.

One more comment from my part.
I noticed that the file contains the following two lines:

The comment of the second line is most certainly a copy and paste error
and should be appropriately fixed by the OTP developers. More importantly,
it would be great if that subtag (0x7) turned out to be unused and could
be used for maps instead of the 0xF one. This might turn out very handy
in the future. I can elaborate on this, if needed.
---
 erts/emulator/beam/erl_term.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index f10a3a9d38..982e63ee31 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -112,11 +112,11 @@ struct erl_node_; /* Declared in erl_node_tables.h */
  *	1000	REFC_BINARY	|		|
  *	1001	HEAP_BINARY	| BINARIES	|
  *	1010	SUB_BINARY	|		|
- *      1011    Not used
+ *      1011    Not used; see comment below
  *      1100    EXTERNAL_PID  |                 |
  *      1101    EXTERNAL_PORT | EXTERNAL THINGS |
  *      1110    EXTERNAL_REF  |                 |
- *      1111    Not used
+ *      1111    MAP
  *
  * COMMENTS:
  *
@@ -140,10 +140,11 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define REFC_BINARY_SUBTAG	(0x8 << _TAG_PRIMARY_SIZE) /* BINARY */
 #define HEAP_BINARY_SUBTAG	(0x9 << _TAG_PRIMARY_SIZE) /* BINARY */
 #define SUB_BINARY_SUBTAG	(0xA << _TAG_PRIMARY_SIZE) /* BINARY */
-#define MAP_SUBTAG		(0xB << _TAG_PRIMARY_SIZE) /* MAP */
+/*   _BINARY_XXX_MASK depends on 0xB being unused */
 #define EXTERNAL_PID_SUBTAG	(0xC << _TAG_PRIMARY_SIZE) /* EXTERNAL_PID */
 #define EXTERNAL_PORT_SUBTAG	(0xD << _TAG_PRIMARY_SIZE) /* EXTERNAL_PORT */
 #define EXTERNAL_REF_SUBTAG	(0xE << _TAG_PRIMARY_SIZE) /* EXTERNAL_REF */
+#define MAP_SUBTAG		(0xF << _TAG_PRIMARY_SIZE) /* MAP */
 
 
 #define _TAG_HEADER_ARITYVAL	(TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG)
@@ -156,11 +157,11 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define _TAG_HEADER_REFC_BIN	(TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG)
 #define _TAG_HEADER_HEAP_BIN	(TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG)
 #define _TAG_HEADER_SUB_BIN	(TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG)
-#define _TAG_HEADER_MAP	 	(TAG_PRIMARY_HEADER|MAP_SUBTAG)
 #define _TAG_HEADER_EXTERNAL_PID  (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG)
 #define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG)
 #define _TAG_HEADER_EXTERNAL_REF  (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG)
 #define _TAG_HEADER_BIN_MATCHSTATE (TAG_PRIMARY_HEADER|BIN_MATCHSTATE_SUBTAG)
+#define _TAG_HEADER_MAP	 	(TAG_PRIMARY_HEADER|MAP_SUBTAG)
 
 
 #define _TAG_HEADER_MASK	0x3F
-- 
cgit v1.2.3


From d1fde60e2814ba1d8013c48983bc206c1112cb98 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 27 Mar 2014 17:13:43 +0100
Subject: erts: Adjust is_external_header() for new map tag

to not mistake a map for an external term (pid, port or ref).
---
 erts/emulator/beam/erl_term.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 982e63ee31..37014ccf94 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -893,7 +893,8 @@ typedef struct external_thing_ {
   (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_EXTERNAL_REF)
 
 #define is_external_header(x) \
-  (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID)
+  (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID \
+   && ((x) & _TAG_HEADER_MASK) != _TAG_HEADER_MAP)
 
 #define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x))))
 
-- 
cgit v1.2.3


From 56018ef95e70c5630c1c6fe16d22b54f8ecd0791 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 27 Mar 2014 20:59:01 +0100
Subject: erts: Fix bug of scheduling a suspended process

This is a race that can cause RUNNING (instead of RUNNING_SYS)
set on a SUSPENDED process. The effect of this race happening is probably
quite benign.

The bug was discovered by process_SUITE:processes_gc_trap on debug VM
crashing on last assert in schedule():

/* Never run a suspended process */
ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
---
 erts/emulator/beam/erl_process.c | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 37e1d07107..bc69d5ad91 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9207,7 +9207,6 @@ Process *schedule(Process *p, int calls)
 	 */
     pick_next_process: {
 	    erts_aint32_t psflg_band_mask;
-	    erts_aint32_t running_flag;
 	    int prio_q;
 	    int qmask;
 
@@ -9269,12 +9268,6 @@ Process *schedule(Process *p, int calls)
 		state = erts_smp_atomic32_read_nob(&p->state);
 	    }
 
-
-	    if (state & ERTS_PSFLG_ACTIVE_SYS)
-		running_flag = ERTS_PSFLG_RUNNING_SYS;
-	    else
-		running_flag = ERTS_PSFLG_RUNNING;
-
 	    while (1) {
 		erts_aint32_t exp, new, tmp;
 		tmp = new = exp = state;
@@ -9284,8 +9277,12 @@ Process *schedule(Process *p, int calls)
 		    tmp = state & (ERTS_PSFLG_SUSPENDED
 				   | ERTS_PSFLG_PENDING_EXIT
 				   | ERTS_PSFLG_ACTIVE_SYS);
-		    if (tmp != ERTS_PSFLG_SUSPENDED)
-			new |= running_flag;
+		    if (tmp != ERTS_PSFLG_SUSPENDED) {
+			if (state & ERTS_PSFLG_ACTIVE_SYS)
+			    new |= ERTS_PSFLG_RUNNING_SYS;
+			else
+			    new |= ERTS_PSFLG_RUNNING;
+		    }
 		}
 		state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp);
 		if (state == exp) {
-- 
cgit v1.2.3


From 49d8e0f92e0b496c9be4c13edecc498b31380cc9 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Tue, 25 Mar 2014 17:41:31 +0100
Subject: erts: Add etp-lc-dump and etp-ppc-stacktrace macro

---
 erts/emulator/beam/erl_lock_check.c | 4 ++--
 erts/emulator/beam/erl_process.c    | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 7e3a90779d..c13eb87012 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -270,9 +270,9 @@ union erts_lc_free_block_t_ {
 
 static ethr_tsd_key locks_key;
 
-static erts_lc_locked_locks_t *erts_locked_locks;
+static erts_lc_locked_locks_t *erts_locked_locks = NULL;
 
-static erts_lc_free_block_t *free_blocks;
+static erts_lc_free_block_t *free_blocks = NULL;
 
 #ifdef ERTS_LC_STATIC_ALLOC
 #define ERTS_LC_FB_CHUNK_SIZE 10000
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8bf0cc9491..baca73f7f6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2672,6 +2672,13 @@ aux_thread(void *unused)
     ErtsThrPrgrCallbacks callbacks;
     int thr_prgr_active = 1;
 
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    {
+	char buf[] = "aux_thread";
+	erts_lc_set_thread_name(buf);
+    }
+#endif
+
     ssi->event = erts_tse_fetch();
 
     callbacks.arg = (void *) ssi;
-- 
cgit v1.2.3


From 98ca47d657fafa4d91b128053e9286114115c0a8 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Fri, 28 Mar 2014 08:58:05 -0400
Subject: fix dirty NIF invalid memory read

Dirty NIF support used an Export structure to facilitate calls to dirty
NIFs and finalizers, but Export isn't large enough to hold all necessary
data. This was causing an invalid memory read in beam_emu.c past the end of
the Export object. Add a local extended Export struct to erl_nif.c that can
hold all the necessary data.
---
 erts/emulator/beam/beam_emu.c |  2 +-
 erts/emulator/beam/erl_nif.c  | 42 ++++++++++++++++++++++++++----------------
 2 files changed, 27 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9634faff1d..1026e5f649 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3525,7 +3525,7 @@ get_map_elements_fail:
 		erts_post_nif(&env);
 #ifdef ERTS_DIRTY_SCHEDULERS
 		if (is_non_value(nif_bif_result) && c_p->freason == TRAP) {
-		    Export* ep = (Export*) c_p->psd->data[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT];
+		    Export* ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(c_p);
 		    ep->code[0] = I[-3];
 		    ep->code[1] = I[-2];
 		}
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 063dba056e..f503b222d0 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1515,19 +1515,28 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 
+/* NIFs exports need one more item than the Export struct provides, the
+ * erl_module_nif*, so the DirtyNifExport below adds that. The Export
+ * member must be first in the struct.
+ */
+typedef struct {
+    Export exp;
+    struct erl_module_nif* m;
+} DirtyNifExport;
+
 static void
-alloc_proc_psd(Process* proc, Export **ep)
+alloc_proc_psd(Process* proc, DirtyNifExport **ep)
 {
     int i;
     if (!*ep) {
-	*ep = erts_alloc(ERTS_ALC_T_PSD, sizeof(Export));
-	sys_memset((void*) *ep, 0, sizeof(Export));
+	*ep = erts_alloc(ERTS_ALC_T_PSD, sizeof(DirtyNifExport));
+	sys_memset((void*) *ep, 0, sizeof(DirtyNifExport));
 	for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	    (*ep)->addressv[i] = &(*ep)->code[3];
+	    (*ep)->exp.addressv[i] = &(*ep)->exp.code[3];
 	}
-	(*ep)->code[3] = (BeamInstr) em_call_nif;
+	(*ep)->exp.code[3] = (BeamInstr) em_call_nif;
     }
-    (void) ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, *ep);
+    (void) ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, &(*ep)->exp);
 }
 
 static ERL_NIF_TERM
@@ -1560,7 +1569,7 @@ enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
     erts_aint32_t state, n, a;
     Process* proc = env->proc;
     Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
-    Export* ep = NULL;
+    DirtyNifExport* ep = NULL;
     int i;
 
     int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND));
@@ -1585,15 +1594,16 @@ enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
 	if (a == state)
 	    break;
     }
-    if (!(ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
+    if (!(ep = (DirtyNifExport*) ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
 	alloc_proc_psd(proc, &ep);
     ERTS_VBUMP_ALL_REDS(proc);
-    ep->code[2] = argc;
+    ep->exp.code[2] = argc;
     for (i = 0; i < argc; i++) {
 	reg[i] = (Eterm) argv[i];
     }
-    proc->i = (BeamInstr*) ep->addressv[0];
-    ep->code[4] = (BeamInstr) fp;
+    proc->i = (BeamInstr*) ep->exp.addressv[0];
+    ep->exp.code[4] = (BeamInstr) fp;
+    ep->m = env->mod_nif;
     proc->freason = TRAP;
 
     return THE_NON_VALUE;
@@ -1609,17 +1619,17 @@ enif_schedule_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result,
 #ifdef USE_THREADS
     Process* proc = env->proc;
     Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
-    Export* ep;
+    DirtyNifExport* ep;
 
     erts_smp_atomic32_read_band_mb(&proc->state,
 				   ~(ERTS_PSFLG_DIRTY_CPU_PROC
 				     |ERTS_PSFLG_DIRTY_IO_PROC
 				     |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q
 				     |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
-    if (!(ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
+    if (!(ep = (DirtyNifExport*) ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
 	alloc_proc_psd(proc, &ep);
     ERTS_VBUMP_ALL_REDS(proc);
-    ep->code[2] = 2;
+    ep->exp.code[2] = 2;
     reg[0] = (Eterm) result;
 #if HAVE_INT64 && SIZEOF_LONG != 8
     ASSERT(sizeof(fp) <= sizeof(ErlNifUInt64));
@@ -1628,8 +1638,8 @@ enif_schedule_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result,
     ASSERT(sizeof(fp) <= sizeof(unsigned long));
     reg[1] = (Eterm) enif_make_ulong(env, (unsigned long) fp);
 #endif
-    proc->i = (BeamInstr*) ep->addressv[0];
-    ep->code[4] = (BeamInstr) execute_dirty_nif_finalizer;
+    proc->i = (BeamInstr*) ep->exp.addressv[0];
+    ep->exp.code[4] = (BeamInstr) execute_dirty_nif_finalizer;
     proc->freason = TRAP;
 
     return THE_NON_VALUE;
-- 
cgit v1.2.3


From 4ec8d3be1936bda8cb69a97619e7b7796c54948a Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Fri, 28 Mar 2014 09:10:48 -0400
Subject: prevent NIF purge during dirty NIF execution

Reference-count the NIF before and after invoking a NIF on dirty schedulers
to prevent having the NIF purged during the call.
---
 erts/emulator/beam/erl_nif.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f503b222d0..ff551ea3af 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1543,7 +1543,7 @@ static ERL_NIF_TERM
 execute_dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     Eterm* reg = ERTS_PROC_GET_SCHDATA(env->proc)->x_reg_array;
-    ERL_NIF_TERM result = (ERL_NIF_TERM) reg[0];
+    ERL_NIF_TERM result, dirty_result = (ERL_NIF_TERM) reg[0];
     typedef ERL_NIF_TERM (*FinalizerFP)(ErlNifEnv*, ERL_NIF_TERM);
     FinalizerFP fp;
 #if HAVE_INT64 && SIZEOF_LONG != 8
@@ -1553,7 +1553,11 @@ execute_dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(sizeof(fp) <= sizeof(unsigned long));
     enif_get_ulong(env, reg[1], (unsigned long *) &fp);
 #endif
-    return (*fp)(env, result);
+    result = (*fp)(env, dirty_result);
+    if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0
+	&& env->mod_nif->mod == NULL)
+	close_lib(env->mod_nif);
+    return result;
 }
 
 #endif /* ERTS_DIRTY_SCHEDULERS */
@@ -1606,6 +1610,8 @@ enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
     ep->m = env->mod_nif;
     proc->freason = TRAP;
 
+    erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
+
     return THE_NON_VALUE;
 #else
     return (*fp)(env, argc, argv);
-- 
cgit v1.2.3


From fa7f3e7ec457f1143ed4a3db1aa9d7aed8005a57 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 8 Apr 2014 15:03:17 +0200
Subject: erts: Fix system_monitor(large_heap) for non-smp VM

No message for large_heap monitoring was ever sent on non-smp VM.
Bug introduced in R16B.
---
 erts/emulator/beam/erl_trace.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 305058ceff..ea5c850a30 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -2524,7 +2524,7 @@ monitor_large_heap(Process *p) {
 #ifndef ERTS_SMP 
     ASSERT(is_internal_pid(system_monitor));
     monitor_p = erts_proc_lookup(system_monitor);
-    if (monitor_p || p == monitor_p) {
+    if (!monitor_p || p == monitor_p) {
 	return;
     }
 #endif
-- 
cgit v1.2.3


From e492b43c3e4366e865e5f1c34d0834df2a91d490 Mon Sep 17 00:00:00 2001
From: Eiichi Tsukata <devel@etsukata.com>
Date: Wed, 23 Apr 2014 21:46:21 +0900
Subject: Add erlang:system_info(tolerant_timeofday)

Add erlang:system_info(tolerant_timeofday), an API to check whether
compensation for sudden changes of system time is enabled or not.
---
 erts/emulator/beam/erl_bif_info.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 2adba9b240..4d5e55aaf5 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2691,6 +2691,11 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) {
         BIF_RET(make_small(erts_db_get_max_tabs()));
     }
+    else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) {
+	BIF_RET(erts_disable_tolerant_timeofday
+		? am_disabled
+		: am_enabled);
+    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
-- 
cgit v1.2.3


From a652d1d882d6f4ddb46a23bf550cb8d12e403e8c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 29 Apr 2014 15:24:41 +0200
Subject: Fix race between ETS table deletion and unfixation

Symptom: VM crash running mnesia_SUITE

Scenario: Process A terminates while still having fixed table T
and process B "at the same time" deletes table T with ets:delete/1 or
by terminating.

Problem: A table scheduled for deallocation do only have a valid
'common' part. The unfix-table-at-process-exit code tried to read the
hash-specific part of such a table.

Solution: Must back off if DB_DELETE flag is set.

Since: R16B
---
 erts/emulator/beam/erl_db.c | 62 ++++++++++++++++++++++++---------------------
 1 file changed, 33 insertions(+), 29 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index a5d67571e2..8f246ffa07 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -259,10 +259,11 @@ static void schedule_free_dbtable(DbTable* tb)
     /*
      * NON-SMP case: Caller is *not* allowed to access the *tb
      *               structure after this function has returned!          
-     * SMP case:     Caller is allowed to access the *tb structure
-     *               until the bif has returned (we typically
-     *               need to unlock the table lock after this
-     *               function has returned).
+     * SMP case:     Caller is allowed to access the *common* part of the *tb
+     *  	     structure until the bif has returned (we typically need to
+     *  	     unlock the table lock after this function has returned).
+     *  	     Caller is *not* allowed to access the specialized part
+     *  	     (hash or tree) of *tb after this function has returned.
      */
     ASSERT(erts_refc_read(&tb->common.ref, 0) == 0);
     erts_schedule_thr_prgr_later_cleanup_op(free_dbtable,
@@ -3279,34 +3280,37 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 		}
 		erts_smp_rwmtx_runlock(mmtl);
 		if (tb) {
-		    int reds;
-		    DbFixation** pp;
+		    int reds = 0;
 
 		    db_lock(tb, LCK_WRITE_REC);
-		    #ifdef ERTS_SMP
-		    erts_smp_mtx_lock(&tb->common.fixlock);
-		    #endif
-		    reds = 10;
-		    
-		    for (pp = &tb->common.fixations; *pp != NULL;
-			  pp = &(*pp)->next) {
-			if ((*pp)->pid == pid) {
-			    DbFixation* fix = *pp;
-			    erts_aint_t diff = -((erts_aint_t) fix->counter);
-			    erts_refc_add(&tb->common.ref,diff,0);
-			    *pp = fix->next;
-			    erts_db_free(ERTS_ALC_T_DB_FIXATION,
-					 tb, fix, sizeof(DbFixation));
-			    ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
-			    break;
+		    if (!(tb->common.status & DB_DELETE)) {
+			DbFixation** pp;
+
+			#ifdef ERTS_SMP
+			erts_smp_mtx_lock(&tb->common.fixlock);
+			#endif
+			reds = 10;
+
+			for (pp = &tb->common.fixations; *pp != NULL;
+			      pp = &(*pp)->next) {
+			    if ((*pp)->pid == pid) {
+				DbFixation* fix = *pp;
+				erts_aint_t diff = -((erts_aint_t) fix->counter);
+				erts_refc_add(&tb->common.ref,diff,0);
+				*pp = fix->next;
+				erts_db_free(ERTS_ALC_T_DB_FIXATION,
+					     tb, fix, sizeof(DbFixation));
+				ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
+				break;
+			    }
+			}
+			#ifdef ERTS_SMP
+			erts_smp_mtx_unlock(&tb->common.fixlock);
+			#endif
+			if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) {
+			    db_unfix_table_hash(&(tb->hash));
+			    reds += 40;
 			}
-		    }
-		    #ifdef ERTS_SMP
-		    erts_smp_mtx_unlock(&tb->common.fixlock);
-		    #endif
-		    if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) {
-			db_unfix_table_hash(&(tb->hash));
-			reds += 40;
 		    }
 		    db_unlock(tb, LCK_WRITE_REC);
 		    BUMP_REDS(c_p, reds);
-- 
cgit v1.2.3


From 23246d73bbd3188e2c0a45408b9bd29fd034ccaf Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 25 Apr 2014 22:31:41 +0200
Subject: erts: Save some space in process struct for hipe

by combining hipe.ncallee and hipe.closure in a union
as the comment indicate should be possible.
---
 erts/emulator/beam/beam_emu.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1026e5f649..f6524f36d2 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4993,14 +4993,14 @@ get_map_elements_fail:
 	  * ... remainder of original BEAM code
 	  */
 	 ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
-	 c_p->hipe.ncallee = (void(*)(void)) I[-4];
+	 c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
 	 cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8);
 	 ++hipe_trap_count;
 	 goto L_hipe_mode_switch;
      }
      OpCase(hipe_trap_call_closure): {
        ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
-       c_p->hipe.ncallee = (void(*)(void)) I[-4];
+       c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
        cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8);
        ++hipe_trap_count;
        goto L_hipe_mode_switch;
-- 
cgit v1.2.3


From c4cdb1847e09d1153a6044d7a6aac05db4fad515 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 25 Apr 2014 23:02:26 +0200
Subject: erts: Fix global tracing of beam function when called from hipe code

Make hipe to beam calls use export entry.

Makes it a lot easier to handle global tracing correctly
(breakpoints in export entry). A beam function should now be traced
correctly regardless how it is called.

This will also fix a SEGV crash when a hipe stub is made pointing into
a traced export entry and tracing is then stopped which clears the export
entry causing the hipe stub to execute beam instruction NULL.

This commit assumes that hipe code never calls local beam functions, which
should be the case nowadays as we only hipe compile entire modules.
---
 erts/emulator/beam/beam_emu.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index f6524f36d2..9b251a6ad1 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5034,7 +5034,10 @@ get_map_elements_fail:
        case HIPE_MODE_SWITCH_RES_RETURN:
 	 ASSERT(is_value(reg[0]));
 	 MoveReturn(reg[0], r(0));
-       case HIPE_MODE_SWITCH_RES_CALL:
+       case HIPE_MODE_SWITCH_RES_CALL_EXPORTED:
+	 c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()];
+	 /*fall through*/
+       case HIPE_MODE_SWITCH_RES_CALL_BEAM:
 	 SET_I(c_p->i);
 	 r(0) = reg[0];
 	 Dispatch();
-- 
cgit v1.2.3


From 88fb069f83c39a47f65809ee5d6e7cadc170e56b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 30 Apr 2014 14:47:24 +0200
Subject: erts: Add erts_internal:map_to_tuple_keys/1

* Used for introspection.
* Will return the internal key tuple if applicable
* Not documented - not for public use
---
 erts/emulator/beam/bif.tab   |  1 +
 erts/emulator/beam/erl_map.c | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 2d888862bf..fbdddf09db 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -157,6 +157,7 @@ bif erts_internal:binary_to_term/2
 bif erts_internal:request_system_task/3
 bif erts_internal:check_process_code/2
 
+bif erts_internal:map_to_tuple_keys/1
 
 # inet_db support
 bif erlang:port_set_data/2
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index fdd2d0c0f6..5e740aacdd 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -58,6 +58,8 @@
  * - maps:size/1
  * - maps:without/2
  *
+ * DEBUG: for sharing calculation
+ * - erts_internal:map_to_tuple_keys/1
  */
 
 /* erlang:map_size/1
@@ -819,3 +821,17 @@ int erts_validate_and_sort_map(map_t* mp)
     }
     return 1;
 }
+
+/*
+ * erts_internal:map_to_tuple_keys/1
+ *
+ * Used in erts_debug:size/1
+ */
+
+BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
+    if (is_map(BIF_ARG_1)) {
+	map_t *mp = (map_t*)map_val(BIF_ARG_1);
+	BIF_RET(mp->keys);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
-- 
cgit v1.2.3


From 8f673dbd8d4cb5c832b6e79bd2304949f4123981 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 28 Apr 2014 15:52:58 +0200
Subject: erts: Fix faulty process suspend assert

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b4b97d7df1..55036a569e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6144,7 +6144,7 @@ suspend_process(Process *c_p, Process *p)
 	if (c_p == p) {
 	    state = erts_smp_atomic32_read_bor_relb(&p->state,
 						    ERTS_PSFLG_SUSPENDED);
-	    ASSERT(state & ERTS_PSFLG_RUNNING);
+	    ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
 	    suspended = (state & ERTS_PSFLG_SUSPENDED) ? -1: 1;
 	}
 	else {
-- 
cgit v1.2.3


From 64cb86b407cf7d0c0f7ac834dd93be6708ed89b1 Mon Sep 17 00:00:00 2001
From: Piotr Nosek <piotr.nosek@erlang-solutions.com>
Date: Thu, 8 May 2014 12:04:14 +0200
Subject: Add Visual Studio macros to erl_driver.h and ei.h

---
 erts/emulator/beam/erl_driver.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 3ecb379326..5ced8c5ca0 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -198,7 +198,7 @@ typedef long long ErlDrvSInt64;
 #error No 64-bit integer type
 #endif
 
-#if defined(__WIN32__)
+#if defined(__WIN32__) || defined(_WIN32)
 typedef ErlDrvUInt ErlDrvSizeT;
 typedef ErlDrvSInt ErlDrvSSizeT;
 #else
-- 
cgit v1.2.3


From 7ff17ddfd337d1c49c1174d8d24b2ce0671b2c9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 12 May 2014 14:26:30 +0200
Subject: BIFs should be considered exported

All BIFs now have stub functions and are exported. For example
in the erlang module:

-export([..., is_list/1, ...]).
.
.
.
is_list(_Term) ->
    erlang:nif_error(undefined).

But erlang:function_exported(erlang, is_list, 1) returns false,
which is weird.

Change erlang:function_exported/3 to return true for BIFs.
---
 erts/emulator/beam/bif.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 06a1230ca0..787a3da1f4 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3984,16 +3984,19 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 
 BIF_RETTYPE function_exported_3(BIF_ALIST_3)
 {
+    int arity;
     if (is_not_atom(BIF_ARG_1) ||
 	is_not_atom(BIF_ARG_2) || 
 	is_not_small(BIF_ARG_3)) {
 	BIF_ERROR(BIF_P, BADARG);
     }
-    if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3),
-			   erts_active_code_ix()) == NULL) {
-	BIF_RET(am_false);
+    arity = signed_val(BIF_ARG_3);
+    if (erts_find_function(BIF_ARG_1, BIF_ARG_2, arity,
+			   erts_active_code_ix()) != NULL ||
+	erts_is_builtin(BIF_ARG_1, BIF_ARG_2, arity)) {
+	BIF_RET(am_true);
     }
-    BIF_RET(am_true);
+    BIF_RET(am_false);
 }
 
 /**********************************************************************/    
-- 
cgit v1.2.3


From 81e77a082ba1de759db17e9cbdb4b8cbe79c0d1b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 12 May 2014 19:03:12 +0200
Subject: erts: Make erlang:display show content of binaries

---
 erts/emulator/beam/erl_printf_term.c | 86 +++++++++++++++++++++++++++++++++---
 1 file changed, 79 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index d18760dc43..1a0c7a9fc9 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -25,6 +25,7 @@
 #include "sys.h"
 #include "big.h"
 #include "erl_map.h"
+#include "erl_binary.h"
 
 #define PRINT_CHAR(CNT, FN, ARG, C)					\
 do {									\
@@ -138,6 +139,25 @@ is_printable_string(Eterm list, Eterm* base)
     return 0;
 }
 
+static int is_printable_ascii(byte* bytep, Uint bytesize, Uint bitoffs)
+{
+    if (!bitoffs) {
+	while (bytesize--) {
+	    if (*bytep < ' ' || *bytep >= 127)
+		return 0;
+	    bytep++;
+	}
+    } else {
+	while (bytesize--) {
+	    byte octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs));
+	    if (octet < ' ' || octet >= 127)
+		return 0;
+	    bytep++;
+	}
+    }
+    return 1;
+}
+
 /* print a atom doing what quoting is necessary */
 static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount)
 {
@@ -446,13 +466,65 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		PRINT_STRING(res, fn, arg, "#MatchState");
 	    }
 	    else {
-		ProcBin* pb = (ProcBin *) binary_val(wobj);
-		if (pb->size == 1)
-		    PRINT_STRING(res, fn, arg, "<<1 byte>>");
-		else {
+		byte* bytep;
+		Uint bytesize = binary_size_rel(obj,obj_base);
+		Uint bitoffs;
+		Uint bitsize;
+		byte octet;
+		ERTS_GET_BINARY_BYTES_REL(obj, bytep, bitoffs, bitsize, obj_base);
+
+		if (bitsize || !bytesize
+		    || !is_printable_ascii(bytep, bytesize, bitoffs)) {
+		    int is_first = 1;
 		    PRINT_STRING(res, fn, arg, "<<");
-		    PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) pb->size);
-		    PRINT_STRING(res, fn, arg, " bytes>>");
+		    while (bytesize) {
+			if (is_first)
+			    is_first = 0;
+			else
+			    PRINT_CHAR(res, fn, arg, ',');
+			if (bitoffs)
+			    octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs));
+			else
+			    octet = bytep[0];
+			PRINT_UWORD(res, fn, arg, 'u', 0, 1, octet);
+			++bytep;
+			--bytesize;
+		    }
+		    if (bitsize) {
+			Uint bits = bitoffs + bitsize;
+			octet = bytep[0];
+			if (bits < 8)
+			    octet >>= 8 - bits;
+			else if (bits > 8) {
+			    bits -= 8;  /* bits in last byte */
+			    octet <<= bits;
+			    octet |= bytep[1] >> (8 - bits);
+			}
+			octet &= (1 << bitsize) - 1;
+			if (is_first)
+			    is_first = 0;
+			else
+			    PRINT_CHAR(res, fn, arg, ',');
+			PRINT_UWORD(res, fn, arg, 'u', 0, 1, octet);
+			PRINT_CHAR(res, fn, arg, ':');
+			PRINT_UWORD(res, fn, arg, 'u', 0, 1, bitsize);
+		    }
+		    PRINT_STRING(res, fn, arg, ">>");
+		}
+		else {
+		    PRINT_STRING(res, fn, arg, "<<\"");
+		    while (bytesize) {
+			if (bitoffs)
+			    octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs));
+			else
+			    octet = bytep[0];
+			if (octet == '"')
+			    PRINT_CHAR(res, fn, arg, '\\');
+			PRINT_CHAR(res, fn, arg, octet);
+			++bytep;
+			--bytesize;
+		    }
+		    PRINT_STRING(res, fn, arg, "\">>");
 		}
 	    }
 	    break;
-- 
cgit v1.2.3


From 94d877a95de7f22fc66f0737da54cc6500a7231c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 2 May 2014 14:57:35 +0200
Subject: HiPE wrappers for BIFs disabling GC

---
 erts/emulator/beam/bif.h      | 45 ++++++++++++++++++++++++++
 erts/emulator/beam/external.c | 74 +++++--------------------------------------
 2 files changed, 53 insertions(+), 66 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 51b77a95ed..669cbfd102 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -392,6 +392,51 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
+#ifndef HIPE
+
+#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)
+
+#else
+
+#include "erl_fun.h"
+#include "hipe_mode_switch.h"
+
+/*
+ * Hipe wrappers used by native code for BIFs that disable GC while trapping.
+ * Also add usage of the wrapper in ../hipe/hipe_bif_list.m4
+ *
+ * Problem:
+ * When native code calls a BIF that traps, hipe_mode_switch will push a
+ * "trap frame" on the Erlang stack in order to find its way back from beam_emu
+ * back to native caller when finally done. If GC is disabled and stack/heap
+ * is full there is no place to push the "trap frame".
+ *
+ * Solution:
+ * We reserve space on stack for the "trap frame" here before the BIF is called.
+ * If the BIF does not trap, the space is reclaimed here before returning.
+ * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame"
+ * already is reserved and use it.
+ */
+
+
+#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)			\
+BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
+						     Eterm* args);	\
+BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
+						     Eterm* args)	\
+{									\
+    BIF_RETTYPE  res;							\
+    hipe_reserve_beam_trap_frame(c_p, args, ARITY);			\
+    res =  BIF_NAME ## _ ## ARITY (c_p, args);				\
+    if (is_value(res) || c_p->freason != TRAP) {			\
+	hipe_unreserve_beam_trap_frame(c_p);				\
+    }									\
+    return res;								\
+}
+
+#endif
+
+
 #include "erl_bif_table.h"
 
 #endif
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 656de7c49a..e376253400 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -44,9 +44,6 @@
 #include "erl_zlib.h"
 #include "erl_map.h"
 
-#ifdef HIPE
-#include "hipe_mode_switch.h"
-#endif
 #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 #define MAX_STRING_LEN 0xffff
@@ -1069,6 +1066,8 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
     }
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 1)
+
 BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
 {
     Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL);
@@ -1081,6 +1080,8 @@ BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
     }
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 2)
+
 BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 {
     Process* p = BIF_P;
@@ -1544,11 +1545,15 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
     BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin);
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 1)
+
 BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1)
 {
     return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 2)
+
 BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2)
 {
     Eterm opts;
@@ -4440,66 +4445,3 @@ error:
 #undef SKIP2
 #undef CHKSIZE
 }
-
-
-#ifdef HIPE
-BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1);
-BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2);
-BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1);
-BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2);
-
-/* Hipe wrappers used by native code for BIFs that disable GC while trapping.
- *
- * Problem:
- * When native code calls a BIF that traps, hipe_mode_switch will push a
- * "trap frame" on the Erlang stack in order to find its way back from beam_emu
- * back to native caller when finally done. If GC is disabled and stack/heap
- * is full there is no place to push the "trap frame".
- *
- * Solution:
- * We reserve space on stack for the "trap frame" here before the BIF is called.
- * If the BIF does not trap, the space is reclaimed here before returning.
- * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame"
- * already is reserved and use it.
- */
-BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1)
-{
-    Eterm res;
-    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1);
-    res = term_to_binary_1(BIF_P, BIF__ARGS);
-    if (is_value(res) || BIF_P->freason != TRAP) {
-	hipe_unreserve_beam_trap_frame(BIF_P);
-    }
-    return res;
-}
-BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2)
-{
-    Eterm res;
-    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2);
-    res = term_to_binary_2(BIF_P, BIF__ARGS);
-    if (is_value(res) || BIF_P->freason != TRAP) {
-	hipe_unreserve_beam_trap_frame(BIF_P);
-    }
-    return res;
-}
-BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1)
-{
-    Eterm res;
-    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1);
-    res = erts_internal_binary_to_term_1(BIF_P, BIF__ARGS);
-    if (is_value(res) || BIF_P->freason != TRAP) {
-	hipe_unreserve_beam_trap_frame(BIF_P);
-    }
-    return res;
-}
-BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2)
-{
-    Eterm res;
-    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2);
-    res = erts_internal_binary_to_term_2(BIF_P, BIF__ARGS);
-    if (is_value(res) || BIF_P->freason != TRAP) {
-	hipe_unreserve_beam_trap_frame(BIF_P);
-    }
-    return res;
-}
-#endif /*HIPE*/
-- 
cgit v1.2.3


From a6c681cf14810a007a2bd4ad0066b4403460bffc Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 7 May 2014 21:27:54 +0200
Subject: Add support for failing in BIF that has trapped

---
 erts/emulator/beam/bif.h | 73 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 669cbfd102..72c55ccb55 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -124,12 +124,85 @@ do {									\
     return THE_NON_VALUE; 			\
 } while(0)
 
+#define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif)		\
+do {								\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    return THE_NON_VALUE; 					\
+} while (0)
+
+#define ERTS_BIF_ERROR_TRAPPED1(Proc, Reason, Bif, A0)		\
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    reg[0] = (Eterm) (A0);					\
+    return THE_NON_VALUE; 					\
+} while (0)
+
+#define ERTS_BIF_ERROR_TRAPPED2(Proc, Reason, Bif, A0, A1)	\
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    reg[0] = (Eterm) (A0);					\
+    reg[1] = (Eterm) (A1);					\
+    return THE_NON_VALUE; 					\
+} while (0)
+
+#define ERTS_BIF_ERROR_TRAPPED3(Proc, Reason, Bif, A0, A1, A2)	\
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    reg[0] = (Eterm) (A0);					\
+    reg[1] = (Eterm) (A1);					\
+    reg[2] = (Eterm) (A2);					\
+    return THE_NON_VALUE; 					\
+} while (0)
+
 #define ERTS_BIF_PREP_ERROR(Ret, Proc, Reason)	\
 do {						\
     (Proc)->freason = (Reason);			\
     (Ret) = THE_NON_VALUE;			\
 } while (0)
 
+#define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif)	\
+do {								\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    (Ret) = THE_NON_VALUE;					\
+} while (0)
+
+#define ERTS_BIF_PREP_ERROR_TRAPPED1(Ret, Proc, Reason, Bif, A0) \
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    reg[0] = (Eterm) (A0);					\
+    (Ret) = THE_NON_VALUE;					\
+} while (0)
+
+#define ERTS_BIF_PREP_ERROR_TRAPPED2(Ret, Proc, Reason, Bif, A0, A1) \
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    reg[0] = (Eterm) (A0);					\
+    reg[1] = (Eterm) (A1);					\
+    (Ret) = THE_NON_VALUE;					\
+} while (0)
+
+#define ERTS_BIF_PREP_ERROR_TRAPPED3(Ret, Proc, Reason, Bif, A0, A1, A2) \
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->freason = (Reason);					\
+    (Proc)->current = (Bif)->code;				\
+    reg[0] = (Eterm) (A0);					\
+    reg[1] = (Eterm) (A1);					\
+    reg[2] = (Eterm) (A2);					\
+    (Ret) = THE_NON_VALUE;					\
+} while (0)
 
 #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc)	\
 do {						\
-- 
cgit v1.2.3


From fdb350a4c98f070c264b8bd4ea9eca2513352f21 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Wed, 14 May 2014 14:46:46 +0200
Subject: Add 'md5' entry for module_info/0/1 functions.

---
 erts/emulator/beam/atom.names  |  1 +
 erts/emulator/beam/beam_load.c | 56 ++++++++++++++++++++++++++++++++++++------
 erts/emulator/beam/beam_load.h |  8 ++++--
 3 files changed, 56 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index d28e519ae1..7d11f7f956 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -333,6 +333,7 @@ atom max
 atom maximum
 atom max_tables max_processes
 atom mbuf_size
+atom md5
 atom memory
 atom memory_internal
 atom memory_types
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index e96177cfd9..5c98d5c0b5 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -245,7 +245,7 @@ typedef struct {
 /*
  * This structure contains all information about the module being loaded.
  */  
-
+#define MD5_SIZE 16
 typedef struct LoaderState {
     /*
      * The current logical file within the binary.
@@ -292,7 +292,7 @@ typedef struct LoaderState {
     StringPatch* string_patches; /* Linked list of position into string table to patch. */
     BeamInstr catches;		/* Linked list of catch_yf instructions. */
     unsigned loaded_size;	/* Final size of code when loaded. */
-    byte mod_md5[16];		/* MD5 for module code. */
+    byte mod_md5[MD5_SIZE];	/* MD5 for module code. */
     int may_load_nif;           /* true if NIFs may later be loaded for this module */  
     int on_load;		/* Index in the code for the on_load function
 				 * (or 0 if there is no on_load function)
@@ -528,6 +528,7 @@ static Eterm exported_from_module(Process* p, Eterm mod);
 static Eterm functions_in_module(Process* p, Eterm mod);
 static Eterm attributes_for_module(Process* p, Eterm mod);
 static Eterm compilation_info_for_module(Process* p, Eterm mod);
+static Eterm md5_of_module(Process* p, Eterm mod);
 static Eterm native_addresses(Process* p, Eterm mod);
 int patch_funentries(Eterm Patchlist);
 int patch(Eterm Addresses, Uint fe);
@@ -648,6 +649,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
     stp->code[MI_COMPILE_PTR] = 0;
     stp->code[MI_COMPILE_SIZE] = 0;
     stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
+    stp->code[MI_MD5_PTR] = 0;
 
     /*
      * Read the atom table.
@@ -4038,7 +4040,7 @@ freeze_code(LoaderState* stp)
     }
     size = (stp->ci * sizeof(BeamInstr)) +
 	(stp->total_literal_size * sizeof(Eterm)) +
-	strtab_size + attr_size + compile_size + line_size;
+	strtab_size + attr_size + compile_size + MD5_SIZE + line_size;
 
     /*
      * Move the code to its final location.
@@ -4247,11 +4249,20 @@ freeze_code(LoaderState* stp)
 	code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size;
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
+    {
+	byte* md5_sum = str_table + strtab_size + attr_size + compile_size;
+	CHKBLK(ERTS_ALC_T_CODE,code);
+	sys_memcpy(md5_sum, stp->mod_md5, MD5_SIZE);
+	CHKBLK(ERTS_ALC_T_CODE,code);
+	code[MI_MD5_PTR] = (BeamInstr) md5_sum;
+	CHKBLK(ERTS_ALC_T_CODE,code);
+    }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Make sure that we have not overflowed the allocated code space.
      */
-    ASSERT(str_table + strtab_size + attr_size + compile_size ==
+    ASSERT(str_table + strtab_size + attr_size + compile_size + MD5_SIZE ==
 	   ((byte *) code) + size);
 
     /*
@@ -5103,6 +5114,7 @@ erts_module_info_0(Process* p, Eterm module)
     hp += 3; \
     list = CONS(hp, tup, list)
 
+    BUILD_INFO(am_md5);
     BUILD_INFO(am_compile);
     BUILD_INFO(am_attributes);
     BUILD_INFO(am_imports);
@@ -5118,6 +5130,8 @@ erts_module_info_1(Process* p, Eterm module, Eterm what)
 	return module;
     } else if (what == am_imports) {
 	return NIL;
+    } else if (what == am_md5) {
+	return md5_of_module(p, module);
     } else if (what == am_exports) {
 	return exported_from_module(p, module);
     } else if (what == am_functions) {
@@ -5306,7 +5320,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
     Eterm result = NIL;
     Eterm* end;
 
-    if (is_not_atom(mod) || (is_not_list(result) && is_not_nil(result))) {
+    if (is_not_atom(mod)) {
 	return THE_NON_VALUE;
     }
 
@@ -5345,7 +5359,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
     Eterm result = NIL;
     Eterm* end;
 
-    if (is_not_atom(mod) || (is_not_list(result) && is_not_nil(result))) {
+    if (is_not_atom(mod)) {
 	return THE_NON_VALUE;
     }
 
@@ -5367,6 +5381,33 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
     return result;
 }
 
+/*
+ * Returns the MD5 checksum for a module
+ *
+ * Returns a tagged term, or 0 on error.
+ */
+
+Eterm
+md5_of_module(Process* p, /* Process whose heap to use. */
+              Eterm mod) /* Tagged atom for module. */
+{
+    Module* modp;
+    BeamInstr* code;
+    Eterm res = NIL;
+
+    if (is_not_atom(mod)) {
+	return THE_NON_VALUE;
+    }
+
+    modp = erts_get_module(mod, erts_active_code_ix());
+    if (modp == NULL) {
+	return THE_NON_VALUE;
+    }
+    code = modp->curr.code;
+    res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE);
+    return res;
+}
+
 /*
  * Build a single {M,F,A,Loction} item to be part of
  * a stack trace.
@@ -5543,7 +5584,7 @@ code_module_md5_1(BIF_ALIST_1)
 	res = am_undefined;
 	goto done;
     }
-    res = new_binary(p, stp->mod_md5, sizeof(stp->mod_md5));
+    res = new_binary(p, stp->mod_md5, MD5_SIZE);
 
  done:
     erts_free_aligned_binary_bytes(temp_alloc);
@@ -5939,6 +5980,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     code[MI_LITERALS_END] = 0;
     code[MI_LITERALS_OFF_HEAP] = 0;
     code[MI_ON_LOAD_FUNCTION_PTR] = 0;
+    code[MI_MD5_PTR] = 0;
     ci = MI_FUNCTIONS + n + 1;
 
     /*
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index bd22b0c4de..0e3ca0bdb0 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -91,7 +91,6 @@ extern Uint erts_total_code_size;
 #define MI_LITERALS_END		8
 #define MI_LITERALS_OFF_HEAP	9
 
-
 /*
  * Pointer to the on_load function (or NULL if none).
  */
@@ -102,6 +101,11 @@ extern Uint erts_total_code_size;
  */
 #define MI_LINE_TABLE 11
 
+/*
+ * Pointer to the module MD5 sum (16 bytes)
+ */
+#define MI_MD5_PTR 12
+
 /*
  * Start of function pointer table.  This table contains pointers to
  * all functions in the module plus an additional pointer just beyond
@@ -111,7 +115,7 @@ extern Uint erts_total_code_size;
  * this table.
  */
 
-#define MI_FUNCTIONS         12
+#define MI_FUNCTIONS         13
 
 /*
  * Layout of the line table.
-- 
cgit v1.2.3


From 4ad572a8b66df8174d92157411326aab95ee8124 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Wed, 14 May 2014 14:49:10 +0200
Subject: Add 'module' entry for module_info/0 function for completeness

as it exist in module_info/1.
---
 erts/emulator/beam/beam_load.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 5c98d5c0b5..8b3fc90826 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5119,6 +5119,7 @@ erts_module_info_0(Process* p, Eterm module)
     BUILD_INFO(am_attributes);
     BUILD_INFO(am_imports);
     BUILD_INFO(am_exports);
+    BUILD_INFO(am_module);
 #undef BUILD_INFO
     return list;
 }
-- 
cgit v1.2.3


From 0dbcbea0cf52672ef1cf051c3cd7640eb7ff728e Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Wed, 14 May 2014 14:54:16 +0200
Subject: Remove obsolete 'imports' entry from module_info/1/2 functions

It previously always returned an empty list
and was documented to be removed.
---
 erts/emulator/beam/atom.names  | 1 -
 erts/emulator/beam/beam_load.c | 3 ---
 2 files changed, 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 7d11f7f956..dc930f2d38 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -278,7 +278,6 @@ atom http httph https http_response http_request http_header http_eoh http_error
 atom id
 atom if_clause
 atom ignore
-atom imports
 atom in
 atom in_exiting
 atom inactive
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 8b3fc90826..a4e72a130a 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5117,7 +5117,6 @@ erts_module_info_0(Process* p, Eterm module)
     BUILD_INFO(am_md5);
     BUILD_INFO(am_compile);
     BUILD_INFO(am_attributes);
-    BUILD_INFO(am_imports);
     BUILD_INFO(am_exports);
     BUILD_INFO(am_module);
 #undef BUILD_INFO
@@ -5129,8 +5128,6 @@ erts_module_info_1(Process* p, Eterm module, Eterm what)
 {
     if (what == am_module) {
 	return module;
-    } else if (what == am_imports) {
-	return NIL;
     } else if (what == am_md5) {
 	return md5_of_module(p, module);
     } else if (what == am_exports) {
-- 
cgit v1.2.3


From f19fe9b2ada7824973660b41da0873825c5781ca Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 5 May 2014 17:46:07 +0200
Subject: Fix conversion of empty string in erts_convert_native_to_filename()

---
 erts/emulator/beam/erl_unicode.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 3a968594f3..f8e1431a53 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2126,6 +2126,8 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
 	mac = 1;
     case ERL_FILENAME_UTF8:
 	size = strlen((char *) bytes);
+	if (size == 0)
+	    return NIL;
 	if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
 	    goto noconvert;
 	}
-- 
cgit v1.2.3


From e291c1776cff0446435b392285b7ec6abe5373bd Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 25 Apr 2014 21:37:17 +0200
Subject: Make binary BIFs converting to lists yield on large input

- erlang:binary_to_list/1
- erlang:binary_to_list/3
- erlang:bitstring_to_list/1
---
 erts/emulator/beam/atom.names |   1 +
 erts/emulator/beam/binary.c   | 149 ++++++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_init.c |   2 +-
 3 files changed, 146 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index d28e519ae1..d6e312fafd 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -116,6 +116,7 @@ atom binary_longest_prefix_trap
 atom binary_longest_suffix_trap
 atom binary_match_trap
 atom binary_matches_trap
+atom binary_to_list_continue
 atom binary_to_term_trap
 atom block
 atom blocked
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index c7926f18af..4fd47210a2 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -38,6 +38,11 @@ static int list_to_bitstr_buf(Eterm obj, char* buf);
 #endif
 static int bitstr_list_len(Eterm obj, Uint* num_bytes);
 
+static Export binary_to_list_continue_export;
+
+static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1);
+
+
 void
 erts_init_binary(void)
 {
@@ -49,6 +54,11 @@ erts_init_binary(void)
 		 "Internal error: Address of orig_bytes[0] of a Binary"
 		 " is *not* 8-byte aligned\n");
     }
+
+    erts_init_trap_export(&binary_to_list_continue_export,
+			  am_erts_internal, am_binary_to_list_continue, 1,
+			  &binary_to_list_continue);
+
 }
 
 /*
@@ -333,6 +343,132 @@ BIF_RETTYPE integer_to_binary_1(BIF_ALIST_1)
     BIF_RET(res);
 }
 
+#define ERTS_B2L_BYTES_PER_REDUCTION 256
+
+typedef struct {
+    Eterm res;
+    Eterm *hp;
+#ifdef DEBUG
+    Eterm *hp_end;
+#endif
+    byte *bytes;
+    Uint size;
+    Uint bitoffs;
+} ErtsB2LState;
+
+static void b2l_state_destructor(Binary *mbp)
+{
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor);
+}
+
+static BIF_RETTYPE
+binary_to_list_chunk(Process *c_p,
+		     Eterm mb_eterm,
+		     ErtsB2LState* sp,
+		     int reds_left,
+		     int gc_disabled)
+{
+    BIF_RETTYPE ret;
+    int bump_reds;
+    Uint size;
+    byte *bytes;
+
+    size = (reds_left + 1)*ERTS_B2L_BYTES_PER_REDUCTION;
+    if (size > sp->size)
+	size = sp->size;
+    bytes = sp->bytes + (sp->size - size);
+
+    bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1;
+    BUMP_REDS(c_p, bump_reds);
+
+    ASSERT(is_list(sp->res) || is_nil(sp->res));
+
+    sp->res = erts_bin_bytes_to_list(sp->res,
+				     sp->hp,
+				     bytes,
+				     size,
+				     sp->bitoffs);
+    sp->size -= size;
+    sp->hp += 2*size;
+
+    if (sp->size > 0) {
+
+	if (!gc_disabled)
+	    erts_set_gc_state(c_p, 0);
+
+	ASSERT(c_p->flags & F_DISABLE_GC);
+	ASSERT(is_value(mb_eterm));
+	ERTS_BIF_PREP_TRAP1(ret,
+			    &binary_to_list_continue_export,
+			    c_p,
+			    mb_eterm);
+    }
+    else {
+
+	ASSERT(sp->hp == sp->hp_end);
+	ASSERT(sp->size == 0);
+
+	if (!gc_disabled || !erts_set_gc_state(c_p, 1))
+	    ERTS_BIF_PREP_RET(ret, sp->res);
+	else
+	    ERTS_BIF_PREP_YIELD_RETURN(ret, c_p, sp->res);
+	ASSERT(!(c_p->flags & F_DISABLE_GC));
+    }
+
+    return ret;
+}
+
+static ERTS_INLINE BIF_RETTYPE
+binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes, Uint size, Uint bitoffs)
+{
+    int reds_left = ERTS_BIF_REDS_LEFT(c_p);
+    if (size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION) {
+	Eterm res;
+	BIF_RETTYPE ret;
+	int bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1;
+	BUMP_REDS(c_p, bump_reds);
+	res = erts_bin_bytes_to_list(tail, hp, bytes, size, bitoffs);
+	ERTS_BIF_PREP_RET(ret, res);
+	return ret;
+    }
+    else {
+	Binary *mbp = erts_create_magic_binary(sizeof(ErtsB2LState),
+					       b2l_state_destructor);
+	ErtsB2LState *sp = ERTS_MAGIC_BIN_DATA(mbp);
+	Eterm mb;
+
+	sp->res = tail;
+	sp->hp = hp;
+#ifdef DEBUG
+	sp->hp_end = sp->hp + 2*size;
+#endif
+	sp->bytes = bytes;
+	sp->size = size;
+	sp->bitoffs = bitoffs;
+
+	hp = HAlloc(c_p, PROC_BIN_SIZE);
+	mb = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp);
+	return binary_to_list_chunk(c_p, mb, sp, reds_left, 0);
+    }
+}
+
+static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1)
+{
+    Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor);
+
+    ASSERT(BIF_P->flags & F_DISABLE_GC);
+
+    return binary_to_list_chunk(BIF_P,
+				BIF_ARG_1,
+				(ErtsB2LState*) ERTS_MAGIC_BIN_DATA(mbp),
+				ERTS_BIF_REDS_LEFT(BIF_P),
+				1);
+}
+
+HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_list, 1)
+
 BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
 {
     Eterm real_bin;
@@ -354,14 +490,15 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
     } else {
 	Eterm* hp = HAlloc(BIF_P, 2 * size);
 	byte* bytes = binary_bytes(real_bin)+offset;
-
-	BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+	return binary_to_list(BIF_P, hp, NIL, bytes, size, bitoffs);
     }
 
     error:
 	BIF_ERROR(BIF_P, BADARG);
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_list, 3)
+
 BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
 {
     byte* bytes;
@@ -387,12 +524,13 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
     }
     i = stop-start+1;
     hp = HAlloc(BIF_P, 2*i);
-    BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
-    
+    return binary_to_list(BIF_P, hp, NIL, bytes+start-1, i, bitoffs);
     error:
 	BIF_ERROR(BIF_P, BADARG);
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(bitstring_to_list, 1)
+
 BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 {
     Eterm real_bin;
@@ -431,7 +569,8 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 	previous = CONS(hp, make_binary(last), previous);
 	hp += 2;
     }
-    BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
+
+    return binary_to_list(BIF_P, hp, previous, bytes, size, bitoffs);
 }
 
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index d54658f1ea..5e6d812242 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -324,7 +324,6 @@ erl_init(int ncpu,
     BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0);
 
     erts_init_trace();
-    erts_init_binary();
     erts_init_bits();
     erts_code_ix_init();
     erts_init_fun_table();
@@ -337,6 +336,7 @@ erl_init(int ncpu,
     erts_ddll_init();
     init_emulator();
     erts_ptab_init(); /* Must be after init_emulator() */
+    erts_init_binary(); /* Must be after init_emulator() */
     erts_bp_init();
     init_db(); /* Must be after init_emulator */
     erts_bif_timer_init();
-- 
cgit v1.2.3


From 3c1e5f014d8c957cfc085d200ca744bd4ad2a7bb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 13 May 2014 16:10:30 +0200
Subject: Do not GC other processes in non-smp runtime

---
 erts/emulator/beam/erl_message.c | 37 -------------------------------------
 1 file changed, 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 0eb8117980..59a677a12c 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1032,7 +1032,6 @@ erts_send_message(Process* sender,
 	}
         BM_SWAP_TIMER(send,system);
     } else {
-#ifdef ERTS_SMP
 	ErlOffHeap *ohp;
         Eterm *hp;
 	erts_aint32_t state;
@@ -1064,42 +1063,6 @@ erts_send_message(Process* sender,
 #endif
 	    );
         BM_SWAP_TIMER(send,system);
-#else
-	ErlMessage* mp = message_alloc();
-        Eterm *hp;
-        BM_SWAP_TIMER(send,size);
-	msize = size_object(message);
-        BM_SWAP_TIMER(size,send);
-	
-	if (receiver->stop - receiver->htop <= msize) {
-            BM_SWAP_TIMER(send,system);
-	    erts_garbage_collect(receiver, msize, receiver->arg_reg, receiver->arity);
-            BM_SWAP_TIMER(system,send);
-	}
-	hp = receiver->htop;
-	receiver->htop = hp + msize;
-        BM_SWAP_TIMER(send,copy);
-	message = copy_struct(message, msize, &hp, &receiver->off_heap);
-	BM_MESSAGE_COPIED(msize);
-        BM_SWAP_TIMER(copy,send);
-        DTRACE6(message_send, sender_name, receiver_name,
-                (uint32_t)msize, tok_label, tok_lastcnt, tok_serial);
-	ERL_MESSAGE_TERM(mp) = message;
-	ERL_MESSAGE_TOKEN(mp) = NIL;
-#ifdef USE_VM_PROBES
-	ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
-	mp->next = NULL;
-	mp->data.attached = NULL;
-	LINK_MESSAGE(receiver, mp);
-	res = receiver->msg.len;
-	erts_proc_notify_new_message(receiver);
-
-	if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
-	    trace_receive(receiver, message);
-	}
-        BM_SWAP_TIMER(send,system);
-#endif /* #ifndef ERTS_SMP */
     }
    return res;
 }
-- 
cgit v1.2.3


From c15cc0737b70ff5d4d1b79e8171106fcafcb1dda Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 28 Apr 2014 23:55:12 +0200
Subject: Make binary BIFs converting from lists yield on large input

- erlang:list_to_binary/1
- erlang:iolist_to_binary/1
- erlang:list_to_bitstring/1
- binary:list_to_bin/1
---
 erts/emulator/beam/atom.names       |    1 +
 erts/emulator/beam/binary.c         | 1012 +++++++++++++++++++++++++++--------
 erts/emulator/beam/erl_bif_binary.c |   11 +-
 erts/emulator/beam/erl_binary.h     |    2 +-
 erts/emulator/beam/global.h         |   53 +-
 erts/emulator/beam/utils.c          |  490 +++++++++++++----
 6 files changed, 1231 insertions(+), 338 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index d6e312fafd..5d06a32941 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -316,6 +316,7 @@ atom line_length
 atom linked_in_driver
 atom links
 atom list
+atom list_to_binary_continue
 atom little
 atom loaded
 atom load_cancelled
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 4fd47210a2..f50d484576 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -31,17 +31,11 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 
-#ifdef DEBUG
-static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len);
-#else
-static int list_to_bitstr_buf(Eterm obj, char* buf);
-#endif
-static int bitstr_list_len(Eterm obj, Uint* num_bytes);
-
 static Export binary_to_list_continue_export;
+static Export list_to_binary_continue_export;
 
 static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1);
-
+static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1);
 
 void
 erts_init_binary(void)
@@ -59,6 +53,10 @@ erts_init_binary(void)
 			  am_erts_internal, am_binary_to_list_continue, 1,
 			  &binary_to_list_continue);
 
+    erts_init_trap_export(&list_to_binary_continue_export,
+			  am_erts_internal, am_list_to_binary_continue, 1,
+			  &list_to_binary_continue);
+
 }
 
 /*
@@ -577,117 +575,433 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 /* Turn a possibly deep list of ints (and binaries) into */
 /* One large binary object                               */
 
-/*
- * This bif also exists in the binary module, under the name
- * binary:list_to_bin/1, why it's divided into interface and
- * implementation. Also the backend for iolist_to_binary_1.
- */
+typedef enum {
+    ERTS_L2B_OK,
+    ERTS_L2B_YIELD,
+    ERTS_L2B_TYPE_ERROR,
+    ERTS_L2B_OVERFLOW_ERROR
+} ErtsL2BResult;
 
-BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
-{
+#define ERTS_L2B_STATE_INITER(C_P, ARG, BIF, SZFunc, TBufFunc)	\
+    {ERTS_IOLIST2BUF_STATE_INITER((C_P), (ARG)),				\
+	    (ARG), THE_NON_VALUE, (BIF), (SZFunc), (TBufFunc)}
+
+#define ERTS_L2B_STATE_MOVE(TO, FROM) \
+    sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsL2BState))
+
+typedef struct ErtsL2BState_ ErtsL2BState;
+
+struct ErtsL2BState_ {
+    ErtsIOList2BufState buf;
+    Eterm arg;
     Eterm bin;
-    Eterm h,t;
-    ErlDrvSizeT size;
-    byte* bytes;
-#ifdef DEBUG
-    ErlDrvSizeT offset;
-#endif
+    Export *bif;
+    int (*iolist_to_buf_size)(ErtsIOListState *);
+    ErlDrvSizeT (*iolist_to_buf)(ErtsIOList2BufState *);
+};
+
+static ERTS_INLINE ErtsL2BResult
+list_to_binary_engine(ErtsL2BState *sp)
+{
+    ErlDrvSizeT res;
+    Process *c_p = sp->buf.iolist.c_p;
+
+    /*
+     * have_size == 0 while sp->iolist_to_buf_size()
+     * has not finished the calculation.
+     */
+
+    if (!sp->buf.iolist.have_size) {
+	switch (sp->iolist_to_buf_size(&sp->buf.iolist)) {
+	case ERTS_IOLIST_YIELD:
+	    return ERTS_L2B_YIELD;
+	case ERTS_IOLIST_OVERFLOW:
+	    return ERTS_L2B_OVERFLOW_ERROR;
+	case ERTS_IOLIST_TYPE:
+	    return ERTS_L2B_TYPE_ERROR;
+	case ERTS_IOLIST_OK:
+	    break;
+	default:
+	    ASSERT(0);
+	    break;
+	}
+
+	ASSERT(sp->buf.iolist.have_size);
+
+	/*
+	 * Size calculated... Setup state for
+	 * sp->iolist_to_buf_*()
+	 */
+
+	sp->bin = new_binary(c_p,
+			     (byte *) NULL,
+			     sp->buf.iolist.size);
+
+	if (sp->buf.iolist.size == 0)
+	    return ERTS_L2B_OK;
+
+	sp->buf.buf = (char *) binary_bytes(sp->bin);
+	sp->buf.len = sp->buf.iolist.size;
+	sp->buf.iolist.obj = sp->arg;
 
-    if (is_nil(arg)) {
-	BIF_RET(new_binary(p,(byte*)"",0));
+	if (sp->buf.iolist.reds_left <= 0) {
+	    BUMP_ALL_REDS(c_p);
+	    return ERTS_L2B_YIELD;
+	}
     }
-    if (is_not_list(arg)) {
-	goto error;
+
+    ASSERT(sp->buf.iolist.size != 0);
+    ASSERT(is_value(sp->bin));
+    ASSERT(sp->buf.buf);
+
+    res = sp->iolist_to_buf(&sp->buf);
+
+    if (!ERTS_IOLIST_TO_BUF_FAILED(res)) {
+	ASSERT(res == 0);
+	return ERTS_L2B_OK;
     }
-    /* check for [binary()] case */
-    h = CAR(list_val(arg));
-    t = CDR(list_val(arg));
-    if (is_binary(h) && is_nil(t) && !(
-		HEADER_SUB_BIN == *(binary_val(h)) && (
-		    ((ErlSubBin *)binary_val(h))->bitoffs != 0 ||
-		    ((ErlSubBin *)binary_val(h))->bitsize != 0
-		))) {
-	return h;
-    }
-    switch (erts_iolist_size(arg, &size)) {
-    case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT);
-    case ERTS_IOLIST_TYPE: goto error;
-    default: ;
-    }
-    bin = new_binary(p, (byte *)NULL, size);
-    bytes = binary_bytes(bin);
-#ifdef DEBUG
-    offset = 
-#endif
-	erts_iolist_to_buf(arg, (char*) bytes, size);
 
-    ASSERT(offset == 0);
-    BIF_RET(bin);
+    switch (res) {
+    case ERTS_IOLIST_TO_BUF_YIELD:
+	return ERTS_L2B_YIELD;
+    case ERTS_IOLIST_TO_BUF_OVERFLOW:
+	return ERTS_L2B_OVERFLOW_ERROR;
+    case ERTS_IOLIST_TO_BUF_TYPE_ERROR:
+	return ERTS_L2B_TYPE_ERROR;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid return value from iolist_to_buf_yielding()");
+	return ERTS_L2B_TYPE_ERROR;
+    }
+}
+
+static void
+l2b_state_destructor(Binary *mbp)
+{
+    ErtsL2BState *sp = ERTS_MAGIC_BIN_DATA(mbp); 
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor);
+    DESTROY_SAVED_ESTACK(&sp->buf.iolist.estack);
+}
+
+static ERTS_INLINE Eterm
+l2b_final_touch(Process *c_p, ErtsL2BState *sp)
+{
+    Eterm *hp;
+    ErlSubBin* sbin; 
+    if (sp->buf.offset == 0)
+	return sp->bin;
+
+    hp = HAlloc(c_p, ERL_SUB_BIN_SIZE);
+    ASSERT(sp->buf.offset > 0);
+    sbin = (ErlSubBin *) hp;
+    sbin->thing_word = HEADER_SUB_BIN;
+    sbin->size = sp->buf.iolist.size-1;
+    sbin->offs = 0;
+    sbin->orig = sp->bin;
+    sbin->bitoffs = 0;
+    sbin->bitsize = sp->buf.offset;
+    sbin->is_writable = 0;
+    return make_binary(sbin);
+}
+
+static BIF_RETTYPE
+list_to_binary_chunk(Eterm mb_eterm,
+		     ErtsL2BState* sp,
+		     int reds_left,
+		     int gc_disabled)
+{
+    Eterm err = BADARG;
+    BIF_RETTYPE ret;
+    Process *c_p = sp->buf.iolist.c_p;
+
+    sp->buf.iolist.reds_left = reds_left;
     
- error:
-    BIF_ERROR(p, BADARG);
+    switch (list_to_binary_engine(sp)) {
+
+    case ERTS_L2B_OK: {
+	Eterm result = l2b_final_touch(c_p, sp);
+	if (!gc_disabled || !erts_set_gc_state(c_p, 1))
+	    ERTS_BIF_PREP_RET(ret, result);
+	else
+	    ERTS_BIF_PREP_YIELD_RETURN(ret, c_p, result);
+	ASSERT(!(c_p->flags & F_DISABLE_GC));
+	break;
+    }
+    case ERTS_L2B_YIELD:
+	if (!gc_disabled) {
+	    /* first yield... */
+	    Eterm *hp;
+	    Binary *mbp = erts_create_magic_binary(sizeof(ErtsL2BState),
+						   l2b_state_destructor);
+	    ErtsL2BState *new_sp = ERTS_MAGIC_BIN_DATA(mbp);
+
+	    ERTS_L2B_STATE_MOVE(new_sp, sp);
+	    sp = new_sp;
+
+	    hp = HAlloc(c_p, PROC_BIN_SIZE);
+	    mb_eterm = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp);
+
+	    ASSERT(is_value(mb_eterm));
+
+	    erts_set_gc_state(c_p, 0);
+	}
+
+	ASSERT(c_p->flags & F_DISABLE_GC);
+
+	ERTS_BIF_PREP_TRAP1(ret,
+			    &list_to_binary_continue_export,
+			    c_p,
+			    mb_eterm);
+	break;
+
+    case ERTS_L2B_OVERFLOW_ERROR:
+	err = SYSTEM_LIMIT;
+	/* fall through */
+
+    case ERTS_L2B_TYPE_ERROR:
+	if (!gc_disabled)
+	    ERTS_BIF_PREP_ERROR(ret, c_p, err);
+	else {
+	    if (erts_set_gc_state(c_p, 1))
+		ERTS_VBUMP_ALL_REDS(c_p);
+
+	    ERTS_BIF_PREP_ERROR_TRAPPED1(ret,
+					 c_p,
+					 err,
+					 sp->bif,
+					 sp->arg);
+	}
+
+	ASSERT(!(c_p->flags & F_DISABLE_GC));
+	break;
+
+    default:
+	ERTS_INTERNAL_ERROR("Invalid return value from list_to_binary_engine()");
+	ERTS_BIF_PREP_ERROR(ret,c_p, EXC_INTERNAL_ERROR);
+	break;
+    }
+    return ret;
+}
+
+static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1)
+{
+    Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor);
+
+    ASSERT(BIF_P->flags & F_DISABLE_GC);
+
+    return list_to_binary_chunk(BIF_ARG_1,
+				ERTS_MAGIC_BIN_DATA(mbp),
+				ERTS_BIF_REDS_LEFT(BIF_P),
+				1);
 }
 
+BIF_RETTYPE erts_list_to_binary_bif(Process *c_p, Eterm arg, Export *bif)
+{
+    BIF_RETTYPE ret;
+
+    if (is_nil(arg))
+	ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) "", 0));
+    else if (is_not_list(arg))
+	ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+    else {
+	/* check for [binary()] case */
+	Eterm h = CAR(list_val(arg));
+	Eterm t = CDR(list_val(arg));
+	if (is_binary(h)
+	    && is_nil(t)
+	    && !(HEADER_SUB_BIN == *(binary_val(h))
+		 && (((ErlSubBin *)binary_val(h))->bitoffs != 0
+		     || ((ErlSubBin *)binary_val(h))->bitsize != 0))) {
+	    ERTS_BIF_PREP_RET(ret, h);
+	}
+	else {
+	    ErtsL2BState state = ERTS_L2B_STATE_INITER(c_p,
+						       arg,
+						       bif,
+						       erts_iolist_size_yielding,
+						       erts_iolist_to_buf_yielding);
+	    int orig_reds_left = ERTS_BIF_REDS_LEFT(c_p);
+
+	    /*
+	     * First try to do it all at once without having to use
+	     * yielding iolist_to_buf().
+	     */
+	    state.buf.iolist.reds_left = orig_reds_left;
+	    switch (erts_iolist_size_yielding(&state.buf.iolist)) {
+	    case ERTS_IOLIST_OK: {
+		ErlDrvSizeT size = state.buf.iolist.size;
+		Eterm bin;
+		char *buf;
+
+		if (size == 0) {
+		    ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) NULL, 0));
+		    break; /* done */
+		}
+
+		bin = new_binary(c_p, (byte *) NULL, size);
+		buf = (char *) binary_bytes(bin);
+
+		if (size < ERTS_IOLIST_TO_BUF_BYTES_PER_RED*CONTEXT_REDS) {
+		    /* An (over) estimation of reductions  needed */
+		    int reds_left = state.buf.iolist.reds_left;
+		    int to_buf_reds = orig_reds_left - reds_left;
+		    to_buf_reds += size/ERTS_IOLIST_TO_BUF_BYTES_PER_RED;
+		    if (to_buf_reds <= reds_left) {
+			ErlDrvSizeT res;
+
+			res = erts_iolist_to_buf(arg, buf, size);
+			if (res == 0) {
+			    BUMP_REDS(c_p, to_buf_reds);
+			    ERTS_BIF_PREP_RET(ret, bin);
+			    break; /* done */
+			}
+			if (!ERTS_IOLIST_TO_BUF_FAILED(res))
+			    ERTS_INTERNAL_ERROR("iolist_size/iolist_to_buf missmatch");
+			if (res == ERTS_IOLIST_TO_BUF_OVERFLOW)
+			    goto overflow;
+			goto type_error;
+		    }
+		}
+		/*
+		 * Since size has been computed list_to_binary_chunk() expects
+		 * state prepared for iolist_to_buf.
+		 */
+		state.bin = bin;
+		state.buf.buf = buf;
+		state.buf.len = size;
+		state.buf.iolist.obj = arg;
+		/* Fall through... */
+	    }
+	    case ERTS_IOLIST_YIELD:
+		ret = list_to_binary_chunk(THE_NON_VALUE,
+					   &state,
+					   state.buf.iolist.reds_left,
+					   0);
+		break;
+	    case ERTS_IOLIST_OVERFLOW:
+	    overflow:
+		ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT);
+		break;
+	    case ERTS_IOLIST_TYPE:
+	    type_error:
+	    default:
+		ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+		break;
+	    }
+	}
+    }
+    return ret;
+}
+
+HIPE_WRAPPER_BIF_DISABLE_GC(list_to_binary, 1)
+
 BIF_RETTYPE list_to_binary_1(BIF_ALIST_1)
 {
-    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_list_to_binary_1]);
 }
 
-/* Turn a possibly deep list of ints (and binaries) into */
-/* One large binary object                               */
+HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1)
 
 BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1)
 {
     if (is_binary(BIF_ARG_1)) {
 	BIF_RET(BIF_ARG_1);
     }
-    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]);
 }
 
+static int bitstr_list_len(ErtsIOListState *);
+static ErlDrvSizeT list_to_bitstr_buf_yielding(ErtsIOList2BufState *);
+static ErlDrvSizeT list_to_bitstr_buf_not_yielding(ErtsIOList2BufState *);
+
+HIPE_WRAPPER_BIF_DISABLE_GC(list_to_bitstring, 1)
+
 BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
 {
-    Eterm bin;
-    Uint sz;
-    int offset;
-    byte* bytes;
-    ErlSubBin* sb1; 
-    Eterm* hp;
-    
-    if (is_nil(BIF_ARG_1)) {
-	BIF_RET(new_binary(BIF_P,(byte*)"",0));
-    }
-    if (is_not_list(BIF_ARG_1)) {
-    error:
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    switch (bitstr_list_len(BIF_ARG_1, &sz)) {
-    case ERTS_IOLIST_TYPE:
-	goto error;
-    case ERTS_IOLIST_OVERFLOW:
-	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
-    }
-    bin = new_binary(BIF_P, (byte *)NULL, sz);
-    bytes = binary_bytes(bin);
-#ifdef DEBUG
-    offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz);
-#else
-    offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes);
-#endif
-    ASSERT(offset >= 0);
-    if (offset > 0) {
-	hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE);
-	sb1 = (ErlSubBin *) hp;
-	sb1->thing_word = HEADER_SUB_BIN;
-	sb1->size = sz-1;
-	sb1->offs = 0;
-	sb1->orig = bin;
-	sb1->bitoffs = 0;
-	sb1->bitsize = offset;
-	sb1->is_writable = 0;
-	bin = make_binary(sb1);
+    BIF_RETTYPE ret;
+
+    if (is_nil(BIF_ARG_1))
+	ERTS_BIF_PREP_RET(ret, new_binary(BIF_P, (byte *) "", 0));
+    else if (is_not_list(BIF_ARG_1))
+	ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+    else {
+	/* check for [bitstring()] case */
+	Eterm h = CAR(list_val(BIF_ARG_1));
+	Eterm t = CDR(list_val(BIF_ARG_1));
+	if (is_binary(h) && is_nil(t)) {
+	    ERTS_BIF_PREP_RET(ret, h);
+	}
+	else {
+	    ErtsL2BState state = ERTS_L2B_STATE_INITER(BIF_P,
+						       BIF_ARG_1,
+						       bif_export[BIF_list_to_bitstring_1],
+						       bitstr_list_len,
+						       list_to_bitstr_buf_yielding);
+	    int orig_reds_left = ERTS_BIF_REDS_LEFT(BIF_P);
+
+	    /*
+	     * First try to do it all at once without having to use
+	     * yielding list_to_bitstr_buf().
+	     */
+	    state.buf.iolist.reds_left = orig_reds_left;
+	    switch (bitstr_list_len(&state.buf.iolist)) {
+	    case ERTS_IOLIST_OK: {
+		ErlDrvSizeT size = state.buf.iolist.size;
+		
+		state.bin = new_binary(BIF_P, (byte *) NULL, size);
+		state.buf.buf = (char *) binary_bytes(state.bin);
+		state.buf.len = size;
+		state.buf.iolist.obj = BIF_ARG_1;
+
+		if (size < ERTS_IOLIST_TO_BUF_BYTES_PER_RED*CONTEXT_REDS) {
+		    /* An (over) estimation of reductions needed */
+		    int reds_left = state.buf.iolist.reds_left;
+		    int to_buf_reds = orig_reds_left - reds_left;
+		    to_buf_reds += size/ERTS_IOLIST_TO_BUF_BYTES_PER_RED;
+		    if (to_buf_reds <= reds_left) {
+			ErlDrvSizeT res;
+
+			res = list_to_bitstr_buf_not_yielding(&state.buf);
+			if (res == 0) {
+			    Eterm res_bin = l2b_final_touch(BIF_P, &state);
+			    BUMP_REDS(BIF_P, to_buf_reds);
+			    ERTS_BIF_PREP_RET(ret, res_bin);
+			    break; /* done */
+			}
+			if (!ERTS_IOLIST_TO_BUF_FAILED(res))
+			    ERTS_INTERNAL_ERROR("iolist_size/iolist_to_buf missmatch");
+			if (res == ERTS_IOLIST_TO_BUF_OVERFLOW)
+			    goto overflow;
+			goto type_error;
+		    }
+		}
+		/*
+		 * Since size has been computed list_to_binary_chunk() expects
+		 * the state prepared for list_to_bitstr_buf.
+		 */
+
+		/* Fall through... */
+	    }
+	    case ERTS_IOLIST_YIELD:
+		ret = list_to_binary_chunk(THE_NON_VALUE,
+					   &state,
+					   state.buf.iolist.reds_left,
+					   0);
+		break;
+	    case ERTS_IOLIST_OVERFLOW:
+	    overflow:
+		ERTS_BIF_PREP_ERROR(ret, BIF_P, SYSTEM_LIMIT);
+		break;
+	    case ERTS_IOLIST_TYPE:
+	    type_error:
+	    default:
+		ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+		break;
+	    }
+	}
     }
-    
-    BIF_RET(bin);
+
+    return ret;
 }
 
 BIF_RETTYPE split_binary_2(BIF_ALIST_2)
@@ -744,123 +1058,353 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2)
  * Local functions.
  */
 
+static int
+list_to_bitstr_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp);
+
 /*
  * The input list is assumed to be type-correct and the buffer is
  * assumed to be of sufficient size. Those assumptions are verified in
  * the DEBUG-built emulator.
  */
-static int
+static ErlDrvSizeT
+list_to_bitstr_buf(int yield_support, ErtsIOList2BufState *state)
+{
+
+#undef LIST_TO_BITSTR_BUF_BCOPY_DBG
+#undef LIST_TO_BITSTR_BUF_BCOPY
 #ifdef DEBUG
-list_to_bitstr_buf(Eterm obj, char* buf, Uint len)
+#define LIST_TO_BITSTR_BUF_BCOPY_DBG					\
+    len -= size + (offset>7);
 #else
-list_to_bitstr_buf(Eterm obj, char* buf)
+#define LIST_TO_BITSTR_BUF_BCOPY_DBG
 #endif
-{
-    Eterm* objp;
-    int offset = 0;
+#define LIST_TO_BITSTR_BUF_BCOPY(CONSP)					\
+    do {								\
+	byte* bptr;							\
+	Uint bitsize;							\
+	Uint bitoffs;							\
+	Uint num_bits;							\
+	size_t size = binary_size(obj);					\
+	if (yield_support) {						\
+	    size_t max_size = ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;	\
+	    if (yield_count > 0)					\
+		max_size *= yield_count+1;				\
+	    if (size > max_size) {					\
+		state->objp = CONSP;					\
+		goto L_bcopy_yield;					\
+	    }								\
+	    if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) {	\
+		int cost = (int) size;					\
+		cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;	\
+		yield_count -= cost;					\
+	    }								\
+	}								\
+	ASSERT(size <= len);						\
+	ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);		\
+	num_bits = 8*size+bitsize;					\
+	copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits);	\
+	offset += bitsize;						\
+	buf += size + (offset>7);					\
+	LIST_TO_BITSTR_BUF_BCOPY_DBG;					\
+	offset = offset & 7;						\
+    } while(0)
+
+#ifdef DEBUG
+    ErlDrvSizeT len;
+#endif
+    Eterm obj;
+    char *buf;
+    Eterm *objp = NULL;
+    int offset;
+    int init_yield_count = 0, yield_count;
     DECLARE_ESTACK(s);
-    goto L_again;
-    
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-    L_again:
-	if (is_list(obj)) {
-	L_iter_list:
-	    objp = list_val(obj);
-	    obj = CAR(objp);
-	    if (is_byte(obj)) {
-		ASSERT(len > 0);
-		if (offset == 0) {
-		    *buf++ = unsigned_val(obj);
-		} else {
-		    *buf =  (char)((unsigned_val(obj) >> offset) | 
-				   ((*buf >> (8-offset)) << (8-offset)));
-		    buf++;
-		    *buf = (unsigned_val(obj) << (8-offset));
-		}   
+
+    obj = state->iolist.obj;
+    buf = state->buf;
+    offset = state->offset;
 #ifdef DEBUG
-		len--;
+    len = state->len;
 #endif
-	    } else if (is_binary(obj)) {
-		byte* bptr;
-		size_t size = binary_size(obj);
-		Uint bitsize;
-		Uint bitoffs;
-		Uint num_bits;
-		
-		ASSERT(size <= len);
-		ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
-		num_bits = 8*size+bitsize;
-		copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits);
-		offset += bitsize;
-		buf += size + (offset>7);
+
+    if (!yield_support) {
+	yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */
+	goto L_again;
+    }
+    else {
+
+	if (state->iolist.reds_left <= 0)
+	    return ERTS_IOLIST_TO_BUF_YIELD;
+
+	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+	init_yield_count = (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED
+			    * state->iolist.reds_left);
+	yield_count = init_yield_count;
+
+	if (!state->iolist.estack.start)
+	    goto L_again;
+	else {
+	    int chk_stack;
+	    /* Restart; restore state... */
+	    ESTACK_RESTORE(s, &state->iolist.estack);
+
+	    if (!state->bcopy.bptr)
+		chk_stack = 0;
+	    else {
+		chk_stack = 1;
+		if (list_to_bitstr_buf_bcopy(state, THE_NON_VALUE, &yield_count)) {
+		    /* Yield again... */
+		    BUMP_ALL_REDS(state->iolist.c_p);
+		    state->iolist.reds_left = 0;
+		    ESTACK_SAVE(s, &state->iolist.estack);
+		    return ERTS_IOLIST_TO_BUF_YIELD;
+		}
+		buf = state->buf;
+		offset = state->offset;
 #ifdef DEBUG
-		len -= size + (offset>7);
+		len = state->len;
 #endif
-		offset = offset & 7;
-	    } else if (is_list(obj)) {
-		ESTACK_PUSH(s, CDR(objp));
-		goto L_iter_list; /* on head */
-	    } else {
-		ASSERT(is_nil(obj));
 	    }
 
-	    obj = CDR(objp);
-	    if (is_list(obj)) {
-		goto L_iter_list; /* on tail */
-	    } else if (is_binary(obj)) {
-		byte* bptr;
-		size_t size = binary_size(obj);
-		Uint bitsize;
-		Uint bitoffs;
-		Uint num_bits;
-
-		ASSERT(size <= len);
-		ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
-		num_bits = 8*size+bitsize;
-		copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits);
-		offset += bitsize;
-		buf += size+(offset>7);
+	    objp = state->objp;
+	    state->objp = NULL;
+
+	    if (objp)
+		goto L_tail;
+	    if (!chk_stack)
+		goto L_again;
+	    /* check stack */
+	}
+    }
+    
+    while (!ESTACK_ISEMPTY(s)) {
+	obj = ESTACK_POP(s);
+    L_again:
+	if (is_list(obj)) {
+	    while (1) { /* Tail loop */
+		while (1) { /* Head loop */
+		    if (yield_support && --yield_count <= 0)
+			goto L_yield;
+		    objp = list_val(obj);
+		    obj = CAR(objp);
+		    if (is_byte(obj)) {
+			ASSERT(len > 0);
+			if (offset == 0) {
+			    *buf++ = unsigned_val(obj);
+			} else {
+			    *buf =  (char)((unsigned_val(obj) >> offset) | 
+					   ((*buf >> (8-offset)) << (8-offset)));
+			    buf++;
+			    *buf = (unsigned_val(obj) << (8-offset));
+			}   
 #ifdef DEBUG
-		len -= size+(offset>7);
+			len--;
 #endif
-		offset = offset & 7;
-	    } else {
-		ASSERT(is_nil(obj));
+		    } else if (is_binary(obj)) {
+			LIST_TO_BITSTR_BUF_BCOPY(objp);
+		    } else if (is_list(obj)) {
+			ESTACK_PUSH(s, CDR(objp));
+			continue; /* Head loop */
+		    } else {
+			ASSERT(is_nil(obj));
+		    }
+		    break;
+		}
+
+	    L_tail:
+
+		obj = CDR(objp);
+		if (is_list(obj)) {
+		    continue; /* Tail loop */
+		} else if (is_binary(obj)) {
+		    LIST_TO_BITSTR_BUF_BCOPY(NULL);
+		} else {
+		    ASSERT(is_nil(obj));
+		}
+		break;
 	    }
 	} else if (is_binary(obj)) {
-	    byte* bptr;
-	    size_t size = binary_size(obj);
-	    Uint bitsize;
-	    Uint bitoffs;
-	    Uint num_bits;
-
-	    ASSERT(size <= len);
-	    ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
-	    num_bits = 8*size+bitsize;
-	    copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits);
-	    offset += bitsize;
-	    buf += size + (offset>7);
-#ifdef DEBUG
-	    len -= size + (offset>7);
-#endif
-	    offset = offset & 7;
+	    LIST_TO_BITSTR_BUF_BCOPY(NULL);
 	} else {
+	    if (yield_support && --yield_count <= 0)
+		goto L_yield;
 	    ASSERT(is_nil(obj));
 	}
     }
     
     DESTROY_ESTACK(s);
-    return offset;
+
+    if (yield_support) {
+	int reds;
+	CLEAR_SAVED_ESTACK(&state->iolist.estack);
+	reds = ((init_yield_count - yield_count - 1)
+		/ ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED) + 1;
+	BUMP_REDS(state->iolist.c_p, reds);
+	state->iolist.reds_left -= reds;
+	if (state->iolist.reds_left < 0)
+	    state->iolist.reds_left = 0;
+    }
+    state->buf = buf;
+    state->offset = offset;
+    return 0;
+
+L_bcopy_yield:
+
+    state->buf = buf;
+    state->offset = offset;
+#ifdef DEBUG
+    state->len = len;
+#endif
+
+    if (list_to_bitstr_buf_bcopy(state, obj, &yield_count) == 0)
+	ERTS_INTERNAL_ERROR("Missing yield");
+
+    BUMP_ALL_REDS(state->iolist.c_p);
+    state->iolist.reds_left = 0;
+    ESTACK_SAVE(s, &state->iolist.estack);
+    return ERTS_IOLIST_TO_BUF_YIELD;
+
+L_yield:
+
+    BUMP_ALL_REDS(state->iolist.c_p);
+    state->iolist.reds_left = 0;
+    state->iolist.obj = obj;
+    state->buf = buf;
+    state->offset = offset;
+    ESTACK_SAVE(s, &state->iolist.estack);
+#ifdef DEBUG
+    state->len = len;
+#endif
+    return ERTS_IOLIST_TO_BUF_YIELD;
+
+
+#undef LIST_TO_BITSTR_BUF_BCOPY_DBG
+#undef LIST_TO_BITSTR_BUF_BCOPY
+
+}
+
+static ErlDrvSizeT
+list_to_bitstr_buf_yielding(ErtsIOList2BufState *state)
+{
+    return list_to_bitstr_buf(1, state);
+}
+
+static ErlDrvSizeT
+list_to_bitstr_buf_not_yielding(ErtsIOList2BufState *state)
+{
+    return list_to_bitstr_buf(0, state);
 }
 
 static int
-bitstr_list_len(Eterm obj, Uint* num_bytes)
+list_to_bitstr_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp)
+{
+    int res;
+    char *buf = state->buf;
+    char *next_buf;
+    int offset = state->offset;
+    int next_offset;
+#ifdef DEBUG
+    ErlDrvSizeT len = state->len;
+    ErlDrvSizeT next_len;
+#endif
+    byte* bptr;
+    size_t size;
+    size_t max_size;
+    Uint bitoffs;
+    Uint num_bits;
+    Uint bitsize;
+    int yield_count = *yield_countp;
+
+    if (state->bcopy.bptr) {
+	bptr = state->bcopy.bptr;
+	size = state->bcopy.size;
+	bitoffs = state->bcopy.bitoffs;
+	bitsize = state->bcopy.bitsize;
+	state->bcopy.bptr = NULL;
+    }
+    else {
+
+	ASSERT(is_binary(obj));
+
+	size = binary_size(obj);
+
+	ASSERT(size <= len);
+
+	ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
+    }
+
+    max_size = (size_t) ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;
+    if (yield_count > 0)
+	max_size *= (size_t) (yield_count+1);
+
+    if (size <= max_size) {
+	if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) {
+	    int cost = (int) size;
+	    cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;
+	    yield_count -= cost;
+	}
+	next_offset = offset + bitsize;
+	next_buf = buf + size+(next_offset>7);
+#ifdef DEBUG
+	next_len = len - size+(next_offset>7);
+#endif
+	next_offset &= 7;
+	num_bits = 8*size+bitsize;
+	res = 0;
+    }
+    else {
+	ASSERT(0 < max_size && max_size < size);
+	yield_count = 0;
+	state->bcopy.bptr = bptr + max_size;
+	state->bcopy.bitoffs = bitoffs;
+	state->bcopy.bitsize = bitsize;
+	state->bcopy.size = size - max_size;
+	next_buf = buf + max_size;
+#ifdef DEBUG
+	next_len = len - max_size;
+#endif
+	next_offset = offset;
+	num_bits = 8*max_size;
+	size = max_size;
+	res = 1;
+    }
+
+    copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits);
+
+    state->offset = next_offset;
+    state->buf = next_buf;
+#ifdef DEBUG
+    state->len = next_len;
+#endif
+    *yield_countp = yield_count;
+
+    return res;
+}
+
+static int
+bitstr_list_len(ErtsIOListState *state)
 {
     Eterm* objp;
-    Uint len = 0;
-    Uint offs = 0;
+    Eterm obj;
+    Uint len, offs;
+    int res, init_yield_count, yield_count;
     DECLARE_ESTACK(s);
+
+    if (state->reds_left <= 0)
+	return ERTS_IOLIST_YIELD;
+
+    len = (Uint) state->size;
+    offs = state->offs;
+    obj = state->obj;
+
+    ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+    init_yield_count = ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED;
+    init_yield_count *= state->reds_left;
+    yield_count = init_yield_count;
+    if (state->estack.start) {
+	/* Restart; restore estack... */
+	ESTACK_RESTORE(s, &state->estack);
+    }
+
     goto L_again;
 
 #define SAFE_ADD(Var, Val)			\
@@ -887,46 +1431,55 @@ bitstr_list_len(Eterm obj, Uint* num_bytes)
 	obj = ESTACK_POP(s);
     L_again:
 	if (is_list(obj)) {
-	L_iter_list:
-	    objp = list_val(obj);
-	    /* Head */
-	    obj = CAR(objp);
-	    if (is_byte(obj)) {
-		len++;
-		if (len == 0) {
-		    goto L_overflow_error;
+	    while (1) { /* Tail loop */
+		while (1) { /* Head loop */
+		    if (--yield_count <= 0)
+			goto L_yield;
+		    objp = list_val(obj);
+		    /* Head */
+		    obj = CAR(objp);
+		    if (is_byte(obj)) {
+			len++;
+			if (len == 0) {
+			    goto L_overflow_error;
+			}
+		    } else if (is_binary(obj)) {
+			SAFE_ADD(len, binary_size(obj));
+			SAFE_ADD_BITSIZE(offs, obj);
+		    } else if (is_list(obj)) {
+			ESTACK_PUSH(s, CDR(objp));
+			continue; /* Head loop */
+		    } else if (is_not_nil(obj)) {
+			goto L_type_error;
+		    }
+		    break;
 		}
-	    } else if (is_binary(obj)) {
-		SAFE_ADD(len, binary_size(obj));
-		SAFE_ADD_BITSIZE(offs, obj);
-	    } else if (is_list(obj)) {
-		ESTACK_PUSH(s, CDR(objp));
-		goto L_iter_list; /* on head */
-	    } else if (is_not_nil(obj)) {
-		goto L_type_error;
+		/* Tail */
+		obj = CDR(objp);
+		if (is_list(obj))
+		    continue; /* Tail loop */
+		else if (is_binary(obj)) {
+		    SAFE_ADD(len, binary_size(obj));
+		    SAFE_ADD_BITSIZE(offs, obj);
+		} else if (is_not_nil(obj)) {
+		    goto L_type_error;
+		}
+		break;
 	    }
-	    /* Tail */
-	    obj = CDR(objp);
-	    if (is_list(obj))
-		goto L_iter_list; /* on tail */
-	    else if (is_binary(obj)) {
+	} else {
+	    if (--yield_count <= 0)
+		goto L_yield;
+	    if (is_binary(obj)) {
 		SAFE_ADD(len, binary_size(obj));
 		SAFE_ADD_BITSIZE(offs, obj);
 	    } else if (is_not_nil(obj)) {
 		goto L_type_error;
 	    }
-	} else if (is_binary(obj)) {
-	    SAFE_ADD(len, binary_size(obj));
-	    SAFE_ADD_BITSIZE(offs, obj);
-	} else if (is_not_nil(obj)) {
-	    goto L_type_error;
 	}
     }
 #undef SAFE_ADD
 #undef SAFE_ADD_BITSIZE
 
-    DESTROY_ESTACK(s);
-
     /*
      * Make sure that the number of bits in the bitstring will fit
      * in an Uint to ensure that the binary can be matched using
@@ -939,15 +1492,42 @@ bitstr_list_len(Eterm obj, Uint* num_bytes)
     if (len << 3 < len) {
 	goto L_overflow_error;
     }
-    *num_bytes = len;
-    return ERTS_IOLIST_OK;
+    state->size = len;
 
- L_type_error:
-    DESTROY_ESTACK(s);
-    return ERTS_IOLIST_TYPE;
+    res = ERTS_IOLIST_OK;
+
+ L_return: {
+	int yc = init_yield_count - yield_count;
+	int reds;
+
+	DESTROY_ESTACK(s);
+	CLEAR_SAVED_ESTACK(&state->estack);
+
+	reds = (yc - 1)/ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED + 1;
+	BUMP_REDS(state->c_p, reds);
+	state->reds_left -= reds;
+	state->size = (ErlDrvSizeT) len;
+	state->have_size = 1;
+	return res;
+    }
 
  L_overflow_error:
-    DESTROY_ESTACK(s);
-    return ERTS_IOLIST_OVERFLOW;
+    res = ERTS_IOLIST_OVERFLOW;
+    len = 0;
+    goto L_return;
+
+ L_type_error:
+    res = ERTS_IOLIST_TYPE;
+    len = 0;
+    goto L_return;
+
+ L_yield:
+    BUMP_ALL_REDS(state->c_p);
+    state->reds_left = 0;
+    state->size = len;
+    state->offs = offs;
+    state->obj = obj;
+    ESTACK_SAVE(s, &state->estack);
+    return ERTS_IOLIST_YIELD;
 }
 
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index ff775691b3..7e0e825a0d 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2294,18 +2294,11 @@ BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P,BADARG);
 }
 
-/*
- * Ok, erlang:list_to_binary does not interrupt, and we really don't want
- * an alternative implementation for the exact same thing, why we
- * have descided to use the old non-restarting implementation for now.
- * In reality, there are seldom many iterations involved in doing this, so the
- * problem of long-running bifs is not really that big in this case.
- * So, for now we use the old implementation also in the module binary.
- */
+HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1)
 
 BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1)
 {
-    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_binary_list_to_bin_1]);
 }
 
 typedef struct {
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 819b19e566..6c9f53ce87 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -166,7 +166,7 @@ Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size,
  * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1
  */
 
-BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg);
+BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg, Export *bif);
 BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple);
 BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen);
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 8fcb95d0e2..891046a8b5 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -435,6 +435,8 @@ do {\
     }\
 } while(0)
 
+#define CLEAR_SAVED_ESTACK(estack) ((void) ((estack)->start = NULL))
+
 /*
  * Use on empty stack, only the allocator can be changed before this.
  * The src stack is reset to NULL.
@@ -551,6 +553,8 @@ do {\
     }\
 } while(0)
 
+#define CLEAR_SAVED_WSTACK(wstack) ((void) ((wstack)->wstart = NULL))
+
 /*
  * Use on empty stack, only the allocator can be changed before this.
  * The src stack is reset to NULL.
@@ -951,20 +955,67 @@ struct Sint_buf {
 };	
 char* Sint_to_buf(Sint, struct Sint_buf*);
 
+#define ERTS_IOLIST_STATE_INITER(C_P, OBJ)	\
+    {(C_P), 0, 0, (OBJ), {NULL, NULL, NULL, ERTS_ALC_T_INVALID}, 0, 0}
+
+#define ERTS_IOLIST_STATE_MOVE(TO, FROM)	\
+    sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOListState))
+
+#define ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED 8
+
+typedef struct {
+    Process *c_p;
+    ErlDrvSizeT size;
+    Uint offs;
+    Eterm obj;
+    ErtsEStack estack;
+    int reds_left;
+    int have_size;
+} ErtsIOListState;
+
+#define ERTS_IOLIST2BUF_STATE_INITER(C_P, OBJ)	\
+    {ERTS_IOLIST_STATE_INITER((C_P), (OBJ)), {NULL, 0, 0, 0}, NULL, 0, NULL, 0}
+
+#define ERTS_IOLIST2BUF_STATE_MOVE(TO, FROM)	\
+    sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOList2BufState))
+
+#define ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT 32
+#define ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED 8
+#define ERTS_IOLIST_TO_BUF_BYTES_PER_RED \
+    (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED*ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT)
+
+typedef struct {
+    ErtsIOListState iolist;
+    struct {
+	byte *bptr;
+	size_t size;
+	Uint bitoffs;
+	Uint bitsize;
+    } bcopy;
+    char *buf;
+    ErlDrvSizeT len;
+    Eterm *objp;
+    int offset;
+} ErtsIOList2BufState;
+
 #define ERTS_IOLIST_OK 0
 #define ERTS_IOLIST_OVERFLOW 1
 #define ERTS_IOLIST_TYPE 2
+#define ERTS_IOLIST_YIELD 3
 
 Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */
 
 #define ERTS_IOLIST_TO_BUF_OVERFLOW	(~((ErlDrvSizeT) 0))
 #define ERTS_IOLIST_TO_BUF_TYPE_ERROR	(~((ErlDrvSizeT) 1))
+#define ERTS_IOLIST_TO_BUF_YIELD	(~((ErlDrvSizeT) 2))
 #define ERTS_IOLIST_TO_BUF_FAILED(R) \
-    (((R) & (~((ErlDrvSizeT) 1))) == (~((ErlDrvSizeT) 1)))
+    (((R) & (~((ErlDrvSizeT) 3))) == (~((ErlDrvSizeT) 3)))
 #define ERTS_IOLIST_TO_BUF_SUCCEEDED(R) \
     (!ERTS_IOLIST_TO_BUF_FAILED((R)))
 
 ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT);
+ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *);
+int erts_iolist_size_yielding(ErtsIOListState *state);
 int erts_iolist_size(Eterm, ErlDrvSizeT *);
 int is_string(Eterm);
 void erl_at_exit(void (*) (void*), void*);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 738f793020..72092ec7b0 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3197,106 +3197,303 @@ buf_to_intlist(Eterm** hpp, const char *buf, size_t len, Eterm tail)
 ** 
 */
 
-ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
+typedef enum {
+    ERTS_IL2B_BCOPY_OK,
+    ERTS_IL2B_BCOPY_YIELD,
+    ERTS_IL2B_BCOPY_OVERFLOW,
+    ERTS_IL2B_BCOPY_TYPE_ERROR
+} ErtsIL2BBCopyRes;
+
+static ErtsIL2BBCopyRes
+iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp);
+
+static ERTS_INLINE ErlDrvSizeT
+iolist_to_buf(const int yield_support,
+	      ErtsIOList2BufState *state,
+	      Eterm obj,
+	      char* buf,
+	      ErlDrvSizeT alloced_len)
 {
-    ErlDrvSizeT len = (ErlDrvSizeT) alloced_len;
-    Eterm* objp;
+#undef IOLIST_TO_BUF_BCOPY
+#define IOLIST_TO_BUF_BCOPY(CONSP)					\
+do {									\
+    size_t size = binary_size(obj);					\
+    if (size > 0) {							\
+	Uint bitsize;							\
+	byte* bptr;							\
+	Uint bitoffs;							\
+	Uint num_bits;							\
+	if (yield_support) {						\
+	    size_t max_size = ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;	\
+	    if (yield_count > 0)					\
+		max_size *= yield_count+1;				\
+	    if (size > max_size) {					\
+		state->objp = CONSP;					\
+		goto L_bcopy_yield;					\
+	    }								\
+	    if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) {	\
+		int cost = (int) size;					\
+		cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;	\
+		yield_count -= cost;					\
+	    }								\
+	}								\
+	if (len < size)							\
+	    goto L_overflow;						\
+	ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);		\
+	if (bitsize != 0)						\
+	    goto L_type_error;						\
+	num_bits = 8*size;						\
+	copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);		\
+	buf += size;							\
+	len -= size;							\
+    }									\
+} while (0)
+
+    ErlDrvSizeT res, len;
+    Eterm* objp = NULL;
+    int init_yield_count;
+    int yield_count;
     DECLARE_ESTACK(s);
-    goto L_again;
-    
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-    L_again:
-	if (is_list(obj)) {
-	L_iter_list:
-	    objp = list_val(obj);
-	    obj = CAR(objp);
-	    if (is_byte(obj)) {
-		if (len == 0) {
-		    goto L_overflow;
-		}
-		*buf++ = unsigned_val(obj);
-		len--;
-	    } else if (is_binary(obj)) {
-		byte* bptr;
-		size_t size = binary_size(obj);
-		Uint bitsize;
-		Uint bitoffs;
-		Uint num_bits;
-		
-		if (len < size) {
+
+    len = (ErlDrvSizeT) alloced_len;
+
+    if (!yield_support) {
+	yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */
+	goto L_again;
+    }
+    else {
+
+	if (state->iolist.reds_left <= 0)
+	    return ERTS_IOLIST_TO_BUF_YIELD;
+
+	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+	init_yield_count = (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED
+			   * state->iolist.reds_left);
+	yield_count = init_yield_count;
+
+	if (!state->iolist.estack.start)
+	    goto L_again;
+	else {
+	    int chk_stack;
+	    /* Restart; restore state... */
+	    ESTACK_RESTORE(s, &state->iolist.estack);
+
+	    if (!state->bcopy.bptr)
+		chk_stack = 0;
+	    else {
+		chk_stack = 1;
+		switch (iolist_to_buf_bcopy(state, THE_NON_VALUE, &yield_count)) {
+		case ERTS_IL2B_BCOPY_OK:
+		    break;
+		case ERTS_IL2B_BCOPY_YIELD:
+		    BUMP_ALL_REDS(state->iolist.c_p);
+		    state->iolist.reds_left = 0;
+		    ESTACK_SAVE(s, &state->iolist.estack);
+		    return ERTS_IOLIST_TO_BUF_YIELD;
+		case ERTS_IL2B_BCOPY_OVERFLOW:
 		    goto L_overflow;
-		}
-		ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
-		if (bitsize != 0) {
+		case ERTS_IL2B_BCOPY_TYPE_ERROR:
 		    goto L_type_error;
 		}
-		num_bits = 8*size;
-		copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
-		buf += size;
-		len -= size;
-	    } else if (is_list(obj)) {
-		ESTACK_PUSH(s, CDR(objp));
-		goto L_iter_list; /* on head */
-	    } else if (is_not_nil(obj)) {
-		goto L_type_error;
 	    }
 
-	    obj = CDR(objp);
-	    if (is_list(obj)) {
-		goto L_iter_list; /* on tail */
-	    } else if (is_binary(obj)) {
-		byte* bptr;
-		size_t size = binary_size(obj);
-		Uint bitsize;
-		Uint bitoffs;
-		Uint num_bits;
-		if (len < size) {
-		    goto L_overflow;
+	    obj = state->iolist.obj;
+	    buf = state->buf;
+	    len = state->len;
+	    objp = state->objp;
+	    state->objp = NULL;
+	    if (objp)
+		goto L_tail;
+	    if (!chk_stack)
+		goto L_again;
+	    /* check stack */
+	}
+    }
+
+    while (!ESTACK_ISEMPTY(s)) {
+	obj = ESTACK_POP(s);
+    L_again:
+	if (is_list(obj)) {
+	    while (1) { /* Tail loop */
+		while (1) { /* Head loop */
+		    if (yield_support && --yield_count <= 0)
+			goto L_yield;
+		    objp = list_val(obj);
+		    obj = CAR(objp);
+		    if (is_byte(obj)) {
+			if (len == 0) {
+			    goto L_overflow;
+			}
+			*buf++ = unsigned_val(obj);
+			len--;
+		    } else if (is_binary(obj)) {
+			IOLIST_TO_BUF_BCOPY(objp);
+		    } else if (is_list(obj)) {
+			ESTACK_PUSH(s, CDR(objp));
+			continue; /* Head loop */
+		    } else if (is_not_nil(obj)) {
+			goto L_type_error;
+		    }
+		    break;
 		}
-		ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
-		if (bitsize != 0) {
+
+	    L_tail:
+
+		obj = CDR(objp);
+
+		if (is_list(obj)) {
+		    continue; /* Tail loop */
+		} else if (is_binary(obj)) {
+		    IOLIST_TO_BUF_BCOPY(NULL);
+		} else if (is_not_nil(obj)) {
 		    goto L_type_error;
 		}
-		num_bits = 8*size;
-		copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
-		buf += size;
-		len -= size;
-	    } else if (is_not_nil(obj)) {
-		goto L_type_error;
+		break;
 	    }
 	} else if (is_binary(obj)) {
-	    byte* bptr;
-	    size_t size = binary_size(obj);
-	    Uint bitsize;
-	    Uint bitoffs;
-	    Uint num_bits;
-	    if (len < size) {
-		goto L_overflow;
-	    }
-	    ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
-	    if (bitsize != 0) {
-		goto L_type_error;
-	    }
-	    num_bits = 8*size;
-	    copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
-	    buf += size;
-	    len -= size;
+	    IOLIST_TO_BUF_BCOPY(NULL);
 	} else if (is_not_nil(obj)) {
 	    goto L_type_error;
-	}
+	} else if (yield_support && --yield_count <= 0)
+	    goto L_yield;
     }
       
+    res = len;
+
+ L_return: 
+
     DESTROY_ESTACK(s);
-    return len;
+
+    if (yield_support) {
+	int reds;
+	CLEAR_SAVED_ESTACK(&state->iolist.estack);
+	reds = ((init_yield_count - yield_count - 1)
+		/ ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED) + 1;
+	BUMP_REDS(state->iolist.c_p, reds);
+	state->iolist.reds_left -= reds;
+	if (state->iolist.reds_left < 0)
+	    state->iolist.reds_left = 0;
+    }
+
+
+    return res;
 
  L_type_error:
-    DESTROY_ESTACK(s);
-    return ERTS_IOLIST_TO_BUF_TYPE_ERROR;
+    res = ERTS_IOLIST_TO_BUF_TYPE_ERROR;
+    goto L_return;
 
  L_overflow:
-    DESTROY_ESTACK(s);
-    return ERTS_IOLIST_TO_BUF_OVERFLOW;
+    res = ERTS_IOLIST_TO_BUF_OVERFLOW;
+    goto L_return;
+
+ L_bcopy_yield:
+
+    state->buf = buf;
+    state->len = len;
+
+    switch (iolist_to_buf_bcopy(state, obj, &yield_count)) {
+    case ERTS_IL2B_BCOPY_OK:
+	ERTS_INTERNAL_ERROR("Missing yield");
+    case ERTS_IL2B_BCOPY_YIELD:
+	BUMP_ALL_REDS(state->iolist.c_p);
+	state->iolist.reds_left = 0;
+	ESTACK_SAVE(s, &state->iolist.estack);
+	return ERTS_IOLIST_TO_BUF_YIELD;
+    case ERTS_IL2B_BCOPY_OVERFLOW:
+	goto L_overflow;
+    case ERTS_IL2B_BCOPY_TYPE_ERROR:
+	goto L_type_error;
+    }
+
+ L_yield:
+
+    BUMP_ALL_REDS(state->iolist.c_p);
+    state->iolist.reds_left = 0;
+    state->iolist.obj = obj;
+    state->buf = buf;
+    state->len = len;
+    ESTACK_SAVE(s, &state->iolist.estack);
+    return ERTS_IOLIST_TO_BUF_YIELD;
+
+#undef IOLIST_TO_BUF_BCOPY
+}
+
+static ErtsIL2BBCopyRes
+iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp)
+{
+    ErtsIL2BBCopyRes res;
+    char *buf = state->buf;
+    ErlDrvSizeT len = state->len;
+    byte* bptr;
+    size_t size;
+    size_t max_size;
+    Uint bitoffs;
+    Uint num_bits;
+    int yield_count = *yield_countp;
+
+    if (state->bcopy.bptr) {
+	bptr = state->bcopy.bptr;
+	size = state->bcopy.size;
+	bitoffs = state->bcopy.bitoffs;
+	state->bcopy.bptr = NULL;
+    }
+    else {
+	Uint bitsize;
+
+	ASSERT(is_binary(obj));
+
+	size = binary_size(obj);
+	if (size <= 0)
+	    return ERTS_IL2B_BCOPY_OK;
+
+	if (len < size)
+	    return ERTS_IL2B_BCOPY_OVERFLOW;
+
+	ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
+	if (bitsize != 0)
+	    return ERTS_IL2B_BCOPY_TYPE_ERROR;
+    }
+
+    ASSERT(size > 0);
+    max_size = (size_t) ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;
+    if (yield_count > 0)
+	max_size *= (size_t) (yield_count+1);
+
+    if (size <= max_size) {
+	if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) {
+	    int cost = (int) size;
+	    cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;
+	    yield_count -= cost;
+	}
+	res = ERTS_IL2B_BCOPY_OK;
+    }
+    else {
+	ASSERT(0 < max_size && max_size < size);
+	yield_count = 0;
+	state->bcopy.bptr = bptr + max_size;
+	state->bcopy.bitoffs = bitoffs;
+	state->bcopy.size = size - max_size;
+	size = max_size;
+	res = ERTS_IL2B_BCOPY_YIELD;
+    }
+
+    num_bits = 8*size;
+    copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
+    state->buf += size;
+    state->len -= size;
+    *yield_countp = yield_count;
+
+    return res;
+}
+
+ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *state)
+{
+    return iolist_to_buf(1, state, state->iolist.obj, state->buf, state->len);
+}
+
+ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
+{
+    return iolist_to_buf(0, NULL, obj, buf, alloced_len);
 }
 
 /*
@@ -3307,11 +3504,32 @@ ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
  * Any input term error detected in erts_iolist_to_buf should also
  * be detected in this function!
  */
-int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
+
+static ERTS_INLINE int
+iolist_size(const int yield_support, ErtsIOListState *state, Eterm obj, ErlDrvSizeT* sizep)
 {
+    int res, init_yield_count, yield_count;
     Eterm* objp;
-    Uint size = 0; /* Intentionally Uint due to halfword heap */
+    Uint size = (Uint) *sizep; /* Intentionally Uint due to halfword heap */
     DECLARE_ESTACK(s);
+
+    if (!yield_support)
+	yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */
+    else {
+	if (state->reds_left <= 0)
+	    return ERTS_IOLIST_YIELD;
+	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+	init_yield_count = ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED;
+	init_yield_count *= state->reds_left;
+	yield_count = init_yield_count;
+	if (state->estack.start) {
+	    /* Restart; restore state... */
+	    ESTACK_RESTORE(s, &state->estack);
+	    size = (Uint) state->size;
+	    obj = state->obj;
+	}
+    }
+
     goto L_again;
 
 #define SAFE_ADD(Var, Val)			\
@@ -3327,51 +3545,101 @@ int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
 	obj = ESTACK_POP(s);
     L_again:
 	if (is_list(obj)) {
-	L_iter_list:
-	    objp = list_val(obj);
-	    /* Head */
-	    obj = CAR(objp);
-	    if (is_byte(obj)) {
-		size++;
-		if (size == 0) {
-		    goto L_overflow_error;
+	    while (1) { /* Tail loop */
+		while (1) { /* Head loop */
+		    if (yield_support && --yield_count <= 0)
+			goto L_yield;
+		    objp = list_val(obj);
+		    /* Head */
+		    obj = CAR(objp);
+		    if (is_byte(obj)) {
+			size++;
+			if (size == 0) {
+			    goto L_overflow_error;
+			}
+		    } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+			SAFE_ADD(size, binary_size(obj));
+		    } else if (is_list(obj)) {
+			ESTACK_PUSH(s, CDR(objp));
+			continue; /* Head loop */
+		    } else if (is_not_nil(obj)) {
+			goto L_type_error;
+		    }
+		    break;
 		}
-	    } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
-		SAFE_ADD(size, binary_size(obj));
-	    } else if (is_list(obj)) {
-		ESTACK_PUSH(s, CDR(objp));
-		goto L_iter_list; /* on head */
-	    } else if (is_not_nil(obj)) {
-		goto L_type_error;
+		/* Tail */
+		obj = CDR(objp);
+		if (is_list(obj))
+		    continue; /* Tail loop */
+		else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+		    SAFE_ADD(size, binary_size(obj));
+		} else if (is_not_nil(obj)) {
+		    goto L_type_error;
+		}
+		break;
 	    }
-	    /* Tail */
-	    obj = CDR(objp);
-	    if (is_list(obj))
-		goto L_iter_list; /* on tail */
-	    else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+	} else {
+	    if (yield_support && --yield_count <= 0)
+		goto L_yield;
+	    if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
 		SAFE_ADD(size, binary_size(obj));
 	    } else if (is_not_nil(obj)) {
 		goto L_type_error;
 	    }
-	} else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
-	    SAFE_ADD(size, binary_size(obj));
-	} else if (is_not_nil(obj)) {
-	    goto L_type_error;
 	}
     }
 #undef SAFE_ADD
 
-    DESTROY_ESTACK(s);
     *sizep = (ErlDrvSizeT) size;
-    return ERTS_IOLIST_OK;
 
- L_overflow_error:
+    res = ERTS_IOLIST_OK;
+
+ L_return:
+
     DESTROY_ESTACK(s);
-    return ERTS_IOLIST_OVERFLOW;
+
+    if (yield_support) {
+	int yc, reds;
+	CLEAR_SAVED_ESTACK(&state->estack);
+	yc = init_yield_count - yield_count;
+	reds = ((yc - 1) / ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED) + 1;
+	BUMP_REDS(state->c_p, reds);
+	state->reds_left -= reds;
+	state->size = (ErlDrvSizeT) size;
+	state->have_size = 1;
+    }
+
+    return res;
+
+ L_overflow_error:
+    res = ERTS_IOLIST_OVERFLOW;
+    size = 0;
+    goto L_return;
 
  L_type_error:
-    DESTROY_ESTACK(s);
-    return ERTS_IOLIST_TYPE;
+    res = ERTS_IOLIST_TYPE;
+    size = 0;
+    goto L_return;
+
+ L_yield:
+    BUMP_ALL_REDS(state->c_p);
+    state->reds_left = 0;
+    state->size = size;
+    state->obj = obj;
+    ESTACK_SAVE(s, &state->estack);
+    return ERTS_IOLIST_YIELD;
+}
+
+int erts_iolist_size_yielding(ErtsIOListState *state)
+{
+    ErlDrvSizeT size = state->size;
+    return iolist_size(1, state, state->obj, &size);
+}
+
+int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
+{
+    *sizep = 0;
+    return iolist_size(0, NULL, obj, sizep);
 }
 
 /* return 0 if item is not a non-empty flat list of bytes */
-- 
cgit v1.2.3


From a933cda32fd2dbae4e1367f98206bebfea39b86b Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Sun, 25 May 2014 14:26:08 +0200
Subject: Fix permissions of some files in the repository

These files aren't supposed to be executable. For reference, the command used to
find them was:

	git ls-files -z | xargs -0 -J % find % -type f -perm ++x
---
 erts/emulator/beam/erl_bif_info.c | 0
 erts/emulator/beam/global.h       | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 erts/emulator/beam/erl_bif_info.c
 mode change 100755 => 100644 erts/emulator/beam/global.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
old mode 100755
new mode 100644
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
old mode 100755
new mode 100644
-- 
cgit v1.2.3


From 7adc2c36b0e839384baac50be35c0999d1e546df Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Sun, 25 May 2014 16:06:17 +0200
Subject: Implement ets:take/2

This new ETS BIF returns and deletes objects from tables.
---
 erts/emulator/beam/bif.tab       |  2 +
 erts/emulator/beam/erl_db.c      | 25 ++++++++++++
 erts/emulator/beam/erl_db_hash.c | 83 ++++++++++++++++++++++++++++++++--------
 erts/emulator/beam/erl_db_tree.c | 24 ++++++++++++
 erts/emulator/beam/erl_db_util.h |  1 +
 5 files changed, 120 insertions(+), 15 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index fbdddf09db..fe25b2d27c 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -596,6 +596,8 @@ bif maps:values/1
 
 bif erts_internal:cmp_term/2
 
+bif ets:take/2
+
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 8f246ffa07..fba1e32230 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -753,6 +753,31 @@ BIF_RETTYPE ets_prev_2(BIF_ALIST_2)
     BIF_RET(ret);
 }
 
+/*
+** take(Tab, Key)
+*/
+BIF_RETTYPE ets_take_2(BIF_ALIST_2)
+{
+    DbTable* tb;
+#ifdef DEBUG
+    int cret;
+#endif
+    Eterm ret;
+    CHECK_TABLES();
+
+    tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC);
+    if (!tb) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+#ifdef DEBUG
+    cret =
+#endif
+        tb->common.meth->db_take(BIF_P, tb, BIF_ARG_2, &ret);
+    ASSERT(cret == DB_ERROR_NONE);
+    db_unlock(tb, LCK_WRITE_REC);
+    BIF_RET(ret);
+}
+
 /* 
 ** update_element(Tab, Key, {Pos, Value})
 ** update_element(Tab, Key, [{Pos, Value}])
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 06dac8f161..633a2272ad 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -426,6 +426,7 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl,
 
 static int db_select_delete_continue_hash(Process *p, DbTable *tbl,
 					  Eterm continuation, Eterm *ret);
+static int db_take_hash(Process *, DbTable *, Eterm, Eterm *);
 static void db_print_hash(int to,
 			  void *to_arg,
 			  int show,
@@ -536,6 +537,7 @@ DbTableMethod db_hash =
     db_select_delete_continue_hash,
     db_select_count_hash,
     db_select_count_continue_hash,
+    db_take_hash,
     db_delete_all_objects_hash,
     db_free_table_hash,
     db_free_table_continue_hash,
@@ -879,34 +881,45 @@ Ldone:
     return ret;
 }
 
+static Eterm
+get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval,
+              HashDbTerm *b1, HashDbTerm **bend)
+{
+    HashDbTerm* b2 = b1->next;
+    Eterm copy;
+
+    if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
+        while (b2 && has_key(tb, b2, key, hval)) {
+            b2 = b2->next;
+        }
+    }
+    copy = build_term_list(p, b1, b2, tb);
+    CHECK_TABLES();
+    if (bend) {
+        *bend = b2;
+    }
+    return copy;
+}
+
 int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 {
     DbTableHash *tb = &tbl->hash;
     HashValue hval;
     int ix;
-    HashDbTerm* b1;
+    HashDbTerm* b;
     erts_smp_rwmtx_t* lck;
 
     hval = MAKE_HASH(key);
     lck = RLOCK_HASH(tb,hval);
     ix = hash_to_ix(tb, hval);
-    b1 = BUCKET(tb, ix);
+    b = BUCKET(tb, ix);
 
-    while(b1 != 0) {
-	if (has_live_key(tb,b1,key,hval)) {
-	    HashDbTerm* b2 = b1->next;
-	    Eterm copy;
-
-	    if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
-		while(b2 != NULL && has_key(tb,b2,key,hval))
-		    b2 = b2->next;
-	    }
-	    copy = build_term_list(p, b1, b2, tb);
-	    CHECK_TABLES();
-	    *ret = copy;
+    while(b != 0) {
+        if (has_live_key(tb, b, key, hval)) {
+            *ret = get_term_list(p, tb, key, hval, b, NULL);
 	    goto done;
 	}
-	b1 = b1->next;
+        b = b->next;
     }
     *ret = NIL;
 done:
@@ -2069,6 +2082,46 @@ trap:
 
 }
     
+static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
+{
+    DbTableHash *tb = &tbl->hash;
+    HashDbTerm **bp, *b;
+    HashValue hval = MAKE_HASH(key);
+    erts_smp_rwmtx_t *lck = WLOCK_HASH(tb, hval);
+    int ix = hash_to_ix(tb, hval);
+    int nitems_diff = 0;
+
+    *ret = NIL;
+    for (bp = &BUCKET(tb, ix), b = *bp; b; bp = &b->next, b = b->next) {
+        if (has_live_key(tb, b, key, hval)) {
+            HashDbTerm *bend;
+
+            *ret = get_term_list(p, tb, key, hval, b, &bend);
+            while (b != bend) {
+                --nitems_diff;
+                if (nitems_diff == -1 && IS_FIXED(tb)) {
+                    /* Pseudo remove (no need to keep several of same key) */
+                    add_fixed_deletion(tb, ix);
+                    bp = &b->next;
+                    b->hvalue = INVALID_HASH;
+                    b = b->next;
+                } else {
+                    *bp = b->next;
+                    free_term(tb, b);
+                    b = *bp;
+                }
+            }
+            break;
+        }
+    }
+    WUNLOCK_HASH(lck);
+    if (nitems_diff) {
+        erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
+        try_shrink(tb);
+    }
+    return DB_ERROR_NONE;
+}
+
 /*
 ** Other interface routines (not directly coupled to one bif)
 */
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index a62a83a928..720c0659c3 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -383,6 +383,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 				 Eterm pattern,  Eterm *ret);
 static int db_select_delete_continue_tree(Process *p, DbTable *tbl, 
 					  Eterm continuation, Eterm *ret);
+static int db_take_tree(Process *, DbTable *, Eterm, Eterm *);
 static void db_print_tree(int to, void *to_arg,
 			  int show, DbTable *tbl);
 static int db_free_table_tree(DbTable *tbl);
@@ -431,6 +432,7 @@ DbTableMethod db_tree =
     db_select_delete_continue_tree,
     db_select_count_tree,
     db_select_count_continue_tree,
+    db_take_tree,
     db_delete_all_objects_tree,
     db_free_table_tree,
     db_free_table_continue_tree,
@@ -1722,6 +1724,28 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 
 }
 
+static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
+{
+    DbTableTree *tb = &tbl->tree;
+    TreeDbTerm *this;
+
+    *ret = NIL;
+    this = linkout_tree(tb, key, NULL);
+    if (this) {
+        Eterm copy, *hp, *hend;
+
+        hp = HAlloc(p, this->dbterm.size + 2);
+        hend = hp + this->dbterm.size + 2;
+        copy = db_copy_object_from_ets(&tb->common,
+                                       &this->dbterm, &hp, &MSO(p));
+        *ret = CONS(hp, copy, NIL);
+        hp += 2;
+        HRelease(p, hend, hp);
+        free_term(tb, this);
+    }
+    return DB_ERROR_NONE;
+}
+
 /*
 ** Other interface routines (not directly coupled to one bif)
 */
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 328b19dfc9..5ace93c8ed 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -165,6 +165,7 @@ typedef struct db_table_method
 				    DbTable* tb, /* [in out] */ 
 				    Eterm continuation, 
 				    Eterm* ret);
+    int (*db_take)(Process *, DbTable *, Eterm, Eterm *);
 
     int (*db_delete_all_objects)(Process* p,
 				 DbTable* db /* [in out] */ );
-- 
cgit v1.2.3


From 1e377a3c944518ecdfe82c113fe73de48a44519c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Tue, 3 Jun 2014 11:18:20 +0200
Subject: erts: tracing on send now works for registered processes

This bug was introduced in R16B. Testcases have been adapted to
verify the correct behaviour.
---
 erts/emulator/beam/bif.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 06a1230ca0..fcbeb6cf5c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1886,8 +1886,13 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 	Eterm id = erts_whereis_name_to_id(p, to);
 
 	rp = erts_proc_lookup(id);
-	if (rp)
+	if (rp) {
+	    if (IS_TRACED(p))
+		trace_send(p, to, msg);
+	    if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
+		save_calls(p, &exp_send);
 	    goto send_message;
+	}
 
 	pt = erts_port_lookup(id,
 			      (erts_port_synchronous_ops
-- 
cgit v1.2.3


From 82c048745efcb5d811ec0489858b821ef39ea387 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 19 May 2014 18:09:13 +0200
Subject: Replace erlang:binary_to_term() Erlang wrappers

Replace the 'erlang:binary_to_term/1' and 'erlang:binary_to_term/2'
Erlang wrappers taking care of failure after yield with management
of this in the hidden yield BIF.
---
 erts/emulator/beam/bif.tab    |  8 +++--
 erts/emulator/beam/external.c | 74 +++++++++++++++++++++++++++----------------
 2 files changed, 52 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 2d888862bf..9816883c9a 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -45,6 +45,7 @@ bif erlang:apply/3
 bif erlang:atom_to_list/1
 bif erlang:binary_to_list/1
 bif erlang:binary_to_list/3
+bif erlang:binary_to_term/1
 bif erlang:crc32/1
 bif erlang:crc32/2
 bif erlang:crc32_combine/3
@@ -151,8 +152,6 @@ bif erts_internal:port_command/3
 bif erts_internal:port_control/3
 bif erts_internal:port_close/1
 bif erts_internal:port_connect/2
-bif erts_internal:binary_to_term/1
-bif erts_internal:binary_to_term/2
 
 bif erts_internal:request_system_task/3
 bif erts_internal:check_process_code/2
@@ -479,6 +478,11 @@ bif erlang:load_nif/2
 bif erlang:call_on_load_function/1
 bif erlang:finish_after_on_load/2
 
+#
+# New Bifs in R13B04
+#
+bif erlang:binary_to_term/2
+
 #
 # The binary match bifs (New in R14A - EEP9)
 #
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index e376253400..8d240355b0 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -108,26 +108,17 @@ static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acm
 
 static Export binary_to_term_trap_export;
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
-static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b);
+static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b,
+				      Export *bif, Eterm arg0, Eterm arg1);
 
 void erts_init_external(void) {
-#if 1 /* In R16 */
     erts_init_trap_export(&term_to_binary_trap_export,
-			  am_erlang, am_term_to_binary_trap, 1,
+			  am_erts_internal, am_term_to_binary_trap, 1,
 			  &term_to_binary_trap_1);
 
     erts_init_trap_export(&binary_to_term_trap_export,
-			  am_erlang, am_binary_to_term_trap, 1,
+			  am_erts_internal, am_binary_to_term_trap, 1,
 			  &binary_to_term_trap_1);
-#else
-    sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export));
-    term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3];
-    term_to_binary_trap_export.code[0] = am_erlang;
-    term_to_binary_trap_export.code[1] = am_term_to_binary_trap;
-    term_to_binary_trap_export.code[2] = 1;
-    term_to_binary_trap_export.code[3] = (BeamInstr) em_apply_bif;
-    term_to_binary_trap_export.code[4] = (BeamInstr) &term_to_binary_trap_1;
-#endif    
     return;
 }
 
@@ -1186,6 +1177,8 @@ typedef struct B2TContext_t {
     Uint32 flags;
     SWord reds;
     Eterm trap_bin;
+    Export *bif;
+    Eterm arg[2];
     enum B2TState state;
     union {
 	B2TSizeContext sc;
@@ -1357,7 +1350,8 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
     Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
 
-    return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin);
+    return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL,
+			      THE_NON_VALUE, THE_NON_VALUE);
 }
 
 
@@ -1392,8 +1386,10 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src)
     return ctx;
 }
 
-static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b)
+static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b,
+				      Export *bif_init, Eterm arg0, Eterm arg1)
 {
+    BIF_RETTYPE ret_val;
 #ifdef EXTREME_B2T_TRAPPING
     SWord initial_reds = 1 + b2t_rand() % 4;
 #else
@@ -1410,6 +1406,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
 	ctx->state = B2TPrepare;
         ctx->aligned_alloc = NULL;
         ctx->flags = flags;
+	ctx->bif = bif_init;
+	ctx->arg[0] = arg0;
+	ctx->arg[1] = arg1;
         IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
     } else {
         is_first_call = 0;
@@ -1505,12 +1504,24 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start);
             /*fall through*/
         case B2TBadArg:
-            b2t_destroy_context(ctx);
-            if (!is_first_call) {
-                erts_set_gc_state(p, 1);
-            }
             BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
-            BIF_ERROR(p, BADARG & ~EXF_SAVETRACE);
+
+	    ASSERT(ctx->bif == bif_export[BIF_binary_to_term_1]
+		   || ctx->bif == bif_export[BIF_binary_to_term_2]);
+
+	    if (is_first_call)
+		ERTS_BIF_PREP_ERROR(ret_val, p, BADARG);
+	    else {
+                erts_set_gc_state(p, 1);
+		if (is_non_value(ctx->arg[1]))
+		    ERTS_BIF_PREP_ERROR_TRAPPED1(ret_val, p, BADARG, ctx->bif,
+						 ctx->arg[0]);
+		else
+		    ERTS_BIF_PREP_ERROR_TRAPPED2(ret_val, p, BADARG, ctx->bif,
+						 ctx->arg[0], ctx->arg[1]);
+	    }
+            b2t_destroy_context(ctx);
+	    return ret_val;
 
         case B2TDone:
             b2t_destroy_context(ctx);
@@ -1525,7 +1536,8 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
                 erts_set_gc_state(p, 1);
             }
             BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
-            return ctx->u.dc.res;
+	    ERTS_BIF_PREP_RET(ret_val, ctx->u.dc.res);
+	    return ret_val;
 
         default:
             ASSERT(!"Unknown state in binary_to_term");
@@ -1542,19 +1554,24 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
         erts_set_gc_state(p, 0);
     }
     BUMP_ALL_REDS(p);
-    BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin);
+
+    ERTS_BIF_PREP_TRAP1(ret_val, &binary_to_term_trap_export,
+			p, ctx->trap_bin);
+
+    return ret_val;
 }
 
-HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 1)
+HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 1)
 
-BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1)
+BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
 {
-    return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL);
+    return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_1],
+			      BIF_ARG_1, THE_NON_VALUE);
 }
 
-HIPE_WRAPPER_BIF_DISABLE_GC(erts_internal_binary_to_term, 2)
+HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 2)
 
-BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2)
+BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
 {
     Eterm opts;
     Eterm opt;
@@ -1575,7 +1592,8 @@ BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2)
     if (is_not_nil(opts))
         goto error;
 
-    return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL);
+    return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_2],
+			      BIF_ARG_1, BIF_ARG_2);
 
 error:
     BIF_ERROR(BIF_P, BADARG);
-- 
cgit v1.2.3


From 6db23381adb9c580e1d03cccf2a8b67c6cc5189b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Jun 2014 18:22:05 +0200
Subject: erts: Fix cache alignment for ETS write_concurrency locks

Effects if sizeof(erts_smp_rwmtx_t) > 64:
+ Better performance due to less false sharing
- Increased memory footprint per table
---
 erts/emulator/beam/erl_db_hash.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 908cec11d4..e68081a5b1 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2014. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +42,7 @@ typedef struct hash_db_term {
 typedef struct db_table_hash_fine_locks {
     union {
 	erts_smp_rwmtx_t lck;
-	byte _cache_line_alignment[64];
+	byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_smp_rwmtx_t))];
     }lck_vec[DB_HASH_LOCK_CNT];
 } DbTableHashFineLocks;
 
-- 
cgit v1.2.3


From 1d583e08cd5235881ede6f92800c25ddee13056f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 27 Jun 2014 20:13:46 +0200
Subject: erts: Fix size overflow bugs in memory allocation

---
 erts/emulator/beam/erl_alloc_util.c |  9 +++++++++
 erts/emulator/beam/erl_binary.h     | 31 ++++++++++++++++++-------------
 erts/emulator/beam/sys.h            |  2 ++
 3 files changed, 29 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 45f0cc4312..a4e164bf51 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3274,6 +3274,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 
     ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC));
 
+    if (umem_sz > (ERTS_UINT_MAX - ERTS_UINT_MAX/100)) {
+	/* Do an overly conservative _overflow_ check here so we don't
+	 * have to deal with it from here on. I guess we could be more accurate
+	 * but I don't think the need to allocate over 99% of the address space
+	 * will ever arise on any machine, neither 32 nor 64 bit.
+	 */
+	return NULL;
+    }
+
     blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 6c9f53ce87..06dfeb1260 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -236,6 +236,8 @@ erts_bin_drv_alloc_fnf(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
     void *res;
+    if (bsize < size) /* overflow */
+	return NULL;
     res = erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     return (Binary *) res;
@@ -246,6 +248,8 @@ erts_bin_drv_alloc(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
     void *res;
+    if (bsize < size) /* overflow */
+	erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size);
     res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     return (Binary *) res;
@@ -257,6 +261,8 @@ erts_bin_nrml_alloc(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
     void *res;
+    if (bsize < size) /* overflow */
+	erts_alloc_enomem(ERTS_ALC_T_BINARY, size);
     res = erts_alloc(ERTS_ALC_T_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     return (Binary *) res;
@@ -267,11 +273,12 @@ erts_bin_realloc_fnf(Binary *bp, Uint size)
 {
     Binary *nbp;
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
+	                                            : ERTS_ALC_T_BINARY;
     ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
-    if (bp->flags & BIN_FLAG_DRV)
-	nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize);
-    else
-	nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize);
+    if (bsize < size) /* overflow */
+	return NULL;
+    nbp = erts_realloc_fnf(type, (void *) bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
     return nbp;
 }
@@ -281,17 +288,14 @@ erts_bin_realloc(Binary *bp, Uint size)
 {
     Binary *nbp;
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
+	                                            : ERTS_ALC_T_BINARY;
     ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
-    if (bp->flags & BIN_FLAG_DRV)
-	nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize);
-    else
-	nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize);
+    if (bsize < size) /* overflow */
+	erts_realloc_enomem(type, bp, size);
+    nbp = erts_realloc_fnf(type, (void *) bp, bsize);
     if (!nbp)
-	erts_realloc_n_enomem(ERTS_ALC_T2N(bp->flags & BIN_FLAG_DRV
-					   ? ERTS_ALC_T_DRV_BINARY
-					   : ERTS_ALC_T_BINARY),
-			      bp,
-			      bsize);
+	erts_realloc_enomem(type, bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
     return nbp;
 }
@@ -312,6 +316,7 @@ erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
 {
     Uint bsize = ERTS_MAGIC_BIN_SIZE(size);
     Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize);
+    ASSERT(bsize > size);
     if (!bptr)
 	erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize);
     ERTS_CHK_BIN_ALIGNMENT(bptr);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 05f07e57b2..3d8dd9c6d0 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -274,6 +274,7 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 typedef unsigned int Eterm;
 typedef unsigned int Uint;
 typedef int          Sint;
+#define ERTS_UINT_MAX UINT_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
 #define ErtsStrToSint strtol
 #else
@@ -347,6 +348,7 @@ typedef long long          Sint;
 
 typedef Uint UWord;
 typedef Sint SWord;
+#define ERTS_UINT_MAX ERTS_UWORD_MAX
 
 #endif /* HALFWORD_HEAP */
 
-- 
cgit v1.2.3


From acf19fc9190985f643af06293141a1083f032563 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Wed, 23 Oct 2013 19:17:11 +0200
Subject: Use offsetof() in io.c

This silences the following UBSan errors:

beam/io.c:7131:27: runtime error: member access within null pointer of type 'ErlDrvSysInfo'
beam/io.c:7140:20: runtime error: member access within null pointer of type 'ErlDrvSysInfo'
beam/io.c:7166:20: runtime error: member access within null pointer of type 'ErlDrvSysInfo'
beam/io.c:7174:20: runtime error: member access within null pointer of type 'ErlDrvSysInfo'
---
 erts/emulator/beam/io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index edf4a28784..c34a5b17d0 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7166,7 +7166,7 @@ char *driver_dl_error(void)
 
 
 #define ERL_DRV_SYS_INFO_SIZE(LAST_FIELD) \
-  (((size_t) &((ErlDrvSysInfo *) 0)->LAST_FIELD) \
+  (offsetof(ErlDrvSysInfo, LAST_FIELD) \
    + sizeof(((ErlDrvSysInfo *) 0)->LAST_FIELD))
 
 void
-- 
cgit v1.2.3


From a8cbf025f6e20a68b6575747200be149c6c09932 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Wed, 23 Oct 2013 19:34:57 +0200
Subject: Properly handle SINT_MIN in small_to_big()

As there is no overflow for signed integers, -SINT_MIN is undefined behaviour
and the cast to unsigned needs to happen before negation. SINT_MIN denotes the
minimum value that can be stored in the Sint type.

beam/big.c:1512:6: runtime error: negation of -9223372036854775808 cannot be represented in type 'Sint' (aka 'long'); cast to an unsigned type to negate this value to itself
---
 erts/emulator/beam/big.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 41a041eba6..4d087bf967 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1506,13 +1506,15 @@ Eterm uword_to_big(UWord x, Eterm *y)
 */
 Eterm small_to_big(Sint x, Eterm *y)
 {
+    Uint xu;
     if (x >= 0) {
+        xu = x;
 	*y = make_pos_bignum_header(1);
     } else {
-	x = -x;
+        xu = -(Uint)x;
 	*y = make_neg_bignum_header(1);
     }
-    BIG_DIGIT(y, 0) = x;
+    BIG_DIGIT(y, 0) = xu;
     return make_big(y);
 }
 
-- 
cgit v1.2.3


From bbfc75ea8795d26a2fe9254f3f646e761f2ad61e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 4 Jul 2014 19:34:24 +0200
Subject: erts: Add histogram to lcnt

---
 erts/emulator/beam/erl_bif_info.c   |  48 +++++++-----
 erts/emulator/beam/erl_lock_count.c | 149 ++++++++++++++++++++----------------
 erts/emulator/beam/erl_lock_count.h |  20 ++++-
 erts/emulator/beam/io.c             |   2 +-
 4 files changed, 129 insertions(+), 90 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4d5e55aaf5..6915765dab 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3856,16 +3856,19 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
     Uint tries = 0, colls = 0;
     unsigned long timer_s = 0, timer_ns = 0, timer_n = 0;
     unsigned int  line = 0;
+    unsigned int  i;
     
     Eterm af, uil;
     Eterm uit, uic;
     Eterm uits, uitns, uitn;
     Eterm tt, tstat, tloc, t;
+    Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE];
 	
     /* term:
-     * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}]
+     * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}},
+     *   { .. histogram .. }]
      */
-    
+
     tries = (Uint) ethr_atomic_read(&stats->tries);
     colls = (Uint) ethr_atomic_read(&stats->colls);
    
@@ -3874,23 +3877,27 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
     timer_ns = stats->timer.ns;
     timer_n  = stats->timer_n;
    
-    af    = erts_atom_put(stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); 
+    af    = erts_atom_put((byte *)stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1);
     uil   = erts_bld_uint( hpp, szp, line);
     tloc  = erts_bld_tuple(hpp, szp, 2, af, uil);
     
-    uit   = erts_bld_uint( hpp, szp, tries);             
-    uic   = erts_bld_uint( hpp, szp, colls);             
-    
+    uit   = erts_bld_uint( hpp, szp, tries);
+    uic   = erts_bld_uint( hpp, szp, colls);
+
     uits  = erts_bld_uint( hpp, szp, timer_s);
     uitns = erts_bld_uint( hpp, szp, timer_ns);
     uitn  = erts_bld_uint( hpp, szp, timer_n);
     tt    = erts_bld_tuple(hpp, szp, 3, uits, uitns, uitn);
 
     tstat = erts_bld_tuple(hpp, szp, 3, uit, uic, tt);
-    
-    t     = erts_bld_tuple(hpp, szp, 2, tloc, tstat);
-    
-    res   = erts_bld_cons( hpp, szp, t, res);
+
+    for(i = 0; i < ERTS_LCNT_HISTOGRAM_SLOT_SIZE; i++) {
+	vhist[i] = erts_bld_uint(hpp, szp, stats->hist.ns[i]);
+    }
+    thist  = erts_bld_tuplev(hpp, szp, ERTS_LCNT_HISTOGRAM_SLOT_SIZE, vhist);
+
+    t   = erts_bld_tuple(hpp, szp, 3, tloc, tstat, thist);
+    res = erts_bld_cons( hpp, szp, t, res);
 
     return res;
 }
@@ -3911,13 +3918,13 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock
     
     ASSERT(ltype);
     
-    type  = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);           
-    name  = erts_atom_put(lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); 
+    type  = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
+    name  = erts_atom_put((byte *)lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1);
 
     if (lock->flag & ERTS_LCNT_LT_ALLOC) {
 	/* use allocator types names as id's for allocator locks */
 	ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id));
-	id    = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
+	id    = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
     } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) {
 	/* use registered names as id's for process locks if available */
 	proc  = erts_proc_lookup(lock->id);
@@ -3928,16 +3935,15 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock
 	    id = lock->id;
 	}
     } else {
-	id    = lock->id;                                    
+	id = lock->id;
     }
-    
+
     for (i = 0; i < lock->n_stats; i++) {
 	stats = lcnt_build_lock_stats_term(hpp, szp, &(lock->stats[i]), stats);
     }
-	
-    t     = erts_bld_tuple(hpp, szp, 4, name, id, type, stats);
-    
-    res   = erts_bld_cons( hpp, szp, t, res);          
+
+    t   = erts_bld_tuple(hpp, szp, 4, name, id, type, stats);
+    res = erts_bld_cons( hpp, szp, t, res);
 
     return res;
 }
@@ -3957,12 +3963,12 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *da
     dtns = erts_bld_uint( hpp, szp, data->duration.ns);
     tdt  = erts_bld_tuple(hpp, szp, 2, dts, dtns);
     
-    adur = erts_atom_put(str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1);
+    adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1);
     tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt);
    
     /* lock tuple */
     
-    aloc = erts_atom_put(str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1);
+    aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1);
     	
     for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) {
 	lloc = lcnt_build_lock_term(hpp, szp, lock, lloc);
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 6f44bf097b..17ddfdc8ae 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -61,6 +61,25 @@ static ERTS_INLINE void lcnt_unlock(void) {
     ethr_mutex_unlock(&lcnt_data_lock); 
 }
 
+const int log2_tab64[64] = {
+    63,  0, 58,  1, 59, 47, 53,  2,
+    60, 39, 48, 27, 54, 33, 42,  3,
+    61, 51, 37, 40, 49, 18, 28, 20,
+    55, 30, 34, 11, 43, 14, 22,  4,
+    62, 57, 46, 52, 38, 26, 32, 41,
+    50, 36, 17, 19, 29, 10, 13, 21,
+    56, 45, 25, 31, 35, 16,  9, 12,
+    44, 24, 15,  8, 23,  7,  6,  5};
+
+static ERTS_INLINE int lcnt_log2(Uint64 v) {
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    v |= v >> 32;
+    return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58];
+}
 
 static char* lcnt_lock_type(Uint16 flag) {
     switch(flag & ERTS_LCNT_LT_ALL) {
@@ -81,19 +100,20 @@ static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) {
     stats->timer_n  = 0;
     stats->file     = (char *)str_undefined;
     stats->line     = 0;
+    sys_memzero(stats->hist.ns, sizeof(stats->hist.ns));
 }
 
 static void lcnt_time(erts_lcnt_time_t *time) {
-#ifdef HAVE_GETHRTIME
+#if 0 || defined(HAVE_GETHRTIME)
     SysHrTime hr_time;
     hr_time  = sys_gethrtime();
     time->s  = (unsigned long)(hr_time / 1000000000LL);
     time->ns = (unsigned long)(hr_time - 1000000000LL*time->s);
-#else    
-    SysTimeval tv;
-    sys_gettimeofday(&tv);
-    time->s  = tv.tv_sec;
-    time->ns = tv.tv_usec*1000LL;
+#else
+  SysTimeval tv;
+  sys_gettimeofday(&tv);
+  time->s  = tv.tv_sec;
+  time->ns = tv.tv_usec*1000LL;
 #endif
 }
 
@@ -111,22 +131,20 @@ static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_
 	dns += 1000000000LL;
     }
 
+    ASSERT(ds >= 0);
+
     d->s  = ds;
     d->ns = dns;
 }
 
-/* difference d must be positive */
+/* difference d must be non-negative */
 
 static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) {
-    unsigned long ngns = 0;
-    
     t->s  += d->s;
     t->ns += d->ns;
 
-    ngns   = t->ns / 1000000000LL;
+    t->s  += t->ns / 1000000000LL;
     t->ns  = t->ns % 1000000000LL;
-    
-    t->s  += ngns;
 }
 
 static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) {
@@ -158,59 +176,64 @@ static char* lock_opt(Uint16 flag) {
     return "--";
 }
 
-static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) {
-    erts_aint_t colls, tries, w_state, r_state;
-    erts_lcnt_lock_stats_t *stats = NULL;
-    
+static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) {
+    erts_aint_t w_state, r_state;
     char *type;
-    int i;
-    
+
+    if (strcmp(lock->name, "run_queue") != 0) return;
     type = lcnt_lock_type(lock->flag);
     r_state = ethr_atomic_read(&lock->r_state);
     w_state = ethr_atomic_read(&lock->w_state);
-
     
     if (lock->flag & flag) {
-        erts_printf("%20s [%30s] [r/w state %4ld/%4ld] id %T %s\r\n", 
-		action, 
-		lock->name, 
-		r_state, 
-		w_state, 
-		lock->id, 
-		extra);
+        erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n",
+		action,
+		lock->name,
+		r_state,
+		w_state,
+		type,
+		lock->id);
     }
 }
-
-static void print_lock(erts_lcnt_lock_t *lock, char *action) {
-    if (strcmp(lock->name, "proc_main") == 0) {
-        print_lock_x(lock, ERTS_LCNT_LT_ALL, action, "");
-    }
-}
-
 #endif
 
 static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char *file, unsigned int line) {
     unsigned int i;
     erts_lcnt_lock_stats_t *stats = NULL;
-    
-    for (i = 0; i < lock->n_stats; i++) {
-	if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) {
-	    return &(lock->stats[i]);
-	}
-    }
-    if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
-	stats = &lock->stats[lock->n_stats];
-	lock->n_stats++;
 
-	stats->file = file;
-	stats->line = line;
-	return stats;
+    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
+	for (i = 0; i < lock->n_stats; i++) {
+	    if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) {
+		return &(lock->stats[i]);
+	    }
+	}
+	if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
+	    stats = &lock->stats[lock->n_stats];
+	    lock->n_stats++;
+	    stats->file = file;
+	    stats->line = line;
+	    return stats;
+	}
     }
     return &lock->stats[0];
+}
+
+static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_wait) {
+    int idx;
+    unsigned long r;
 
+    if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) {
+	idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1;
+    } else {
+	r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT;
+	if (r) idx = lcnt_log2(r);
+	else idx = 0;
+    }
+    hist->ns[idx]++;
 }
 
-static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_wait) {
+static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict,
+	                      erts_lcnt_time_t *time_wait) {
     
     ethr_atomic_inc(&stats->tries);
 
@@ -220,6 +243,7 @@ static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflic
     if (time_wait) {
 	lcnt_time_add(&(stats->timer), time_wait);
 	stats->timer_n++;
+	lcnt_update_stats_hist(&stats->hist,time_wait);
     }
 }
 
@@ -330,8 +354,9 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock)
 /* interface to erl_threads.h */
 /* only lock on init and destroy, all others should use atomics */
 void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { 
-    erts_lcnt_init_lock_x(lock, name, flag, am_undefined);
+    erts_lcnt_init_lock_x(lock, name, flag, NIL);
 }
+
 void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { 
     int i;
     if (!name) {
@@ -360,7 +385,6 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter
     }
 
     erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock);
-    
     lcnt_unlock();
 }
 
@@ -417,8 +441,9 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
     
     if ((w_state > 0) || (r_state > 0)) {
 	eltd->lock_in_conflict = 1;
-	if (eltd->timer_set == 0)
+	if (eltd->timer_set == 0) {
 	    lcnt_time(&eltd->timer);
+	}
 	eltd->timer_set++;
     } else {
 	eltd->lock_in_conflict = 0;
@@ -433,7 +458,7 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
     if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
 
     w_state = ethr_atomic_read(&lock->w_state);
-    ethr_atomic_inc( &lock->w_state);
+    ethr_atomic_inc(&lock->w_state);
 
     eltd = lcnt_get_thread_data();
 
@@ -446,10 +471,10 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
 	 * 'atomicly'. All other locks will block the thread if w_state > 0
 	 * i.e. locked.
 	 */
-	if (eltd->timer_set == 0)
+	if (eltd->timer_set == 0) {
 	    lcnt_time(&eltd->timer);
+	}
 	eltd->timer_set++;
-
     } else {
 	eltd->lock_in_conflict = 0;
     }
@@ -459,11 +484,10 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
 
 void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) {
     /* should check if this thread was "waiting" */
-    
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
     if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
 
-    ethr_atomic_dec( &lock->w_state);
+    ethr_atomic_dec(&lock->w_state);
 }
 
 /* erts_lcnt_lock_post
@@ -491,7 +515,7 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
     if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
 	flowstate = ethr_atomic_read(&lock->flowstate);
 	ASSERT(flowstate == 0);
-    	ethr_atomic_inc( &lock->flowstate);
+	ethr_atomic_inc(&lock->flowstate);
     }
 #endif
     
@@ -500,19 +524,12 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
     ASSERT(eltd);
 
     /* if lock was in conflict, time it */
-    
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
-	stats = lcnt_get_lock_stats(lock, file, line);
-    } else {
-	stats = &lock->stats[0];
-    }
-
+    stats = lcnt_get_lock_stats(lock, file, line);
     if (eltd->timer_set) {
 	lcnt_time(&timer);
 	
 	lcnt_time_diff(&time_wait, &timer, &(eltd->timer));
 	lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait);
-	
 	eltd->timer_set--;
 	ASSERT(eltd->timer_set >= 0);
     } else {
@@ -541,11 +558,11 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
     /* flowstate */
     flowstate = ethr_atomic_read(&lock->flowstate);
     ASSERT(flowstate == 1);
-    ethr_atomic_dec( &lock->flowstate);
+    ethr_atomic_dec(&lock->flowstate);
     
     /* write state */
     w_state = ethr_atomic_read(&lock->w_state);
-    ASSERT(w_state > 0)
+    ASSERT(w_state > 0);
 #endif
     ethr_atomic_dec(&lock->w_state);
 }
@@ -582,9 +599,7 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
     	ethr_atomic_inc( &lock->flowstate);
 #endif
 	ethr_atomic_inc(&lock->w_state);
-	
 	lcnt_update_stats(&(lock->stats[0]), 0, NULL);
-
     } else {
         ethr_atomic_inc(&lock->stats[0].tries);
         ethr_atomic_inc(&lock->stats[0].colls);
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 75f7cd028b..ffbb93da1b 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -35,6 +35,10 @@
  * 	| | | - collisions (including trylock busy)
  * 	| | | - timer (time spent in waiting for lock)
  * 	| | | - n_timer (collisions excluding trylock busy)
+ * 	| | | - histogram
+ * 	| | | | - # 0 = log2(lock wait_time ns)
+ * 	| | | | - ...
+ * 	| | | | - # n = log2(lock wait_time ns)
  *
  * 	Each instance of a lock is the unique lock, i.e. set and id in that set.
  * 	For each lock there is a set of statistics with where and what impact
@@ -68,8 +72,17 @@
 
 #include "ethread.h"
 
+#define ERTS_LCNT_MAX_LOCK_LOCATIONS  (10)
 
-#define ERTS_LCNT_MAX_LOCK_LOCATIONS (10)
+/* histogram */
+#define ERTS_LCNT_HISTOGRAM_MAX_NS    (((unsigned long)1LL << 28) - 1)
+#if 0 || defined(HAVE_GETHRTIME)
+#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30)
+#define ERTS_LCNT_HISTOGRAM_RSHIFT    (0)
+#else
+#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (20)
+#define ERTS_LCNT_HISTOGRAM_RSHIFT    (10)
+#endif
 
 #define ERTS_LCNT_LT_SPINLOCK   (((Uint16) 1) << 0)
 #define ERTS_LCNT_LT_RWSPINLOCK (((Uint16) 1) << 1)
@@ -104,6 +117,10 @@ typedef struct {
     
 extern erts_lcnt_time_t timer_start;
 
+typedef struct {
+   Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */
+} erts_lcnt_hist_t;
+
 typedef struct erts_lcnt_lock_stats_s {
     /* "tries" and "colls" needs to be atomic since
      * trylock busy does not aquire a lock and there
@@ -118,6 +135,7 @@ typedef struct erts_lcnt_lock_stats_s {
     
     unsigned long timer_n;    /* #times waited for lock */
     erts_lcnt_time_t timer;   /* total wait time for lock */
+    erts_lcnt_hist_t hist;
 } erts_lcnt_lock_stats_t;
 
 /* rw locks uses both states, other locks only uses w_state */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index edf4a28784..d028737664 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -6129,7 +6129,7 @@ driver_pdl_create(ErlDrvPort dp)
 	return NULL;
     pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK,
 		     sizeof(struct erl_drv_port_data_lock));
-    erts_mtx_init(&pdl->mtx, "port_data_lock");
+    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1);
     pdl_init_refc(pdl);
     erts_port_inc_refc(pp);
     pdl->prt = pp;
-- 
cgit v1.2.3


From e1f903d5210fb5b91ff47229fb57bf3b0edeb79e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 11 Jun 2014 23:27:06 +0200
Subject: erts: Introduce erlang:fun_info_mfa/1

Introduced for proc_lib:init_p/3
---
 erts/emulator/beam/bif.tab        |  4 ++++
 erts/emulator/beam/erl_bif_info.c | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 011e49f1fe..e68b8e6274 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -600,6 +600,10 @@ bif maps:values/1
 
 bif erts_internal:cmp_term/2
 
+#
+# New in 17.1.
+#
+bif erlang:fun_info_mfa/1
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4d5e55aaf5..faa2655c11 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3055,6 +3055,25 @@ fun_info_2(BIF_ALIST_2)
     return TUPLE2(hp, what, val);
 }
 
+BIF_RETTYPE
+fun_info_mfa_1(BIF_ALIST_1)
+{
+    Process* p = BIF_P;
+    Eterm fun = BIF_ARG_1;
+    Eterm* hp;
+
+    if (is_fun(fun)) {
+	ErlFunThing* funp = (ErlFunThing *) fun_val(fun);
+	hp = HAlloc(p, 4);
+	BIF_RET(TUPLE3(hp,funp->fe->module,funp->fe->address[-2],make_small(funp->arity)));
+    } else if (is_export(fun)) {
+	Export* exp = (Export *) ((UWord) (export_val(fun))[1]);
+	hp = HAlloc(p, 4);
+	BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2])));
+    }
+    BIF_ERROR(p, BADARG);
+}
+
 BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) 
 {
    if(is_internal_pid(BIF_ARG_1)) {
-- 
cgit v1.2.3


From 961cfeb7b30d721ac8264261d89bb7a4bd3182e5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 5 Aug 2014 17:33:19 +0200
Subject: Fix abort of nosuspend-tasks in erts_port_task_schedule()

The counter for the amount of outstanding data in the port queue
became inconsistent when aborting nosuspend-signals in
erts_port_task_schedule(). This since the counter was subtracted
by the data size of the signal although the data size had never
been added to it. This inconsistency caused the port queue to
remain in a busy state forever.
---
 erts/emulator/beam/erl_port_task.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 31d9a1e26e..2fc95ed5d8 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -798,12 +798,13 @@ schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp)
 static ERTS_INLINE void
 abort_nosuspend_task(Port *pp,
 		     ErtsPortTaskType type,
-		     ErtsPortTaskTypeData *tdp)
+		     ErtsPortTaskTypeData *tdp,
+		     int bpq_data)
 {
 
     ASSERT(type == ERTS_PORT_TASK_PROC_SIG);
 
-    if (!pp->sched.taskq.bpq)
+    if (!bpq_data)
 	tdp->psig.callback(NULL,
 			   ERTS_PORT_SFLG_INVALID,
 			   ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND,
@@ -1345,7 +1346,7 @@ erts_port_task_abort_nosuspend_tasks(Port *pp)
 #endif
 	schedule_port_task_handle_list_free(pthlp);
 
-	abort_nosuspend_task(pp, type, &td);
+	abort_nosuspend_task(pp, type, &td, pp->sched.taskq.bpq != NULL);
     }
 }
 
@@ -1525,7 +1526,7 @@ abort_nosuspend:
 	erts_port_dec_refc(pp);
 #endif
 
-    abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td);
+    abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td, 0);
 
     ASSERT(ns_pthlp);
     erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp);
-- 
cgit v1.2.3


From ad57501215fc02a39abf3fbce978dbc43d010859 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 4 Aug 2014 11:32:52 +0200
Subject: erts: Print error reason when malloc fails

---
 erts/emulator/beam/erl_lock_check.c | 17 +++++++----------
 erts/emulator/beam/erl_lock_count.c | 12 ++++++++++++
 erts/emulator/beam/erl_process.c    |  6 ++++++
 erts/emulator/beam/utils.c          |  3 +++
 4 files changed, 28 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index c13eb87012..c665aa51a2 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -227,8 +227,7 @@ rw_op_str(Uint16 flags)
     case ERTS_LC_FLG_LO_READ:
 	return " (r)";
     case ERTS_LC_FLG_LO_WRITE:
-	erts_fprintf(stderr, "\nInternal error\n");
-	lc_abort();
+	ERTS_INTERNAL_ERROR("Only write flag present");
     default:
 	break;
     }
@@ -311,8 +310,7 @@ static ERTS_INLINE void lc_free(void *p)
 static void *lc_core_alloc(void)
 {
     lc_unlock();
-    erts_fprintf(stderr, "Lock checker out of memory!\n");
-    lc_abort();
+    ERTS_INTERNAL_ERROR("Lock checker out of memory!\n");
 }
 
 #else
@@ -325,8 +323,7 @@ static void *lc_core_alloc(void)
     fbs = (erts_lc_free_block_t *) malloc(sizeof(erts_lc_free_block_t)
 					  * ERTS_LC_FB_CHUNK_SIZE);
     if (!fbs) {
-	erts_fprintf(stderr, "Lock checker failed to allocate memory!\n");
-	lc_abort();
+        ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
     }
     for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
@@ -366,11 +363,11 @@ create_locked_locks(char *thread_name)
 {
     erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t));
     if (!l_lcks)
-	lc_abort();
+	ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
 
     l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
     if (!l_lcks->thread_name)
-	lc_abort();
+	ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
 
     l_lcks->emu_thread = 0;
     l_lcks->tid = erts_thr_self();
@@ -691,7 +688,7 @@ erts_lc_set_thread_name(char *thread_name)
 	free((void *) l_lcks->thread_name);
 	l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
 	if (!l_lcks->thread_name)
-	    lc_abort();
+	    ERTS_INTERNAL_ERROR("strdup failed");
     }
     l_lcks->emu_thread = 1;
 }
@@ -1330,7 +1327,7 @@ erts_lc_init(void)
 #endif /* #ifdef ERTS_LC_STATIC_ALLOC */
 
     if (ethr_spinlock_init(&free_blocks_lock) != 0)
-	lc_abort();
+	ERTS_INTERNAL_ERROR("spinlock_init failed");
 
     erts_tsd_key_create(&locks_key,"erts_lock_check_key");
 }
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 17ddfdc8ae..cf6996ea06 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -151,6 +151,9 @@ static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) {
     erts_lcnt_thread_data_t *eltd;
    
     eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t));
+    if (!eltd) {
+        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
+    }
     eltd->timer_set = 0;
     eltd->lock_in_conflict = 0;
 
@@ -272,6 +275,9 @@ void erts_lcnt_init() {
     
     /* init lcnt structure */
     erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t));
+    if (!erts_lcnt_data) {
+        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
+    }
     erts_lcnt_data->current_locks = erts_lcnt_list_init();
     erts_lcnt_data->deleted_locks = erts_lcnt_list_init();
 
@@ -293,6 +299,9 @@ erts_lcnt_lock_list_t *erts_lcnt_list_init(void) {
     erts_lcnt_lock_list_t *list;
     
     list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t));
+    if (!list) {
+        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
+    }
     list->head = NULL;
     list->tail = NULL;
     list->n    = 0;
@@ -399,6 +408,9 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
 	/* copy structure and insert the copy */
 
 	deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t));
+        if (!deleted_lock) {
+            ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
+        }
 	memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t));
 
 	deleted_lock->next = NULL;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b73f9b7f92..1606ad119d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2211,6 +2211,9 @@ aux_work_timeout_early_init(int no_schedulers)
     p = (UWord) malloc((sizeof(ErtsAuxWorkTmo)
 			+ sizeof(erts_atomic32_t)*(no_schedulers+1))
 		       + ERTS_CACHE_LINE_SIZE-1);
+    if (!p) {
+        ERTS_INTERNAL_ERROR("malloc failed to allocate memory!");
+    }
     if (p & ERTS_CACHE_LINE_MASK)
 	p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
     ASSERT((p & ERTS_CACHE_LINE_MASK) == 0);
@@ -7818,6 +7821,9 @@ erts_start_schedulers(void)
 
 #ifdef ETHR_HAVE_THREAD_NAMES
     opts.name = malloc(80);
+    if (!opts.name) {
+        ERTS_INTERNAL_ERROR("malloc failed to allocate memory!");
+    }
 #endif
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 72092ec7b0..55f9e68e78 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3948,6 +3948,9 @@ erts_save_emu_args(int argc, char **argv)
 	size += sz+1;
     } 
     ptr = (char *) malloc(size);
+    if (!ptr) {
+        ERTS_INTERNAL_ERROR("malloc failed to allocate memory!");
+    }
 #ifdef DEBUG
     end_ptr = ptr + size;
 #endif
-- 
cgit v1.2.3


From 18c03b13d1a53282e69160fa71a7195a11e84392 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 7 Aug 2014 22:02:25 +0200
Subject: Fix emigrate bug in erts_port_task_schedule()

While current run-queue lock is unlocked in the call to
erts_check_emigration_need() from erts_port_task_schedule() the
port can be migrated to another run-queue by another thread.
The code in erts_port_task_schedule() needs to check if this
has occurred when returning from erts_check_emigration_need(), and
if so respect the migration decision. When this was not done,
the thread calling erts_port_task_schedule() held the wrong
run-queue lock which caused invalid updates of the port task queue.

This bug was automatically fixed by the rewrites in the branch
rickard/r16b/port-optimizations-fixes/OTP-10336 (commit
56cef897ca3ad2377e34a6ea5800a54a28cbeb6e) introduced in erts-5.10
and do not effect erts versions after that.
---
 erts/emulator/beam/erl_port_task.c | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 3dc7c14faf..11027dda7a 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -576,8 +576,20 @@ erts_port_task_schedule(Eterm id,
     if (!pp->sched.taskq && !pp->sched.exe_taskq) {
 #ifdef ERTS_SMP
 	ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
-	if (xrunq) {
-	    /* Port emigrated ... */
+	ERTS_SMP_LC_ASSERT(runq != xrunq);
+	if (runq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)) {
+	    /*
+	     * Someone else migrated this port while we had the run-queue
+	     * lock on runq unlocked in erts_check_emigration_need(). Respect
+	     * that migration decision...
+	     */
+	    erts_smp_runq_unlock(runq);
+	    if (xrunq)
+		erts_smp_runq_unlock(xrunq);
+	    runq = erts_port_runq(pp);
+	}
+	else if (xrunq) {
+	    /* Emigrate port... */
 	    erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
 	    erts_smp_runq_unlock(runq);
 	    runq = xrunq;
@@ -929,6 +941,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
 #ifdef ERTS_SMP
 	xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
+	ERTS_SMP_LC_ASSERT(runq != xrunq);
+	ERTS_SMP_LC_ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
 	if (!xrunq) {
 #endif
 	    enqueue_port(runq, pp);
@@ -938,7 +952,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 #ifdef ERTS_SMP
 	}
 	else {
-	    /* Port emigrated ... */
+	    /* Emigrate port... */
 	    erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
 	    enqueue_port(xrunq, pp);
 	    ASSERT(pp->sched.exe_taskq);
-- 
cgit v1.2.3


From 9d3c22934491afe85fbcf8543ae43fb2eb1ab387 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 4 Aug 2014 12:04:45 +0200
Subject: erts: Fix neg int overflow when sint is min size

When INT64_MIN is the value of a Sint64 we have to first cast it to
an Uint64 before negating it. Otherwise we get an integer overflow
which is undefined behaviour and in gcc 4.9 this results in -0 instead
of -9223372036854775808 in gcc 4.8.
---
 erts/emulator/beam/big.c            | 18 ++++++++++++------
 erts/emulator/beam/big.h            |  2 +-
 erts/emulator/beam/erl_bif_binary.c | 16 ++++++++--------
 erts/emulator/beam/erl_init.c       |  6 ++++--
 4 files changed, 25 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 41a041eba6..520034d68a 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -274,6 +274,9 @@
 	_b = _b << _s;							\
 	_vn1 = _b >> H_EXP;						\
 	_vn0 = _b & LO_MASK;						\
+        /* Sometimes _s is 0 which triggers undefined behaviour for the \
+           (_a0>>(D_EXP-_s)) shift, but this is ok because the          \
+           & -s will make it all to 0 later anyways. */                 \
 	_un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1)));	\
 	_un10 = _a0 << _s;						\
 	_un1 = _un10 >> H_EXP;						\
@@ -1540,21 +1543,24 @@ Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp)
 Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 {
     Eterm *hp = *hpp;
+    Uint64 ux;
     int neg;
-    if (x >= 0)
+    if (x >= 0) {
 	neg = 0;
+        ux = x;
+    }
     else {
 	neg = 1;
-	x = -x;
+	ux = -(Uint64)x;
     }
 #if defined(ARCH_32) || HALFWORD_HEAP
-    if (x >= (((Uint64) 1) << 32)) {
+    if (ux >= (((Uint64) 1) << 32)) {
 	if (neg)
 	    *hp = make_neg_bignum_header(2);
 	else
 	    *hp = make_pos_bignum_header(2);
-	BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff));
-	BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff));
+	BIG_DIGIT(hp, 0) = (Uint) (ux & ((Uint) 0xffffffff));
+	BIG_DIGIT(hp, 1) = (Uint) ((ux >> 32) & ((Uint) 0xffffffff));
 	*hpp += 3;
     }
     else
@@ -1564,7 +1570,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	    *hp = make_neg_bignum_header(1);
 	else
 	    *hp = make_pos_bignum_header(1);
-	BIG_DIGIT(hp, 0) = (Uint) x;
+	BIG_DIGIT(hp, 0) = (Uint) ux;
 	*hpp += 2;
     }
     return make_big(hp);
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index d80111822e..da31876d75 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -101,7 +101,7 @@ typedef Uint  dsize_t;	 /* Vector size type */
 #define ERTS_SINT64_HEAP_SIZE(X)				\
   (IS_SSMALL((X))						\
    ? 0								\
-   : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(X)))
+   : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(Uint64)(X)))
 #define ERTS_UINT64_HEAP_SIZE(X)				\
   (IS_USMALL(0, (X)) ? 0 : ERTS_UINT64_BIG_HEAP_SIZE__((X)))
 
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 7e0e825a0d..3bf78adce7 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1324,9 +1324,9 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
 		goto badarg;
 	    }
 	    if (len < 0) {
-		Sint lentmp = -len;
+		Uint lentmp = -(Uint)len;
 		/* overflow */
-		if (lentmp == len || lentmp < 0 || -lentmp != len) {
+		if ((Sint)lentmp < 0) {
 		    goto badarg;
 		}
 		len = lentmp;
@@ -1555,9 +1555,9 @@ BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen)
 	goto badarg;
     }
     if (len < 0) {
-	Sint lentmp = -len;
+	Uint lentmp = -(Uint)len;
 	/* overflow */
-	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	if ((Sint)lentmp < 0) {
 	    goto badarg;
 	}
 	len = lentmp;
@@ -1644,9 +1644,9 @@ BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is
 	goto badarg;
     }
     if (len < 0) {
-	Sint lentmp = -len;
+	Uint lentmp = -(Uint)len;
 	/* overflow */
-	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	if ((Sint)lentmp < 0) {
 	    goto badarg;
 	}
 	len = lentmp;
@@ -2213,9 +2213,9 @@ static BIF_RETTYPE binary_bin_to_list_common(Process *p,
 	goto badarg;
     }
     if (len < 0) {
-	Sint lentmp = -len;
+	Uint lentmp = -(Uint)len;
 	/* overflow */
-	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	if ((Sint)lentmp < 0) {
 	    goto badarg;
 	}
 	len = lentmp;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 5e6d812242..88c4006934 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2066,8 +2066,10 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
     system_cleanup(flush_async);
 
     save_statistics();
-
-    an = abs(n);
+    if (n < 0)
+        an = -(unsigned int)n;
+    else
+        an = n;
 
     if (erts_mtrace_enabled)
 	erts_mtrace_exit((Uint32) an);
-- 
cgit v1.2.3


From b6a5919cf78028c2778985e8ec6eccc467bb6039 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 14 Aug 2014 16:04:56 +0200
Subject: Verify run-queue asserts

---
 erts/emulator/beam/erl_port_task.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 11027dda7a..738f7d318b 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -76,6 +76,13 @@ do {					\
 #define  DTRACE_DRIVER(PROBE_NAME, PP) do {} while(0)
 #endif
 
+#define ERTS_SMP_LC_VERIFY_RQ(RQ, PP)					\
+    do {								\
+	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));		\
+	ERTS_SMP_LC_ASSERT((RQ) == ((ErtsRunQueue *)			\
+				    erts_smp_atomic_read_nob(&(PP)->run_queue))); \
+    } while (0)
+
 erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
 
 struct ErtsPortTaskQueue_ {
@@ -786,7 +793,9 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	erts_smp_port_lock(pp);
 	erts_smp_runq_lock(runq);
     }
-    
+
+    ERTS_SMP_LC_VERIFY_RQ(runq, pp);
+
     if (erts_sched_stat.enabled) {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
 	Uint old = ERTS_PORT_SCHED_ID(pp, esdp->no);
@@ -828,6 +837,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	case ERTS_PORT_TASK_FREE: /* May be pushed in q at any time */
 	    reds += ERTS_PORT_REDS_FREE;
 	    erts_smp_runq_lock(runq);
+	    ERTS_SMP_LC_VERIFY_RQ(runq, pp);
 
 	    erts_unblock_fpe(fpe_was_unmasked);
 	    ASSERT(pp->status & ERTS_PORT_SFLG_FREE_SCHEDULED);
@@ -906,6 +916,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	port_task_free(ptp);
 
 	erts_smp_runq_lock(runq);
+	ERTS_SMP_LC_VERIFY_RQ(runq, pp);
 
 	ptp = pop_task(ptqp);
     }
@@ -942,7 +953,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 #ifdef ERTS_SMP
 	xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
 	ERTS_SMP_LC_ASSERT(runq != xrunq);
-	ERTS_SMP_LC_ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
+	ERTS_SMP_LC_VERIFY_RQ(runq, pp);
 	if (!xrunq) {
 #endif
 	    enqueue_port(runq, pp);
@@ -1052,6 +1063,7 @@ handle_remaining_tasks(ErtsRunQueue *runq, Port *pp)
 	    port_task_free(ptp);
 
 	    erts_smp_runq_lock(runq);
+	    ERTS_SMP_LC_VERIFY_RQ(runq, pp);
 	    ptp = pop_task(ptqps[i]);
 	}
     }
-- 
cgit v1.2.3


From 631474e50fd84fd526dd1a5db4753b3d27ebb8e7 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Fri, 27 Jun 2014 14:41:12 +0200
Subject: Return pointer to value in erts_maps_get()

While at it, implement erts_maps_get_rel() for halfword support.
---
 erts/emulator/beam/erl_map.c | 91 +++++++++++++++++++-------------------------
 erts/emulator/beam/erl_map.h | 12 +++++-
 erts/emulator/beam/erl_nif.c |  8 +++-
 3 files changed, 57 insertions(+), 54 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 5e740aacdd..b2a16eb5ed 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -113,36 +113,55 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
  * return value if key *matches* a key in the map
  */
 
-int erts_maps_find(Eterm key, Eterm map, Eterm *value) {
-
-    Eterm *ks,*vs;
+const Eterm *
+#if HALFWORD_HEAP
+erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base)
+#else
+erts_maps_get(Eterm key, Eterm map)
+#endif
+{
+    Eterm *ks, *vs;
     map_t *mp;
-    Uint n,i;
+    Uint n, i;
 
-    mp  = (map_t*)map_val(map);
+    mp  = (map_t *)map_val_rel(map, map_base);
     n   = map_get_size(mp);
-    ks  = map_get_keys(mp);
+
+    if (n == 0) {
+        return NULL;
+    }
+
+    ks  = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1;
     vs  = map_get_values(mp);
 
-    for( i = 0; i < n; i++) {
-	if (EQ(ks[i], key)) {
-	    *value = vs[i];
-	    return 1;
-	}
+    if (is_immed(key)) {
+        for (i = 0; i < n; i++) {
+            if (ks[i] == key) {
+                return &vs[i];
+            }
+        }
+    }
+
+    for (i = 0; i < n; i++) {
+        if (eq_rel(ks[i], NULL, key, map_base)) {
+            return &vs[i];
+        }
     }
-    return 0;
+    return NULL;
 }
 
 BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
-	Eterm *hp, value,res;
+        Eterm *hp, res;
+        const Eterm *value;
 
-	if (erts_maps_find(BIF_ARG_1, BIF_ARG_2, &value)) {
+        value = erts_maps_get(BIF_ARG_1, BIF_ARG_2);
+	if (value) {
 	    hp    = HAlloc(BIF_P, 3);
 	    res   = make_tuple(hp);
 	    *hp++ = make_arityval(2);
 	    *hp++ = am_ok;
-	    *hp++ = value;
+            *hp++ = *value;
 	    BIF_RET(res);
 	}
 
@@ -150,52 +169,22 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
+
 /* maps:get/2
  * return value if key *matches* a key in the map
  * exception bad_key if none matches
  */
 
-
-int erts_maps_get(Eterm key, Eterm map, Eterm *value) {
-    Eterm *ks,*vs;
-    map_t *mp;
-    Uint n,i;
-
-    mp  = (map_t*)map_val(map);
-    n   = map_get_size(mp);
-
-    if (n == 0)
-	return 0;
-
-    ks  = map_get_keys(mp);
-    vs  = map_get_values(mp);
-
-    if (is_immed(key)) {
-	for( i = 0; i < n; i++) {
-	    if (ks[i] == key) {
-		*value = vs[i];
-		return 1;
-	    }
-	}
-    }
-
-    for( i = 0; i < n; i++) {
-	if (EQ(ks[i], key)) {
-	    *value = vs[i];
-	    return 1;
-	}
-    }
-    return 0;
-}
-
 BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
 	Eterm *hp;
-	Eterm value, error;
+        Eterm error;
+        const Eterm *value;
 	char *s_error;
 
-	if (erts_maps_get(BIF_ARG_1, BIF_ARG_2, &value)) {
-	    BIF_RET(value);
+        value = erts_maps_get(BIF_ARG_1, BIF_ARG_2);
+        if (value) {
+            BIF_RET(*value);
 	}
 
 	s_error = "bad_key";
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index cfacb2ec28..2e02ca4677 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -64,9 +64,17 @@ typedef struct map_s {
 
 Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
 int   erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
-int   erts_maps_find(Eterm key, Eterm map, Eterm *value);
-int   erts_maps_get(Eterm key, Eterm map, Eterm *value);
 int   erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
 int   erts_validate_and_sort_map(map_t* map);
+
+#if HALFWORD_HEAP
+const Eterm *
+erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base);
+#  define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL)
+#else
+const Eterm *
+erts_maps_get(Eterm key, Eterm map);
+#  define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B)
 #endif
 
+#endif
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ff551ea3af..dd88f59908 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1733,10 +1733,16 @@ int enif_get_map_value(ErlNifEnv* env,
 		       Eterm key,
 		       Eterm *value)
 {
+    Eterm *ret;
     if (is_not_map(map)) {
 	return 0;
     }
-    return erts_maps_get(key, map, value);
+    ret = erts_maps_get(key, map);
+    if (ret) {
+        *value = *ret;
+        return 1;
+    }
+    return 0;
 }
 
 int enif_make_map_update(ErlNifEnv* env,
-- 
cgit v1.2.3


From a46f30f38f0e757e9aa335e1452879db608d8489 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Fri, 27 Jun 2014 14:47:26 +0200
Subject: Properly support maps in match_specs

---
 erts/emulator/beam/erl_db_util.c | 316 +++++++++++++++++++++++++++++++--------
 1 file changed, 253 insertions(+), 63 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 3927615e04..b9fd3b208e 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -198,11 +198,6 @@ set_match_trace(Process *tracee_p, Eterm fail_term, Eterm tracer,
     return ret;
 }
 
-
-/* Type checking... */
-
-#define BOXED_IS_TUPLE(Boxed) is_arity_value(*boxed_val((Boxed)))
-
 /*
 **
 ** Types and enum's (compiled matches)
@@ -218,6 +213,8 @@ typedef enum {
     matchTuple,
     matchPushT,
     matchPushL,
+    matchPushM,
+    matchPushK,
     matchPop,
     matchBind,
     matchCmp,
@@ -227,11 +224,13 @@ typedef enum {
     matchEqRef,
     matchEq,
     matchList,
+    matchMap,
     matchSkip,
     matchPushC,
     matchConsA, /* Car is below Cdr */
     matchConsB, /* Cdr is below Car (unusual) */
     matchMkTuple,
+    matchMkMap,
     matchCall0,
     matchCall1,
     matchCall2,
@@ -856,6 +855,13 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info);
 static Uint my_size_object(Eterm t);
 static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap);
 
+/* Guard subroutines */
+static void
+dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
+                        int textpos, Eterm *p, Uint nelems);
+static DMCRet
+dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
+          Eterm *p, Uint nelems, int *constant);
 /* Guard compilation */
 static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
 			     Eterm t);
@@ -869,6 +875,9 @@ static DMCRet dmc_tuple(DMCContext *context,
 		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant);
+static DMCRet
+dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
+        Eterm t, int *constant);
 static DMCRet dmc_variable(DMCContext *context,
 			   DMCHeap *heap,
 			   DMC_STACK_TYPE(UWord) *text,
@@ -888,12 +897,14 @@ static DMCRet compile_guard_expr(DMCContext *context,
 				    DMCHeap *heap,
 				    DMC_STACK_TYPE(UWord) *text,
 				    Eterm t);
-/* match expression subroutine */
+/* match expression subroutines */
 static DMCRet dmc_one_term(DMCContext *context, 
 			   DMCHeap *heap,
 			   DMC_STACK_TYPE(Eterm) *stack,
 			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm c);
+static Eterm
+dmc_private_copy(DMCContext *context, Eterm c);
 
 
 #ifdef DMC_DEBUG
@@ -1364,7 +1375,51 @@ restart:
 	for (;;) {
 	    switch (t & _TAG_PRIMARY_MASK) {
 	    case TAG_PRIMARY_BOXED:
-		if (!BOXED_IS_TUPLE(t)) {
+                if (is_map(t)) {
+                    num_iters = map_get_size(map_val(t));
+                    if (!structure_checked) {
+                        DMC_PUSH(text, matchMap);
+                        DMC_PUSH(text, num_iters);
+                    }
+                    structure_checked = 0;
+                    for (i = 0; i < num_iters; ++i) {
+                        Eterm key = map_get_keys(map_val(t))[i];
+                        if (db_is_variable(key) >= 0) {
+                            if (context.err_info) {
+                                add_dmc_err(context.err_info,
+                                            "Variable found in map key.",
+                                            -1, 0UL, dmcError);
+                            }
+                            goto error;
+                        } else if (key == am_Underscore) {
+                            if (context.err_info) {
+                                add_dmc_err(context.err_info,
+                                            "Underscore found in map key.",
+                                            -1, 0UL, dmcError);
+                            }
+                            goto error;
+                        }
+                        DMC_PUSH(text, matchPushK);
+                        ++(context.stack_used);
+                        DMC_PUSH(text, dmc_private_copy(&context, key));
+                    }
+                    if (context.stack_used > context.stack_need) {
+                        context.stack_need = context.stack_used;
+                    }
+                    for (i = num_iters; i--; ) {
+                        Eterm value = map_get_values(map_val(t))[i];
+                        DMC_PUSH(text, matchPop);
+                        --(context.stack_used);
+                        res = dmc_one_term(&context, &heap, &stack, &text,
+                                           value);
+                        ASSERT(res != retFail);
+                        if (res == retRestart) {
+                            goto restart;
+                        }
+                    }
+                    break;
+                }
+		if (!is_tuple(t)) {
 		    goto simple_term;
 		}
 		num_iters = arityval(*tuple_val(t));
@@ -1715,10 +1770,8 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
 		    Uint32 *return_flags)
 {
     MatchProg *prog = Binary2MatchProg(bprog);
-    Eterm *ep;
-    Eterm *tp;
+    const Eterm *ep, *tp, **sp;
     Eterm t;
-    Eterm **sp;
     Eterm *esp;
     MatchVariable* variables;
     BeamInstr *cp;
@@ -1808,7 +1861,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
 restart:
     ep = &term;
     esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset);
-    sp = (Eterm **) esp;
+    sp = (const Eterm **)esp;
     ret = am_true;
     do_catch = 0;
     fail_label = -1;
@@ -1887,6 +1940,34 @@ restart:
 	    *sp++ = list_val_rel(*ep,base);
 	    ++ep;
 	    break;
+        case matchMap:
+            if (!is_map_rel(*ep, base)) {
+                FAIL();
+            }
+            n = *pc++;
+            if (map_get_size(map_val_rel(*ep, base)) < n) {
+                FAIL();
+            }
+            ep = map_val_rel(*ep, base);
+            break;
+        case matchPushM:
+            if (!is_map_rel(*ep, base)) {
+                FAIL();
+            }
+            n = *pc++;
+            if (map_get_size(map_val_rel(*ep, base)) < n) {
+                FAIL();
+            }
+            *sp++ = map_val_rel(*ep++, base);
+            break;
+        case matchPushK:
+            t = (Eterm) *pc++;
+            tp = erts_maps_get_rel(t, make_map_rel(ep, base), base);
+            if (!tp) {
+                FAIL();
+            }
+            *sp++ = tp;
+            break;
 	case matchPop:
 	    ep = *(--sp);
 	    break;
@@ -1987,6 +2068,23 @@ restart:
 	    }
 	    *esp++ = t;
 	    break;
+        case matchMkMap:
+            n = *pc++;
+            ehp = HAllocX(build_proc, 1 + MAP_HEADER_SIZE + n, HEAP_XTRA);
+            t = *ehp++ = *--esp;
+            {
+                map_t *m = (map_t *)ehp;
+                m->thing_word = MAP_HEADER;
+                m->size = n;
+                m->keys = t;
+            }
+            t = make_map(ehp);
+            ehp += MAP_HEADER_SIZE;
+            while (n--) {
+                *ehp++ = *--esp;
+            }
+            *esp++ = t;
+            break;
 	case matchCall0:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
 	    t = (*bif)(build_proc, bif_args);
@@ -3168,7 +3266,7 @@ int db_has_variable(Eterm obj)
 	return(db_has_variable(obj));  /* Non wellformed list or [] */
     }
     case TAG_PRIMARY_BOXED: 
-	if (!BOXED_IS_TUPLE(obj)) {
+	if (!is_tuple(obj)) {
 	    return 0;
 	} else {
 	    Eterm *tuple = tuple_val(obj);
@@ -3243,7 +3341,6 @@ static DMCRet dmc_one_term(DMCContext *context,
 {
     Sint n;
     Eterm *hp;
-    ErlHeapFragment *tmp_mb;
     Uint sz, sz2, sz3;
     Uint i, j;
 
@@ -3334,6 +3431,13 @@ static DMCRet dmc_one_term(DMCContext *context,
 	    DMC_PUSH(*text, n);
 	    DMC_PUSH(*stack, c);
 	    break;
+        case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):
+            n = map_get_size(map_val(c));
+            DMC_PUSH(*text, matchPushM);
+            ++(context->stack_used);
+            DMC_PUSH(*text, n);
+            DMC_PUSH(*stack, c);
+            break;
 	case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
 	{
 	    Eterm* ref_val = internal_ref_val(c);
@@ -3415,16 +3519,8 @@ static DMCRet dmc_one_term(DMCContext *context,
 #endif
 	    break;
 	default: /* BINARY, FUN, VECTOR, or EXTERNAL */
-	    /*
-	    ** Make a private copy...
-	    */
-	    n = size_object(c);
-	    tmp_mb = new_message_buffer(n);
-	    hp = tmp_mb->mem;
 	    DMC_PUSH(*text, matchEqBin);
-	    DMC_PUSH(*text, copy_struct(c, n, &hp, &(tmp_mb->off_heap)));
-	    tmp_mb->next = context->save;
-	    context->save = tmp_mb;
+            DMC_PUSH(*text, dmc_private_copy(context, c));
 	    break;
 	}
 	break;
@@ -3436,6 +3532,22 @@ static DMCRet dmc_one_term(DMCContext *context,
     return retOk;
 }
 
+/*
+** Make a private copy of a term in a context.
+*/
+
+static Eterm
+dmc_private_copy(DMCContext *context, Eterm c)
+{
+    Uint n = size_object(c);
+    ErlHeapFragment *tmp_mb = new_message_buffer(n);
+    Eterm *hp = tmp_mb->mem;
+    Eterm copy = copy_struct(c, n, &hp, &(tmp_mb->off_heap));
+    tmp_mb->next = context->save;
+    context->save = tmp_mb;
+    return copy;
+}
+
 /*
 ** Match guard compilation
 */
@@ -3527,57 +3639,78 @@ static DMCRet dmc_list(DMCContext *context,
     return retOk;
 }
 
-static DMCRet dmc_tuple(DMCContext *context,
-		       DMCHeap *heap,
-		       DMC_STACK_TYPE(UWord) *text,
-		       Eterm t,
-		       int *constant)
+static void
+dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
+                        int textpos, Eterm *p, Uint nelems)
 {
     DMC_STACK_TYPE(UWord) instr_save;
+    Uint i;
+
+    DMC_INIT_STACK(instr_save);
+    while (DMC_STACK_NUM(*text) > textpos) {
+        DMC_PUSH(instr_save, DMC_POP(*text));
+    }
+    for (i = nelems; i--;) {
+        do_emit_constant(context, text, p[i]);
+    }
+    while(!DMC_EMPTY(instr_save)) {
+        DMC_PUSH(*text, DMC_POP(instr_save));
+    }
+    DMC_FREE(instr_save);
+}
+
+static DMCRet
+dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
+          Eterm *p, Uint nelems, int *constant)
+{
     int all_constant = 1;
     int textpos = DMC_STACK_NUM(*text);
-    Eterm *p = tuple_val(t);
-    Uint nelems = arityval(*p);
     Uint i;
-    int c;
-    DMCRet ret;
 
     /*
-    ** We remember where we started to layout code, 
+    ** We remember where we started to layout code,
     ** assume all is constant and back up and restart if not so.
-    ** The tuple should be laid out with the last element first,
-    ** so we can memcpy the tuple to the eheap.
+    ** The array should be laid out with the last element first,
+    ** so we can memcpy it to the eheap.
     */
-    for (i = nelems; i > 0; --i) {
-	if ((ret = dmc_expr(context, heap, text, p[i], &c)) != retOk)
-	    return ret;
-	if (!c && all_constant) {
-	    all_constant = 0;
-	    if (i < nelems) {
-		Uint j;
+    for (i = nelems; i--;) {
+        DMCRet ret;
+        int c;
 
-		/*
-		 * Oops, we need to relayout the constants.
-		 * Save the already laid out instructions.
-		 */
-		DMC_INIT_STACK(instr_save);
-		while (DMC_STACK_NUM(*text) > textpos) 
-		    DMC_PUSH(instr_save, DMC_POP(*text));
-		for (j = nelems; j > i; --j)
-		    do_emit_constant(context, text, p[j]);
-		while(!DMC_EMPTY(instr_save))
-		    DMC_PUSH(*text, DMC_POP(instr_save));
-		DMC_FREE(instr_save);
-	    }
-	} else if (c && !all_constant) {
-	    /* push a constant */
-	    do_emit_constant(context, text, p[i]);
-	}
+        ret = dmc_expr(context, heap, text, p[i], &c);
+        if (ret != retOk) {
+            return ret;
+        }
+        if (!c && all_constant) {
+            all_constant = 0;
+            if (i < nelems - 1) {
+                dmc_rearrange_constants(context, text, textpos,
+                                        p + i + 1, nelems - i - 1);
+            }
+        } else if (c && !all_constant) {
+            do_emit_constant(context, text, p[i]);
+        }
+    }
+    *constant = all_constant;
+    return retOk;
+}
+
+static DMCRet
+dmc_tuple(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
+          Eterm t, int *constant)
+{
+    int all_constant;
+    Eterm *p = tuple_val(t);
+    Uint nelems = arityval(*p);
+    DMCRet ret;
+
+    ret = dmc_array(context, heap, text, p + 1, nelems, &all_constant);
+    if (ret != retOk) {
+        return ret;
     }
-    
     if (all_constant) {
-	*constant = 1;
-	return retOk;
+        *constant = 1;
+        return retOk;
     }
     DMC_PUSH(*text, matchMkTuple);
     DMC_PUSH(*text, nelems);
@@ -3586,6 +3719,36 @@ static DMCRet dmc_tuple(DMCContext *context,
     return retOk;
 }
 
+static DMCRet
+dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
+        Eterm t, int *constant)
+{
+    map_t *m = (map_t *)map_val(t);
+    Eterm *values = map_get_values(m);
+    int nelems = map_get_size(m);
+    int constant_values;
+    DMCRet ret;
+
+    ret = dmc_array(context, heap, text, values, nelems, &constant_values);
+    if (ret != retOk) {
+        return ret;
+    }
+    if (constant_values) {
+        *constant = 1;
+        return retOk;
+    }
+    DMC_PUSH(*text, matchPushC);
+    DMC_PUSH(*text, dmc_private_copy(context, m->keys));
+    if (++context->stack_used > context->stack_need) {
+        context->stack_need = context->stack_used;
+    }
+    DMC_PUSH(*text, matchMkMap);
+    DMC_PUSH(*text, nelems);
+    context->stack_used -= nelems;
+    *constant = 0;
+    return retOk;
+}
+
 static DMCRet dmc_whole_expression(DMCContext *context,
 				   DMCHeap *heap,
 				   DMC_STACK_TYPE(UWord) *text,
@@ -4580,7 +4743,10 @@ static DMCRet dmc_expr(DMCContext *context,
 	    return ret;
 	break;
     case TAG_PRIMARY_BOXED:
-	if (!BOXED_IS_TUPLE(t)) {
+        if (is_map(t)) {
+            return dmc_map(context, heap, text, t, constant);
+        }
+	if (!is_tuple(t)) {
 	    goto simple_term;
 	}
 	p = tuple_val(t);
@@ -4855,7 +5021,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap)
 	*hp += 2;
 	break;
     case TAG_PRIMARY_BOXED:
-	if (BOXED_IS_TUPLE(t)) {
+	if (is_tuple(t)) {
 	    if (arityval(*tuple_val(t)) == 1 && 
 		is_tuple(a = tuple_val(t)[1])) {
 		Uint i,n;
@@ -5126,6 +5292,12 @@ void db_match_dis(Binary *bp)
 	    ++t;
 	    erts_printf("Tuple\t%beu\n", n);
 	    break;
+        case matchMap:
+            ++t;
+            n = *t;
+            ++t;
+            erts_printf("Map\t%beu\n", n);
+            break;
 	case matchPushT:
 	    ++t;
 	    n = *t;
@@ -5136,6 +5308,18 @@ void db_match_dis(Binary *bp)
 	    ++t;
 	    erts_printf("PushL\n");
 	    break;
+        case matchPushM:
+            ++t;
+            n = *t;
+            ++t;
+            erts_printf("PushM\t%beu\n", n);
+            break;
+        case matchPushK:
+            ++t;
+            p = (Eterm) *t;
+            ++t;
+            erts_printf("PushK\t%p (%T)\n", t, p);
+            break;
 	case matchPop:
 	    ++t;
 	    erts_printf("Pop\n");
@@ -5252,6 +5436,12 @@ void db_match_dis(Binary *bp)
 	    ++t;
 	    erts_printf("MkTuple\t%beu\n", n);
 	    break;
+        case matchMkMap:
+            ++t;
+            n = *t;
+            ++t;
+            erts_printf("MkMapA\t%beu\n", n);
+            break;
 	case matchOr:
 	    ++t;
 	    n = *t;
-- 
cgit v1.2.3


From 05c183c014c658810fc5a3391429eba9db14ac8f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 18 Aug 2014 13:03:35 +0200
Subject: Ensure "runnable proc" trace messages are not sent out of order

---
 erts/emulator/beam/erl_message.c | 16 ++++++-
 erts/emulator/beam/erl_process.c | 92 ++++++++++++++++++++++++++++++----------
 erts/emulator/beam/erl_process.h |  8 ++--
 erts/emulator/beam/io.c          | 15 ++++++-
 4 files changed, 101 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 59a677a12c..8870fac7d9 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -415,7 +415,13 @@ erts_queue_dist_message(Process *rcvr,
 	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 
-	erts_proc_notify_new_message(rcvr);
+	erts_proc_notify_new_message(rcvr,
+#ifdef ERTS_SMP
+				     *rcvr_locks
+#else
+				     0
+#endif
+	    );
     }
 }
 
@@ -542,7 +548,13 @@ queue_message(Process *c_p,
     if (locked_msgq)
 	erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
 
-    erts_proc_notify_new_message(receiver);
+    erts_proc_notify_new_message(receiver,
+#ifdef ERTS_SMP
+				 *receiver_locks
+#else
+				 0
+#endif
+	);
 
 #ifndef ERTS_SMP
     ERTS_HOLE_CHECK(receiver);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1606ad119d..1571a78247 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5877,6 +5877,9 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
     case ERTS_ENQUEUE_NOT:
 	if (erts_system_profile_flags.runnable_procs) {
 
+	    /* Status lock prevents out of order "runnable proc" trace msgs */
+	    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+
 	    if (!(a & ERTS_PSFLG_ACTIVE_SYS)
 		&& (!(a & ERTS_PSFLG_ACTIVE)
 		    || (a & ERTS_PSFLG_SUSPENDED))) {
@@ -5990,7 +5993,8 @@ change_proc_schedule_state(Process *p,
 			   erts_aint32_t clear_state_flags,
 			   erts_aint32_t set_state_flags,
 			   erts_aint32_t *statep,
-			   erts_aint32_t *enq_prio_p)
+			   erts_aint32_t *enq_prio_p,
+			   ErtsProcLocks locks)
 {
     /*
      * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS and
@@ -5999,6 +6003,11 @@ change_proc_schedule_state(Process *p,
      */
     erts_aint32_t a = *statep, n;
     int enqueue; /* < 0 -> use proxy */
+    unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs;
+    unsigned int lock_status = (prof_runnable_procs
+				&& !(locks & ERTS_PROC_LOCK_STATUS));
+
+    ERTS_SMP_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p));
 
     ASSERT(!(a & ERTS_PSFLG_PROXY));
     ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING
@@ -6008,6 +6017,9 @@ change_proc_schedule_state(Process *p,
 			       | ERTS_PSFLG_RUNNING_SYS
 			       | ERTS_PSFLG_ACTIVE_SYS)) == 0);
 
+    if (lock_status)
+	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+
     while (1) {
 	erts_aint32_t e;
 	n = e = a;
@@ -6043,7 +6055,9 @@ change_proc_schedule_state(Process *p,
 	    break;
     }
 
-    if (erts_system_profile_flags.runnable_procs) {
+    if (prof_runnable_procs) {
+
+	/* Status lock prevents out of order "runnable proc" trace msgs */
 
 	if (((n & (ERTS_PSFLG_SUSPENDED
 		   | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE)
@@ -6056,15 +6070,18 @@ change_proc_schedule_state(Process *p,
 	    profile_runnable_proc(p, am_active);
 	}
 
+	if (lock_status)
+	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
 
+
     *statep = a;
 
     return enqueue;
 }
 
 static ERTS_INLINE void
-schedule_process(Process *p, erts_aint32_t in_state)
+schedule_process(Process *p, erts_aint32_t in_state, ErtsProcLocks locks)
 {
     erts_aint32_t enq_prio  = -1;
     erts_aint32_t state = in_state;
@@ -6072,7 +6089,8 @@ schedule_process(Process *p, erts_aint32_t in_state)
 					     0,
 					     ERTS_PSFLG_ACTIVE,
 					     &state,
-					     &enq_prio);
+					     &enq_prio,
+					     locks);
     if (enqueue != ERTS_ENQUEUE_NOT)
 	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
 		 state,
@@ -6080,16 +6098,27 @@ schedule_process(Process *p, erts_aint32_t in_state)
 }
 
 void
-erts_schedule_process(Process *p, erts_aint32_t state)
+erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks)
 {
-    schedule_process(p, state);
+    schedule_process(p, state, locks);
 }
 
 static void
 schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 {
+    /*
+     * Expects status lock to be locked when called, and
+     * returns with status lock unlocked...
+     */
     erts_aint32_t a = state, n, enq_prio = -1;
     int enqueue; /* < 0 -> use proxy */
+    unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs;
+
+    /* Status lock prevents out of order "runnable proc" trace msgs */
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+
+    if (!prof_runnable_procs)
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
     ASSERT(!(state & ERTS_PSFLG_PROXY));
 
@@ -6098,7 +6127,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 	n = e = a;
 
 	if (a & ERTS_PSFLG_FREE)
-	    return; /* We don't want to schedule free processes... */
+	    goto cleanup; /* We don't want to schedule free processes... */
 
 	enqueue = ERTS_ENQUEUE_NOT;
 	n |= ERTS_PSFLG_ACTIVE_SYS;
@@ -6111,7 +6140,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 	    goto cleanup;
     }
 
-    if (erts_system_profile_flags.runnable_procs) {
+    if (prof_runnable_procs) {
 
 	if (!(a & (ERTS_PSFLG_ACTIVE_SYS
 		   | ERTS_PSFLG_RUNNING
@@ -6121,6 +6150,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 	    profile_runnable_proc(p, am_active);
 	}
 
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	prof_runnable_procs = 0;
     }
 
     if (enqueue != ERTS_ENQUEUE_NOT) {
@@ -6135,8 +6166,14 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
     }
 
 cleanup:
+
+    if (prof_runnable_procs)
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+
     if (proxy)
 	free_proxy_proc(proxy);
+
+    ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
 }
 
 static ERTS_INLINE int
@@ -6203,7 +6240,7 @@ suspend_process(Process *c_p, Process *p)
 }
 
 static ERTS_INLINE void
-resume_process(Process *p)
+resume_process(Process *p, ErtsProcLocks locks)
 {
     erts_aint32_t state, enq_prio = -1;
     int enqueue;
@@ -6220,7 +6257,8 @@ resume_process(Process *p)
 					 ERTS_PSFLG_SUSPENDED,
 					 0,
 					 &state,
-					 &enq_prio);
+					 &enq_prio,
+					 locks);
     if (enqueue)
 	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
 		 state,
@@ -8036,7 +8074,8 @@ handle_pend_sync_suspend(Process *suspendee,
 	}
 	/* suspender is suspended waiting for suspendee to suspend;
 	   resume suspender */
-	resume_process(suspender);
+	ASSERT(suspendee != suspender);
+	resume_process(suspender, ERTS_PROC_LOCK_STATUS);
 	erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
     }
 }
@@ -8071,7 +8110,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	ASSERT(c_p->flags & F_P2PNR_RESCHED);
 	c_p->flags &= ~F_P2PNR_RESCHED;
 	if (!suspend && rp)
-	    resume_process(rp);
+	    resume_process(rp, rp_locks);
     }
     else {
 
@@ -8229,7 +8268,8 @@ handle_pend_bif_sync_suspend(Process *suspendee,
 	}
 	/* suspender is suspended waiting for suspendee to suspend;
 	   resume suspender */
-	resume_process(suspender);
+	ASSERT(suspender != suspendee);
+	resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
 	erts_smp_proc_unlock(suspender,
 			     ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
     }
@@ -8589,7 +8629,8 @@ resume_process_1(BIF_ALIST_1)
 
 	ASSERT(ERTS_PSFLG_SUSPENDED
 	       & erts_smp_atomic32_read_nob(&suspendee->state));
-	resume_process(suspendee);
+	ASSERT(BIF_P != suspendee);
+	resume_process(suspendee, ERTS_PROC_LOCK_STATUS);
 
 	erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
     }
@@ -8719,7 +8760,7 @@ erts_resume(Process* process, ErtsProcLocks process_locks)
     ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process));
     if (!(process_locks & ERTS_PROC_LOCK_STATUS))
 	erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS);
-    resume_process(process);
+    resume_process(process, process_locks|ERTS_PROC_LOCK_STATUS);
     if (!(process_locks & ERTS_PROC_LOCK_STATUS))
 	erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS);
 }
@@ -8738,7 +8779,7 @@ erts_resume_processes(ErtsProcList *list)
 	proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS);
 	if (proc) {
 	    if (erts_proclist_same(plp, proc)) {
-		resume_process(proc);
+		resume_process(proc, ERTS_PROC_LOCK_STATUS);
 		nresumed++;
 	    }
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS);
@@ -9974,8 +10015,10 @@ erts_internal_request_system_task_3(BIF_ALIST_3)
 	rp_state = n;
     }
 
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-
+    /*
+     * schedule_process_sys_task() unlocks status
+     * lock on process.
+     */
     schedule_process_sys_task(rp, rp_state, NULL);
 
     if (free_stqs)
@@ -10720,7 +10763,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
      * Schedule process for execution.
      */
 
-    schedule_process(p, state);
+    schedule_process(p, state, 0);
 
     VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id));
 
@@ -11041,7 +11084,8 @@ set_proc_exiting(Process *p,
 					 ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT,
 					 ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
 					 &state,
-					 &enq_prio);
+					 &enq_prio,
+					 ERTS_PROC_LOCKS_ALL);
 
     p->fvalue = reason;
     if (bp)
@@ -11082,7 +11126,8 @@ set_proc_self_exiting(Process *c_p)
 				   ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT,
 				   ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
 				   &state,
-				   &enq_prio);
+				   &enq_prio,
+				   ERTS_PROC_LOCKS_ALL);
 
     ASSERT(!enqueue);
     return state;
@@ -11727,8 +11772,9 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p)
     Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN,
 				       smon->pid, ERTS_PROC_LOCK_STATUS);
     if (suspendee) {
+	ASSERT(suspendee != vc_p);
 	if (smon->active)
-	    resume_process(suspendee);
+	    resume_process(suspendee, ERTS_PROC_LOCK_STATUS);
 	erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
     }
     erts_destroy_suspend_monitor(smon);
@@ -12061,7 +12107,7 @@ timeout_proc(Process* p)
 
     state = erts_smp_atomic32_read_acqb(&p->state);
     if (!(state & ERTS_PSFLG_ACTIVE))
-	schedule_process(p, state);
+	schedule_process(p, state, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 }
 
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index ed6dadbffa..084399726c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1704,17 +1704,17 @@ ErtsSchedulerData *erts_get_scheduler_data(void)
 #endif
 #endif
 
-void erts_schedule_process(Process *, erts_aint32_t);
+void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);
 
-ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p);
+ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 ERTS_GLB_INLINE void
-erts_proc_notify_new_message(Process *p)
+erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
 {
     /* No barrier needed, due to msg lock */
     erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
     if (!(state & ERTS_PSFLG_ACTIVE))
-	erts_schedule_process(p, state);
+	erts_schedule_process(p, state, locks);
 }
 #endif
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 0f86d8e41d..50e08de8a4 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1253,12 +1253,18 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	goto locked_fail;
     }
 
-
     if (!c_p)
 	reds_left_in = CONTEXT_REDS/10;
     else {
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
 	    trace_virtual_sched(c_p, am_out);
+	/*
+	 * No status lock held while sending runnable
+	 * proc trace messages. It is however not needed
+	 * in this case, since only this thread can send
+	 * such messages for this process until the process
+	 * has been scheduled out.
+	 */
 	if (erts_system_profile_flags.runnable_procs
 	    && erts_system_profile_flags.exclusive)
 	    profile_runnable_proc(c_p, am_inactive);
@@ -1319,6 +1325,13 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
 	    trace_virtual_sched(c_p, am_in);
+	/*
+	 * No status lock held while sending runnable
+	 * proc trace messages. It is however not needed
+	 * in this case, since only this thread can send
+	 * such messages for this process until the process
+	 * has been scheduled out.
+	 */
 	if (erts_system_profile_flags.runnable_procs
 	    && erts_system_profile_flags.exclusive)
 	    profile_runnable_proc(c_p, am_active);
-- 
cgit v1.2.3


From e8c43908d0634763c8347883ff3e69635263e039 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 18 Aug 2014 15:11:39 +0200
Subject: Ensure "runnable port" trace messages are not sent out of order

---
 erts/emulator/beam/erl_lock_check.c |  2 +-
 erts/emulator/beam/erl_port_task.c  | 48 +++++++++++++++++++--------
 erts/emulator/beam/erl_port_task.h  |  2 ++
 erts/emulator/beam/io.c             | 65 ++++++++++++++++++++++++++++---------
 4 files changed, 86 insertions(+), 31 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index c665aa51a2..b105ece6f1 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -139,7 +139,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"drv_tsd",				NULL			},
     {	"async_enq_mtx",			NULL			},
 #ifdef ERTS_SMP
-    {	"sys_msg_q", 				NULL			},
     {	"atom_tab",				NULL			},
     {	"make_ref",				NULL			},
     {	"misc_op_list_pre_alloc_lock",		"address"		},
@@ -148,6 +147,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"btm_pre_alloc_lock",			NULL,			},
     {	"dist_entry_out_queue",			"address"		},
     {	"port_sched_lock",			"port_id"		},
+    {	"sys_msg_q", 				NULL			},
     {   "port_table",                           NULL                    },
 #endif
     {	"mtrace_op",				NULL			},
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 4103d1192a..682f6f8f4b 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -999,6 +999,7 @@ static ERTS_INLINE int
 finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q)
 {
     erts_aint32_t act;
+    unsigned int prof_runnable_ports;
 
     if (!processing_busy_q)
 	pp->sched.taskq.local.first = *execq;
@@ -1015,6 +1016,10 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q)
     if (act & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q)
 	act = check_unset_busy_port_q(pp, act, pp->sched.taskq.bpq);
 
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&pp->sched);
+
     while (1) {
 	erts_aint32_t new, exp;
 
@@ -1026,12 +1031,24 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q)
 
 	act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp);
 
-	ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ));
+	ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ));
+	ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_EXEC_IMM));
 
 	if (exp == act)
 	    break;
     }
 
+    if (prof_runnable_ports | IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) {
+	/* trace port scheduling, out */
+	if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS))
+	    trace_sched_ports(pp, am_out);
+	if (prof_runnable_ports) {
+	    if (!(act & (ERTS_PTS_FLG_EXEC_IMM|ERTS_PTS_FLG_HAVE_TASKS)))
+		profile_runnable_port(pp, am_inactive);
+	    erts_port_task_sched_unlock(&pp->sched);
+	}
+    }
+
     return (act & ERTS_PTS_FLG_HAVE_TASKS) != 0;
 }
 
@@ -1377,6 +1394,7 @@ erts_port_task_schedule(Eterm id,
     Port *pp;
     ErtsPortTask *ptp = NULL;
     erts_aint32_t act, add_flags;
+    unsigned int prof_runnable_ports;
 
     if (pthp && erts_port_task_is_scheduled(pthp)) {
 	ASSERT(0);
@@ -1465,6 +1483,10 @@ erts_port_task_schedule(Eterm id,
     if (ns_pthlp)
 	add_flags |= ERTS_PTS_FLG_HAVE_NS_TASKS;
 
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&pp->sched);
+
     while (1) {
 	erts_aint32_t new, exp;
 
@@ -1489,6 +1511,13 @@ erts_port_task_schedule(Eterm id,
 	    goto done; /* Died after our task insert... */
     }
 
+    if (prof_runnable_ports) {
+	if (!(act & ERTS_PTS_FLG_EXEC_IMM))
+	    profile_runnable_port(pp, am_active);
+	erts_port_task_sched_unlock(&pp->sched);
+	prof_runnable_ports = 0;
+    }
+
     /* Enqueue port on run-queue */
 
     runq = erts_port_runq(pp);
@@ -1510,10 +1539,6 @@ erts_port_task_schedule(Eterm id,
 #endif
 
     enqueue_port(runq, pp);
-	    
-    if (erts_system_profile_flags.runnable_ports) {
-	profile_runnable_port(pp, am_active);
-    }
 
     erts_smp_runq_unlock(runq);
 
@@ -1521,6 +1546,9 @@ erts_port_task_schedule(Eterm id,
 
 done:
 
+    if (prof_runnable_ports)
+	erts_port_task_sched_unlock(&pp->sched);
+
 #ifdef ERTS_SMP
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 	erts_port_dec_refc(pp);
@@ -1777,10 +1805,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
     erts_unblock_fpe(fpe_was_unmasked);
 
-    /* trace port scheduling, out */
-    if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) {
-    	trace_sched_ports(pp, am_out);
-    }
 
     if (io_tasks_executed) {
 	ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
@@ -1803,11 +1827,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
     erts_smp_runq_lock(runq);
  
-    if (!active) {
-	if (erts_system_profile_flags.runnable_ports)
-	    profile_runnable_port(pp, am_inactive);
-    }
-    else {
+    if (active) {
 #ifdef ERTS_SMP
 	ErtsRunQueue *xrunq;
 #endif
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 1d30465ec9..9ef0cfcedc 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -78,6 +78,7 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
 #define ERTS_PTS_FLG_PARALLELISM		(((erts_aint32_t) 1) <<  9)
 #define ERTS_PTS_FLG_FORCE_SCHED		(((erts_aint32_t) 1) << 10)
 #define ERTS_PTS_FLG_EXITING			(((erts_aint32_t) 1) << 11)
+#define ERTS_PTS_FLG_EXEC_IMM			(((erts_aint32_t) 1) << 12)
 
 #define ERTS_PTS_FLGS_BUSY \
     (ERTS_PTS_FLG_BUSY_PORT | ERTS_PTS_FLG_BUSY_PORT_Q)
@@ -87,6 +88,7 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
      | ERTS_PTS_FLG_HAVE_BUSY_TASKS		\
      | ERTS_PTS_FLG_HAVE_TASKS			\
      | ERTS_PTS_FLG_EXEC			\
+     | ERTS_PTS_FLG_EXEC_IMM			\
      | ERTS_PTS_FLG_FORCE_SCHED			\
      | ERTS_PTS_FLG_EXITING)
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 50e08de8a4..ae053fc191 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1218,9 +1218,10 @@ typedef struct {
 static ERTS_INLINE ErtsTryImmDrvCallResult
 try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 {
+    unsigned int prof_runnable_ports;
     ErtsTryImmDrvCallResult res;
     int reds_left_in;
-    erts_aint32_t invalid_state, invalid_sched_flags;
+    erts_aint32_t act, exp, invalid_state, invalid_sched_flags;
     Port *prt = sp->port;
     Process *c_p = sp->c_p;
 
@@ -1247,11 +1248,26 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	goto locked_fail;
     }
 
-    sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
-    if (sp->sched_flags & invalid_sched_flags) {
-	res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
-	goto locked_fail;
-    }
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&prt->sched);
+
+    act = erts_smp_atomic32_read_nob(&prt->sched.flags);
+
+    do {
+	erts_aint32_t new;
+	
+	if (act & invalid_sched_flags) {
+	    res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
+	    sp->sched_flags = act;
+	    goto locked_fail;
+	}
+	exp = act;
+	new = act | ERTS_PTS_FLG_EXEC_IMM;
+	act = erts_smp_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp);
+    } while (act != exp);
+    
+    sp->sched_flags = act;
 
     if (!c_p)
 	reds_left_in = CONTEXT_REDS/10;
@@ -1279,11 +1295,14 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-    if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
-	trace_sched_ports_where(prt, am_in, sp->port_op);
-    if (erts_system_profile_flags.runnable_ports
-	&& !erts_port_is_scheduled(prt))
-    	profile_runnable_port(prt, am_active);
+    if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
+	if (prof_runnable_ports && !(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
+	    profile_runnable_port(prt, am_active);
+	if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+	    trace_sched_ports_where(prt, am_in, sp->port_op);
+	if (prof_runnable_ports)
+	    erts_port_task_sched_unlock(&prt->sched);
+    }
 
     sp->fpe_was_unmasked = erts_block_fpe();
 
@@ -1300,17 +1319,31 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
     int reds;
     Port *prt = sp->port;
     Process *c_p = sp->c_p;
+    erts_aint32_t act;
+    unsigned int prof_runnable_ports;
 
     reds = prt->reds;
     reds += erts_port_driver_callback_epilogue(prt, NULL);
 
     erts_unblock_fpe(sp->fpe_was_unmasked);
 
-    if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
-	trace_sched_ports_where(prt, am_out, sp->port_op);
-    if (erts_system_profile_flags.runnable_ports
-	&& !erts_port_is_scheduled(prt))
-    	profile_runnable_port(prt, am_inactive);
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&prt->sched);
+
+    act = erts_smp_atomic32_read_band_mb(&prt->sched.flags,
+					 ~ERTS_PTS_FLG_EXEC_IMM);
+    ERTS_SMP_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM);
+
+    if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
+	if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+	    trace_sched_ports_where(prt, am_out, sp->port_op);
+	if (prof_runnable_ports) {
+	    if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
+		profile_runnable_port(prt, am_inactive);
+	    erts_port_task_sched_unlock(&prt->sched);
+	}
+    }
 
     erts_port_release(prt);
 
-- 
cgit v1.2.3


From 44edeae19bec364b9689f84b06da140e47401eb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 16 Jun 2014 14:47:18 +0200
Subject: erts: Add BIF erlang:get_keys/0

Returns a list of all keys in the process dictionary.
---
 erts/emulator/beam/bif.tab            | 11 ++++++--
 erts/emulator/beam/erl_process_dict.c | 52 +++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index e68b8e6274..55ac778475 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -578,7 +578,7 @@ bif io:printable_range/0
 bif os:unsetenv/1
 
 #
-# New in R17A
+# New in 17.0
 #
 
 bif re:inspect/2
@@ -601,9 +601,16 @@ bif maps:values/1
 bif erts_internal:cmp_term/2
 
 #
-# New in 17.1.
+# New in 17.1
 #
+
 bif erlang:fun_info_mfa/1
+
+# New in 18.0
+#
+
+bif erlang:get_keys/0
+
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 23e5bf737f..3ce707efda 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -82,6 +82,7 @@
 static void pd_hash_erase(Process *p, Eterm id, Eterm *ret);
 static void pd_hash_erase_all(Process *p);
 static Eterm pd_hash_get_keys(Process *p, Eterm value);
+static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd);
 static Eterm pd_hash_get_all(Process *p, ProcDict *pd);
 static Eterm pd_hash_put(Process *p, Eterm id, Eterm value);
 
@@ -275,6 +276,16 @@ BIF_RETTYPE get_1(BIF_ALIST_1)
     BIF_RET(ret);
 }
 
+BIF_RETTYPE get_keys_0(BIF_ALIST_0)
+{
+    Eterm ret;
+
+    PD_CHECK(BIF_P->dictionary);
+    ret = pd_hash_get_all_keys(BIF_P,BIF_P->dictionary);
+    PD_CHECK(BIF_P->dictionary);
+    BIF_RET(ret);
+}
+
 BIF_RETTYPE get_keys_1(BIF_ALIST_1)
 {
     Eterm ret;
@@ -412,6 +423,47 @@ Eterm erts_pd_hash_get(Process *p, Eterm id)
     return am_undefined;
 }
 
+#define PD_GET_TKEY(Dst,Src)                            \
+do {                                                    \
+    ASSERT(is_tuple((Src)));                            \
+    ASSERT(arityval(*((Eterm*)tuple_val((Src)))) == 2); \
+    (Dst) = ((Eterm*)tuple_val((Src)))[1];              \
+} while(0)
+
+static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd) {
+    Eterm* hp;
+    Eterm res = NIL;
+    Eterm tmp, tmp2;
+    unsigned int i;
+    unsigned int num;
+
+    if (pd == NULL) {
+	return res;
+    }
+
+    num = HASH_RANGE(pd);
+    hp = HAlloc(p, pd->numElements * 2);
+
+    for (i = 0; i < num; ++i) {
+	tmp = ARRAY_GET(pd, i);
+	if (is_boxed(tmp)) {
+	    PD_GET_TKEY(tmp,tmp);
+	    res = CONS(hp, tmp, res);
+	    hp += 2;
+	} else if (is_list(tmp)) {
+	    while (tmp != NIL) {
+		tmp2 = TCAR(tmp);
+		PD_GET_TKEY(tmp2,tmp2);
+		res = CONS(hp, tmp2, res);
+		hp += 2;
+		tmp = TCDR(tmp);
+	    }
+	}
+    }
+    return res;
+}
+#undef PD_GET_TKEY
+
 static Eterm pd_hash_get_keys(Process *p, Eterm value) 
 {
     Eterm *hp;
-- 
cgit v1.2.3


From e167bca85a86cc7a149d53da5cdd08b0905e71a6 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Thu, 29 May 2014 22:10:06 -0400
Subject: add enif_schedule_nif() to NIF API

In the #erlang IRC channel Anthony Ramine once mentioned the idea of
allowing a NIF to use an emulator trap, similar to a BIF trap, to schedule
another NIF for execution. This is exactly how dirty NIFs were implemented
for Erlang/OTP 17.0, so this commit refactors and generalizes that dirty
NIF code to support a new enif_schedule_nif() API function.

The enif_schedule_nif() function allows a long-running NIF to be broken
into separate NIF invocations. The NIF first executes part of the
long-running task, then calls enif_schedule_nif() to schedule a NIF for
later execution to continue the task. Any number of NIFs can be scheduled
in this manner, one after another. Since the emulator regains control
between invocations, this helps avoid problems caused by native code tying
up scheduler threads for too long.

The enif_schedule_nif() function also replaces the original experimental
dirty NIF API. The function takes a flags parameter that a caller can use
to indicate the NIF should be scheduled onto either a dirty CPU scheduler
thread, a dirty I/O scheduler thread, or scheduled as a regular NIF on a
regular scheduler thread. With this change, the original experimental
enif_schedule_dirty_nif(), enif_schedule_dirty_nif_finalizer() and
enif_dirty_nif_finalizer() API functions are no longer needed and have been
removed. Explicit scheduling of a dirty NIF finalization function is no
longer necessary; if an application wants similar functionality, it can
have a dirty NIF just invoke enif_schedule_nif() to schedule a non-dirty
NIF to complete its task.

Lift the restriction that dirty NIFs can't call enif_make_badarg() to raise
an exception. This was a problem with the original dirty NIF API because it
forced developers to get and check all incoming arguments in a regular NIF,
and then schedule the dirty NIF which then had to get all the arguments
again. Now, the argument checking can be done in the dirty NIF and it can
call enif_make_badarg() itself to flag incorrect arguments.

Extend the ErlNifFunc struct with a new flags field that allows NIFs to be
declared as dirty. The default value for this field is 0, indicating a
regular NIF, so it's backwards compatible with all existing statically
initialized ErlNifFunc struct instances, and so such instances require no
code changes. Defining the flags field with a value of
ERL_NIF_DIRTY_JOB_CPU_BOUND indicates that the NIF should execute on a
dirty CPU scheduler thread, or defining it with a value of
ERL_NIF_DIRTY_JOB_IO_BOUND indicates that the NIF should execute on a dirty
I/O scheduler thread. Any other flags field value causes a NIF library
loading error.

Extend the ErlNifEntry struct with a new options field that indicates
whether a NIF library was built with support for optional features such as
dirty NIFs. When a NIF library is loaded, the runtime checks the options
field to ensure compatibility. If a NIF library built with dirty NIF
support is loaded into a runtime that does not support dirty NIFs, and the
library defines one or more ErlNifFunc entries with non-zero flags fields
indicating dirty NIFs, a NIF library loading error results. There is no
error if a NIF library built with dirty NIF support is loaded into a
runtime that does not support dirty NIFs but the library does not have any
dirty NIFs. It is also not an error if a library without dirty NIF support
is loaded into a runtime built with dirty NIF support.

Add documentation and tests for enif_schedule_nif().
---
 erts/emulator/beam/beam_emu.c          |   8 +-
 erts/emulator/beam/beam_load.c         |   6 +-
 erts/emulator/beam/erl_gc.c            |  14 +
 erts/emulator/beam/erl_nif.c           | 513 +++++++++++++++++++++++++--------
 erts/emulator/beam/erl_nif.h           |  26 +-
 erts/emulator/beam/erl_nif_api_funcs.h |   8 +-
 erts/emulator/beam/erl_process.c       |  34 ++-
 erts/emulator/beam/erl_process.h       |  24 +-
 8 files changed, 473 insertions(+), 160 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1026e5f649..8bfb7d2ad2 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3503,6 +3503,7 @@ get_map_elements_fail:
 	     * I[0]: &&call_nif
 	     * I[1]: Function pointer to NIF function
 	     * I[2]: Pointer to erl_module_nif
+	     * I[3]: Function pointer to dirty NIF
 	     */
 	    BifFunction vbf;
 
@@ -3523,13 +3524,6 @@ get_map_elements_fail:
 		reg[0] = r(0);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		erts_post_nif(&env);
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (is_non_value(nif_bif_result) && c_p->freason == TRAP) {
-		    Export* ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(c_p);
-		    ep->code[0] = I[-3];
-		    ep->code[1] = I[-2];
-		}
-#endif
 	    }
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index e96177cfd9..cfc6146b0a 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2363,7 +2363,11 @@ load_code(LoaderState* stp)
 
 		if (stp->may_load_nif) {
 		    const int finfo_ix = ci - FUNC_INFO_SZ;
-		    enum { MIN_FUNC_SZ = 3 };		    
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    enum { MIN_FUNC_SZ = 4 };
+#else
+		    enum { MIN_FUNC_SZ = 3 };
+#endif
 		    if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) {		   
 			/* Must make room for call_nif op */
 			int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index aa15d2cc57..0db42d4325 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2018,6 +2018,20 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	roots[n].sz = 1;
 	n++;
     }
+
+    /*
+     * If a NIF has saved arguments, they need to be added
+     */
+    if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) {
+	Eterm* argv;
+	int argc;
+	if (erts_setup_nif_gc(p, &argv, &argc)) {
+	    roots[n].v = argv;
+	    roots[n].sz = argc;
+	    n++;
+	}
+    }
+
     ASSERT(n <= rootset->size);
 
     mp = p->msg.first;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ff551ea3af..1414744763 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1513,72 +1513,251 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent)
     return ERTS_BIF_REDS_LEFT(env->proc) == 0;
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-
-/* NIFs exports need one more item than the Export struct provides, the
- * erl_module_nif*, so the DirtyNifExport below adds that. The Export
- * member must be first in the struct.
+/*
+ * NIF exports need a few more items than the Export struct provides,
+ * including the erl_module_nif* and a NIF function pointer, so the
+ * NifExport below adds those. The Export member must be first in the
+ * struct. The saved_mfa, saved_argc, nif_level, alloced_argv_sz and argv
+ * members are used to track the MFA and arguments of the top NIF in case a
+ * chain of one or more enif_schedule_nif() calls results in an exception,
+ * since in that case the original MFA and registers have to be restored
+ * before returning to Erlang to ensure stacktrace information associated
+ * with the exception is correct.
  */
+typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]);
+
 typedef struct {
     Export exp;
     struct erl_module_nif* m;
-} DirtyNifExport;
+    NativeFunPtr fp;
+    Eterm saved_mfa[3];
+    int saved_argc;
+    int alloced_argv_sz;
+    Eterm argv[1];
+} NifExport;
 
-static void
-alloc_proc_psd(Process* proc, DirtyNifExport **ep)
+/*
+ * If a process has saved arguments, they need to be part of the GC
+ * rootset. The function below is called from setup_rootset() in
+ * erl_gc.c. This function is declared in erl_process.h.
+ */
+int
+erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj)
+{
+    NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    int gc = (ep && ep->saved_argc > 0);
+
+    if (gc) {
+	*objv = ep->argv;
+	*nobj = ep->saved_argc;
+    }
+    return gc;
+}
+
+/*
+ * Allocate a NifExport and set it in proc specific data
+ */
+static NifExport*
+allocate_nif_sched_data(Process* proc, int argc)
 {
+    NifExport* ep;
+    size_t argv_extra, total;
     int i;
-    if (!*ep) {
-	*ep = erts_alloc(ERTS_ALC_T_PSD, sizeof(DirtyNifExport));
-	sys_memset((void*) *ep, 0, sizeof(DirtyNifExport));
-	for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	    (*ep)->exp.addressv[i] = &(*ep)->exp.code[3];
-	}
-	(*ep)->exp.code[3] = (BeamInstr) em_call_nif;
+
+    argv_extra = argc > 1 ? sizeof(Eterm)*(argc-1) : 0;
+    total = sizeof(NifExport) + argv_extra;
+    ep = erts_alloc(ERTS_ALC_T_PSD, total);
+    sys_memset((void*) ep, 0, total);
+    ep->alloced_argv_sz = argc;
+    for (i=0; i<ERTS_NUM_CODE_IX; i++) {
+	ep->exp.addressv[i] = &ep->exp.code[3];
     }
-    (void) ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, &(*ep)->exp);
+    ep->exp.code[3] = (BeamInstr) em_call_nif;
+    (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, &ep->exp);
+    return ep;
 }
 
+/*
+ * Initialize a NifExport struct. Create it if needed and store it in the
+ * proc. The direct_fp function is what will be invoked by op_call_nif, and
+ * the indirect_fp function, if not NULL, is what the direct_fp function
+ * will call. If the allocated NifExport isn't enough to hold all of argv,
+ * allocate a larger one. Save MFA and registers only if the need_save
+ * parameter is true.
+ */
 static ERL_NIF_TERM
-execute_dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
+		    int need_save, int argc, const ERL_NIF_TERM argv[])
 {
-    Eterm* reg = ERTS_PROC_GET_SCHDATA(env->proc)->x_reg_array;
-    ERL_NIF_TERM result, dirty_result = (ERL_NIF_TERM) reg[0];
-    typedef ERL_NIF_TERM (*FinalizerFP)(ErlNifEnv*, ERL_NIF_TERM);
-    FinalizerFP fp;
-#if HAVE_INT64 && SIZEOF_LONG != 8
-    ASSERT(sizeof(fp) <= sizeof(ErlNifUInt64));
-    enif_get_uint64(env, reg[1], (ErlNifUInt64 *) &fp);
-#else
-    ASSERT(sizeof(fp) <= sizeof(unsigned long));
-    enif_get_ulong(env, reg[1], (unsigned long *) &fp);
-#endif
-    result = (*fp)(env, dirty_result);
-    if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0
-	&& env->mod_nif->mod == NULL)
-	close_lib(env->mod_nif);
-    return result;
+    Process* proc = env->proc;
+    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
+    NifExport* ep;
+    int i;
+
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    if (!ep)
+	ep = allocate_nif_sched_data(proc, argc);
+    else if (need_save && ep->alloced_argv_sz < argc) {
+	NifExport* new_ep = allocate_nif_sched_data(proc, argc);
+	erts_free(ERTS_ALC_T_PSD, (void*) ep);
+	ep = new_ep;
+    }
+    ERTS_VBUMP_ALL_REDS(proc);
+    for (i = 0; i < argc; i++) {
+	if (need_save)
+	    ep->argv[i] = reg[i];
+	reg[i] = (Eterm) argv[i];
+    }
+    if (need_save) {
+	ep->saved_mfa[0] = proc->current[0];
+	ep->saved_mfa[1] = proc->current[1];
+	ep->saved_mfa[2] = proc->current[2];
+	ep->saved_argc = argc;
+    }
+    proc->i = (BeamInstr*) ep->exp.addressv[0];
+    ep->exp.code[0] = (BeamInstr) proc->current[0];
+    ep->exp.code[1] = (BeamInstr) proc->current[1];
+    ep->exp.code[2] = argc;
+    ep->exp.code[4] = (BeamInstr) direct_fp;
+    ep->m = env->mod_nif;
+    ep->fp = indirect_fp;
+    proc->freason = TRAP;
+    return THE_NON_VALUE;
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
+/*
+ * Restore saved MFA and registers. Registers are restored only when the
+ * exception flag is true.
+ */
+static void
+restore_nif_mfa(Process* proc, NifExport* ep, int exception)
+{
+    int i;
+    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
 
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+    proc->current[0] = ep->saved_mfa[0];
+    proc->current[1] = ep->saved_mfa[1];
+    proc->current[2] = ep->saved_mfa[2];
+    if (exception)
+	for (i = 0; i < ep->saved_argc; i++)
+	    reg[i] = ep->argv[i];
+    ep->saved_argc = 0;
+    ep->saved_mfa[0] = THE_NON_VALUE;
+}
 
-ERL_NIF_TERM
-enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
-			ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]),
-			int argc, const ERL_NIF_TERM argv[])
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+/*
+ * Finalize a dirty NIF call. This function is scheduled to cause the VM to
+ * switch the process off a dirty scheduler thread and back onto a regular
+ * scheduler thread, and then return the result from the dirty NIF. It also
+ * restores the original NIF MFA when necessary based on the value of
+ * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL
+ * means restore, NULL means do not restore.
+ */
+static ERL_NIF_TERM
+dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    Process* proc = env->proc;
+    NifExport* ep;
+
+    ASSERT(argc == 1);
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ASSERT(ep);
+    if (ep->fp)
+	restore_nif_mfa(proc, ep, 0);
+    return argv[0];
+}
+
+/* Finalize a dirty NIF call that raised an exception.  Otherwise same as
+ * the dirty_nif_finalizer() function.
+ */
+static ERL_NIF_TERM
+dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    Process* proc = env->proc;
+    NifExport* ep;
+
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ASSERT(ep);
+    if (ep->fp)
+	restore_nif_mfa(proc, ep, 1);
+    return enif_make_badarg(env);
+}
+
+/*
+ * Dirty NIF execution wrapper function. Invoke an application's dirty NIF,
+ * then check the result and schedule the appropriate finalizer function
+ * where needed. Also restore the original NIF MFA when appropriate.
+ */
+static ERL_NIF_TERM
+execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    Process* proc = env->proc;
+    NativeFunPtr fp = (NativeFunPtr) proc->current[6];
+    NifExport* ep;
+    ERL_NIF_TERM result;
+
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
+
+    /*
+     * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution
+     */
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ASSERT(ep);
+    ep->fp = NULL;
+    result = (*fp)(env, argc, argv);
+    erts_smp_atomic32_read_band_mb(&proc->state,
+				   ~(ERTS_PSFLG_DIRTY_CPU_PROC
+				     |ERTS_PSFLG_DIRTY_IO_PROC
+				     |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q
+				     |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
+    if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL)
+	close_lib(env->mod_nif);
+    /*
+     * If no more NIFs were scheduled by the native call via
+     * enif_schedule_nif(), then ep->fp will still be NULL as set above, in
+     * which case we need to restore the original NIF calling
+     * context. Reuse fp essentially as a boolean for this, passing it to
+     * init_nif_sched_data below. Both dirty_nif_exception and
+     * dirty_nif_finalizer then check ep->fp to decide whether or not to
+     * restore the original calling context.
+     */
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ASSERT(ep);
+    if (ep->fp)
+	fp = NULL;
+    if (is_non_value(result)) {
+	if (proc->freason != TRAP) {
+	    ASSERT(proc->freason == BADARG);
+	    return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv);
+	} else {
+	    if (ep->fp == NULL)
+		restore_nif_mfa(proc, ep, 1);
+	    return result;
+	}
+    }
+    else
+	return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result);
+}
+
+/*
+ * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute
+ * via the execute_dirty_nif() wrapper function. The dirty scheduler thread
+ * type (CPU or I/O) is indicated in flags parameter.
+ */
+static ERTS_INLINE ERL_NIF_TERM
+schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[])
 {
-#ifdef USE_THREADS
     erts_aint32_t state, n, a;
     Process* proc = env->proc;
-    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
-    DirtyNifExport* ep = NULL;
-    int i;
+    NativeFunPtr fp = (NativeFunPtr) proc->current[6];
+    NifExport* ep;
+    int need_save;
 
-    int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND));
-    if (chkflgs != ERL_NIF_DIRTY_JOB_IO_BOUND && chkflgs != ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	return enif_make_badarg(env);
+    ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND);
 
     a = erts_smp_atomic32_read_acqb(&proc->state);
     while (1) {
@@ -1590,7 +1769,7 @@ enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
 	 */
 	n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC
 	       |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q);
-	if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND)
+	if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
 	    n |= ERTS_PSFLG_DIRTY_CPU_PROC;
 	else
 	    n |= ERTS_PSFLG_DIRTY_IO_PROC;
@@ -1598,69 +1777,100 @@ enif_schedule_dirty_nif(ErlNifEnv* env, int flags,
 	if (a == state)
 	    break;
     }
-    if (!(ep = (DirtyNifExport*) ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
-	alloc_proc_psd(proc, &ep);
-    ERTS_VBUMP_ALL_REDS(proc);
-    ep->exp.code[2] = argc;
-    for (i = 0; i < argc; i++) {
-	reg[i] = (Eterm) argv[i];
-    }
-    proc->i = (BeamInstr*) ep->exp.addressv[0];
-    ep->exp.code[4] = (BeamInstr) fp;
-    ep->m = env->mod_nif;
-    proc->freason = TRAP;
-
     erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    need_save = (ep == NULL || is_non_value(ep->saved_mfa[0]));
+    return init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv);
+}
 
-    return THE_NON_VALUE;
-#else
-    return (*fp)(env, argc, argv);
-#endif
+static ERL_NIF_TERM
+schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv);
+}
+
+static ERL_NIF_TERM
+schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv);
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+/*
+ * NIF execution wrapper used by enif_schedule_nif() for regular NIFs. It
+ * calls the actual NIF, restores original NIF MFA if necessary, and
+ * then returns the NIF result.
+ */
+static ERL_NIF_TERM
+execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    Process* proc = env->proc;
+    NativeFunPtr fp = (NativeFunPtr) proc->current[6];
+    NifExport* ep;
+    ERL_NIF_TERM result;
+
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ASSERT(ep);
+    ep->fp = NULL;
+    result = (*fp)(env, argc, argv);
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ASSERT(ep);
+    /*
+     * If no NIFs were scheduled by the native call via
+     * enif_schedule_nif(), then ep->fp will still be NULL as set above, in
+     * which case we need to restore the original NIF MFA.
+     */
+    if (ep->fp == NULL)
+	restore_nif_mfa(proc, ep, is_non_value(result) && proc->freason != TRAP);
+    return result;
 }
 
 ERL_NIF_TERM
-enif_schedule_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result,
-				  ERL_NIF_TERM (*fp)(ErlNifEnv*, ERL_NIF_TERM))
+enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
+		  ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]),
+		  int argc, const ERL_NIF_TERM argv[])
 {
-#ifdef USE_THREADS
     Process* proc = env->proc;
-    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
-    DirtyNifExport* ep;
+    NifExport* ep;
+    ERL_NIF_TERM fun_name_atom, result;
+    int need_save;
 
-    erts_smp_atomic32_read_band_mb(&proc->state,
-				   ~(ERTS_PSFLG_DIRTY_CPU_PROC
-				     |ERTS_PSFLG_DIRTY_IO_PROC
-				     |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q
-				     |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
-    if (!(ep = (DirtyNifExport*) ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc)))
-	alloc_proc_psd(proc, &ep);
-    ERTS_VBUMP_ALL_REDS(proc);
-    ep->exp.code[2] = 2;
-    reg[0] = (Eterm) result;
-#if HAVE_INT64 && SIZEOF_LONG != 8
-    ASSERT(sizeof(fp) <= sizeof(ErlNifUInt64));
-    reg[1] = (Eterm) enif_make_uint64(env, (ErlNifUInt64) fp);
-#else
-    ASSERT(sizeof(fp) <= sizeof(unsigned long));
-    reg[1] = (Eterm) enif_make_ulong(env, (unsigned long) fp);
-#endif
-    proc->i = (BeamInstr*) ep->exp.addressv[0];
-    ep->exp.code[4] = (BeamInstr) execute_dirty_nif_finalizer;
-    proc->freason = TRAP;
+    if (argc > MAX_ARG)
+	return enif_make_badarg(env);
+    fun_name_atom = enif_make_atom(env, fun_name);
+    if (enif_is_exception(env, fun_name_atom))
+	return fun_name_atom;
 
-    return THE_NON_VALUE;
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    need_save = (ep == NULL || is_non_value(ep->saved_mfa[0]));
+
+    if (flags) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+	NativeFunPtr sched_fun;
+	int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND));
+	if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND)
+	    sched_fun = schedule_dirty_io_nif;
+	else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND)
+	    sched_fun = schedule_dirty_cpu_nif;
+	else
+	    return enif_make_badarg(env);
+	result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv);
 #else
-    return (*fp)(env, result);
+	return enif_make_badarg(env);
 #endif
-}
+    }
+    else
+	result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv);
 
-/* A simple finalizer that just returns its result argument */
-ERL_NIF_TERM
-enif_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result)
-{
+    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ASSERT(ep);
+    ep->exp.code[1] = (BeamInstr) fun_name_atom;
     return result;
 }
 
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+
 int
 enif_is_on_dirty_scheduler(ErlNifEnv* env)
 {
@@ -1977,6 +2187,35 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, ..
     return ret;
 }
 
+/*
+ * The function below is for looping through ErlNifFunc arrays, helping
+ * provide backwards compatibility across the version 2.7 change that added
+ * the "flags" field to ErlNifFunc.
+ */
+static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func)
+{
+    ASSERT(incrp);
+    if (!*incrp) {
+	if (entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
+	    *incrp = sizeof(ErlNifFunc);
+	else {
+	    /*
+	     * ErlNifFuncV1 below is what ErlNifFunc was before the
+	     * addition of the flags field for 2.7, and is needed to handle
+	     * backward compatibility.
+	     */
+	    typedef struct {
+		const char* name;
+		unsigned arity;
+		ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+	    }ErlNifFuncV1;
+	    *incrp = sizeof(ErlNifFuncV1);
+	}
+    }
+    return (ErlNifFunc*) ((char*)func + *incrp);
+}
+
+
 BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 {
     static const char bad_lib[] = "bad_lib";
@@ -2086,22 +2325,48 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     }
     else {
 	/*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/
-    
+
+	int maybe_dirty_nifs = ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
+				&& (entry->options & ERL_NIF_DIRTY_NIF_OPTION));
+	int incr = 0;
+	ErlNifFunc* f = entry->funcs;
 	for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
 	    BeamInstr** code_pp;
-	    ErlNifFunc* f = &entry->funcs[i];
 	    if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1)
 		|| (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) {
 		ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u",
 				     mod_atom, f->name, f->arity);
-	    }    
-	    else if (code_pp[1] - code_pp[0] < (5+3)) {
+	    }
+	    else if (maybe_dirty_nifs && f->flags) {
+		/*
+		 * If the flags field is non-zero and this emulator was
+		 * built with dirty scheduler support, check that the flags
+		 * value is legal. But if this emulator was built without
+		 * dirty scheduler support, treat a non-zero flags field as
+		 * a load error.
+		 */
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND)
+		    ret = load_nif_error(BIF_P, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u",
+					 f->flags, mod_atom, f->name, f->arity);
+#else
+		ret = load_nif_error(BIF_P, bad_lib, "NIF %T:%s/%u requires a runtime with dirty scheduler support.",
+				     mod_atom, f->name, f->arity);
+#endif
+	    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    else if (code_pp[1] - code_pp[0] < (5+4))
+#else
+	    else if (code_pp[1] - code_pp[0] < (5+3))
+#endif
+	    {
 		ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif"
-				     " in module (%T:%s/%u to small)",
-				     mod_atom, entry->funcs[i].name, entry->funcs[i].arity);
+				     " in module (%T:%s/%u too small)",
+				     mod_atom, f->name, f->arity);
 	    }
 	    /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n",
-			 mod_atom, entry->funcs[i].name, entry->funcs[i].arity);*/
+	      mod_atom, f->name, f->arity);*/
+	    f = next_func(entry, &incr, f);
 	}
     }
 
@@ -2127,7 +2392,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	 * is deprecated and was only ment as a development feature not to
 	 * be used in production systems. (See warning below)
 	 */
-	int k;
+	int k, old_incr = 0;
+	ErlNifFunc* old_func;
         lib->priv_data = mod->curr.nif->priv_data;
 
 	ASSERT(mod->curr.nif->entry != NULL);
@@ -2136,13 +2402,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    goto error;
 	}
 	/* Check that no NIF is removed */
+	old_func = mod->curr.nif->entry->funcs;
 	for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) {
-	    ErlNifFunc* old_func = &mod->curr.nif->entry->funcs[k];
+	    int incr = 0;
+	    ErlNifFunc* f = entry->funcs;
 	    for (i=0; i < entry->num_of_funcs; i++) {
-		if (old_func->arity == entry->funcs[i].arity
-		    && sys_strcmp(old_func->name, entry->funcs[i].name) == 0) {			   
+		if (old_func->arity == f->arity
+		    && sys_strcmp(old_func->name, f->name) == 0) {
 		    break;
 		}
+		f = next_func(entry, &incr, f);
 	    }
 	    if (i == entry->num_of_funcs) {
 		ret = load_nif_error(BIF_P,reload,"Reloaded library missing "
@@ -2150,7 +2419,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 				     old_func->name, old_func->arity);
 		goto error;
 	    }
-	}       
+	    old_func = next_func(mod->curr.nif->entry, &old_incr, old_func);
+	}
 	erts_pre_nif(&env, BIF_P, lib);
 	veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2);
 	erts_post_nif(&env);
@@ -2197,13 +2467,17 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	/*
 	** Everything ok, patch the beam code with op_call_nif
 	*/
-        mod->curr.nif = lib;
+
+	int incr = 0;
+	ErlNifFunc* f = entry->funcs;
+
+	mod->curr.nif = lib;
 	for (i=0; i < entry->num_of_funcs; i++)
 	{
 	    BeamInstr* code_ptr;
-	    erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom, ERTS_ATOM_ENC_LATIN1); 
-	    code_ptr = *get_func_pp(mod->curr.code, f_atom, entry->funcs[i].arity);
-	    
+	    erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1);
+	    code_ptr = *get_func_pp(mod->curr.code, f_atom, f->arity);
+
 	    if (code_ptr[1] == 0) {
 		code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif);
 	    }
@@ -2211,10 +2485,21 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		GenericBp* g = (GenericBp *) code_ptr[1];
 		ASSERT(code_ptr[5+0] ==
 		       (BeamInstr) BeamOp(op_i_generic_breakpoint));
-	        g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
-	    }	    
-	    code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr;
+		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
+	    }
+	    if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
+		&& (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) {
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+		code_ptr[5+3] = (BeamInstr) f->fptr;
+		code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
+		    (BeamInstr) schedule_dirty_io_nif :
+		    (BeamInstr) schedule_dirty_cpu_nif;
+#endif
+	    }
+	    else
+		code_ptr[5+1] = (BeamInstr) f->fptr;
 	    code_ptr[5+2] = (BeamInstr) lib;
+	    f = next_func(entry, &incr, f);
 	}
     }
     else {
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 5b93c2398e..226fc199a1 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -42,9 +42,13 @@
 ** 2.5: R17 Maps API additions
 ** 2.6: R17 with maps
 **      R17 dirty schedulers
+** 2.7: 17.3 add enif_schedule_nif
+**           remove enif_schedule_dirty_nif, enif_schedule_dirty_nif_finalizer, enif_dirty_nif_finalizer
+**           add ErlNifEntry options
+**           add ErlNifFunc flags
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 6
+#define ERL_NIF_MINOR_VERSION 7
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
@@ -125,8 +129,10 @@ typedef struct
     const char* name;
     unsigned arity;
     ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+    unsigned flags;
 }ErlNifFunc;
 
+
 typedef struct enif_entry_t
 {
     int major;
@@ -139,8 +145,11 @@ typedef struct enif_entry_t
     int  (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
     void (*unload) (ErlNifEnv*, void* priv_data);
     const char* vm_variant;
+    unsigned options;
 }ErlNifEntry;
 
+/* Field bits for ErlNifEntry options */
+#define ERL_NIF_DIRTY_NIF_OPTION 1
 
 
 typedef struct
@@ -232,10 +241,21 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  else
 #    define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
 #  endif
-#  define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
+#  ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+#    define ERL_NIF_INIT_BODY do {					\
+	memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)); \
+	entry.options = ERL_NIF_DIRTY_NIF_OPTION;			\
+     } while(0)
+#  else
+#    define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
+#  endif
 #else 
 #  define ERL_NIF_INIT_GLOB
-#  define ERL_NIF_INIT_BODY
+#  ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+#    define ERL_NIF_INIT_BODY entry.options = ERL_NIF_DIRTY_NIF_OPTION
+#  else
+#    define ERL_NIF_INIT_BODY
+#  endif
 #  ifdef STATIC_ERLANG_NIF
 #    define ERL_NIF_INIT_DECL(MODNAME)  ErlNifEntry* MODNAME ## _nif_init(void)
 #  else
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index d7c554e60b..be39816a64 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -141,10 +141,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg));
 ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg));
 ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_dirty_nif,(ErlNifEnv*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
-ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_dirty_nif_finalizer,(ErlNifEnv*,ERL_NIF_TERM,ERL_NIF_TERM (*)(ErlNifEnv*,ERL_NIF_TERM)));
-ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_dirty_nif_finalizer,(ErlNifEnv*,ERL_NIF_TERM));
 ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
 #endif
@@ -289,10 +287,8 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa
 #  define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen)
 #  define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym)
 #  define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice)
+#  define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif)
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-#  define enif_schedule_dirty_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_dirty_nif)
-#  define enif_schedule_dirty_nif_finalizer ERL_NIF_API_FUNC_MACRO(enif_schedule_dirty_nif_finalizer)
-#  define enif_dirty_nif_finalizer ERL_NIF_API_FUNC_MACRO(enif_dirty_nif_finalizer)
 #  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
 #  define enif_have_dirty_schedulers ERL_NIF_API_FUNC_MACRO(enif_have_dirty_schedulers)
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1606ad119d..e96d5c4ccd 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -590,12 +590,10 @@ erts_pre_init_process(void)
      erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks
 	 = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-     erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].get_locks
-	 = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].set_locks
-	 = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_SET_LOCKS;
-#endif
+     erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks
+	 = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS;
+     erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks
+	 = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS;
 
      /* Check that we have locks for all entries */
      for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
@@ -3758,17 +3756,25 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	    }
 #ifdef ERTS_DIRTY_SCHEDULERS
 	    else if (state & ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q) {
-		erts_aint32_t old;
-		old = erts_smp_atomic32_read_band_nob(&proc->state,
-						      ~(ERTS_PSFLG_DIRTY_CPU_PROC
-							| ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q));
+#ifdef DEBUG
+		erts_aint32_t old =
+#else
+		(void)
+#endif
+		    erts_smp_atomic32_read_band_nob(&proc->state,
+						    ~(ERTS_PSFLG_DIRTY_CPU_PROC
+						      | ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q));
 		/* assert that no other dirty flags are set */
 		ASSERT(!(old & (ERTS_PSFLG_DIRTY_IO_PROC|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)));
 	    } else if (state & ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) {
-		erts_aint32_t old;
-		old = erts_smp_atomic32_read_band_nob(&proc->state,
-						      ~(ERTS_PSFLG_DIRTY_IO_PROC
-							| ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
+#ifdef DEBUG
+		erts_aint32_t old =
+#else
+		(void)
+#endif
+		    erts_smp_atomic32_read_band_nob(&proc->state,
+						    ~(ERTS_PSFLG_DIRTY_IO_PROC
+						      | ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
 		/* assert that no other dirty flags are set */
 		ASSERT(!(old & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q)));
 	    }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index ed6dadbffa..31f4a09c94 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -734,13 +734,9 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_DIST_ENTRY			3
 #define ERTS_PSD_CALL_TIME_BP			4
 #define ERTS_PSD_DELAYED_GC_TASK_QS		5
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT	6
+#define ERTS_PSD_NIF_TRAP_EXPORT		6
 
 #define ERTS_PSD_SIZE				7
-#else
-#define ERTS_PSD_SIZE				6
-#endif
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -767,10 +763,8 @@ typedef struct {
 #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
-#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
-#endif
+#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
 typedef struct {
     ErtsProcLocks get_locks;
@@ -1367,6 +1361,8 @@ Uint64 erts_get_proc_interval(void);
 Uint64 erts_ensure_later_proc_interval(Uint64);
 Uint64 erts_step_proc_interval(void);
 
+int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
+
 ErtsProcList *erts_proclist_create(Process *);
 void erts_proclist_destroy(ErtsProcList *);
 
@@ -1817,12 +1813,10 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
     ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(P) \
-    ((Export *) erts_psd_get((P), ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT))
-#define ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(P, L, DSTE) \
-    ((Export *) erts_psd_set((P), (L), ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT, (void *) (DSTE)))
-#endif
+#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
+    ((Export *) erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT))
+#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, DSTE) \
+    ((Export *) erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (DSTE)))
 
 
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
-- 
cgit v1.2.3


From 88b094b6439737b61c117cd6873beea4518757a8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 1 Jul 2014 22:04:55 +0200
Subject: erts: Implement yielding for distributed send of large messages

Use same mechanism as term_to_binary to yield
while encoding large messages for distributed send.
---
 erts/emulator/beam/atom.names |   1 +
 erts/emulator/beam/bif.c      | 204 +++++++++++++-----
 erts/emulator/beam/dist.c     | 475 +++++++++++++++++++++++++-----------------
 erts/emulator/beam/dist.h     |  97 ++++++++-
 erts/emulator/beam/external.c |  97 ++++-----
 erts/emulator/beam/external.h |  11 +-
 6 files changed, 576 insertions(+), 309 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 5d06a32941..ae2c6a3c67 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -198,6 +198,7 @@ atom dotall
 atom driver
 atom driver_options
 atom dsend
+atom dsend_continue_trap
 atom dunlink
 atom duplicate_bag
 atom dupnames
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index fcbeb6cf5c..99566443aa 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -46,7 +46,7 @@ static Export* set_cpu_topology_trap = NULL;
 static Export* await_proc_exit_trap = NULL;
 static Export* await_port_send_result_trap = NULL;
 Export* erts_format_cpu_topology_trap = NULL;
-
+static Export dsend_continue_trap_export;
 static Export *await_sched_wall_time_mod_trap;
 static erts_smp_atomic32_t sched_wall_time;
 
@@ -1777,6 +1777,8 @@ BIF_RETTYPE whereis_1(BIF_ALIST_1)
  * erlang:'!'/2
  */
 
+HIPE_WRAPPER_BIF_DISABLE_GC(ebif_bang, 2)
+
 BIF_RETTYPE
 ebif_bang_2(BIF_ALIST_2)
 {
@@ -1795,34 +1797,36 @@ ebif_bang_2(BIF_ALIST_2)
 #define SEND_USER_ERROR		(-5)
 #define SEND_INTERNAL_ERROR	(-6)
 #define SEND_AWAIT_RESULT	(-7)
+#define SEND_YIELD_CONTINUE     (-8)
+
 
-Sint do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp);
+Sint do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext*);
 
 static Sint remote_send(Process *p, DistEntry *dep,
-			Eterm to, Eterm full_to, Eterm msg, int suspend)
+			Eterm to, Eterm full_to, Eterm msg,
+			ErtsSendContext* ctx)
 {
     Sint res;
     int code;
-    ErtsDSigData dsd;
 
     ASSERT(is_atom(to) || is_external_pid(to));
 
-    code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, !suspend);
+    code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend);
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
     case ERTS_DSIG_PREP_NOT_CONNECTED:
 	res = SEND_TRAP;
 	break;
     case ERTS_DSIG_PREP_WOULD_SUSPEND:
-	ASSERT(!suspend);
+	ASSERT(!ctx->suspend);
 	res = SEND_YIELD;
 	break;
     case ERTS_DSIG_PREP_CONNECTED: {
 
 	if (is_atom(to))
-	    code = erts_dsig_send_reg_msg(&dsd, to, msg);
+	    code = erts_dsig_send_reg_msg(to, msg, ctx);
 	else
-	    code = erts_dsig_send_msg(&dsd, to, msg);
+	    code = erts_dsig_send_msg(to, msg, ctx);
 	/*
 	 * Note that reductions have been bumped on calling
 	 * process by erts_dsig_send_reg_msg() or
@@ -1830,6 +1834,8 @@ static Sint remote_send(Process *p, DistEntry *dep,
 	 */
 	if (code == ERTS_DSIG_SEND_YIELD)
 	    res = SEND_YIELD_RETURN;
+	else if (code == ERTS_DSIG_SEND_CONTINUE)
+	    res = SEND_YIELD_CONTINUE;
 	else
 	    res = 0;
 	break;
@@ -1850,7 +1856,8 @@ static Sint remote_send(Process *p, DistEntry *dep,
 }
 
 Sint
-do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
+do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
+{
     Eterm portid;
     Port *pt;
     Process* rp;
@@ -1881,7 +1888,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 	    erts_send_error_to_logger(p->group_leader, dsbufp);
 	    return 0;
 	}
-	return remote_send(p, dep, to, to, msg, suspend);
+	return remote_send(p, dep, to, to, msg, ctx);
     } else if (is_atom(to)) {
 	Eterm id = erts_whereis_name_to_id(p, to);
 
@@ -1936,7 +1943,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 	ret_val = 0;
         
 	if (pt) {
-	    int ps_flags = suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND;
+	    int ps_flags = ctx->suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND;
 	    *refp = NIL;
 
 	    switch (erts_port_command(p, ps_flags, pt, msg, refp)) {
@@ -1945,12 +1952,12 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 		return SEND_USER_ERROR;
 	    case ERTS_PORT_OP_BUSY:
 		/* Nothing has been sent */
-		if (suspend)
+		if (ctx->suspend)
 		    erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt);
 		return SEND_YIELD;
 	    case ERTS_PORT_OP_BUSY_SCHEDULED:
 		/* Message was sent */
-		if (suspend) {
+		if (ctx->suspend) {
 		    erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt);
 		    ret_val = SEND_YIELD_RETURN;
 		    break;
@@ -2030,9 +2037,14 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
 	    return 0;
 	}
 
-	ret = remote_send(p, dep, tp[1], to, msg, suspend);
-	if (dep)
-	    erts_deref_dist_entry(dep);
+	ret = remote_send(p, dep, tp[1], to, msg, ctx);
+	if (ret != SEND_YIELD_CONTINUE) {
+	    if (dep) {
+		erts_deref_dist_entry(dep);
+	    }
+	} else {
+	    ctx->dep_to_deref = dep;
+	}
 	return ret;
     } else {
 	if (IS_TRACED(p)) /* XXX Is this really neccessary ??? */
@@ -2063,9 +2075,11 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
     }
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(send, 3)
 
 BIF_RETTYPE send_3(BIF_ALIST_3)
 {
+    BIF_RETTYPE retval;
     Eterm ref;
     Process *p = BIF_P;
     Eterm to = BIF_ARG_1;
@@ -2073,34 +2087,44 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     Eterm opts = BIF_ARG_3;
 
     int connect = !0;
-    int suspend = !0;
     Eterm l = opts;
     Sint result;
-    
+    DeclareTypedTmpHeap(ErtsSendContext, ctx, BIF_P);
+    UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P);
+
+    ctx->suspend = !0;
+    ctx->dep_to_deref = NULL;
+    ctx->return_term = am_ok;
+    ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR);
+    ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT;
+
     while (is_list(l)) {
 	if (CAR(list_val(l)) == am_noconnect) {
 	    connect = 0;
 	} else if (CAR(list_val(l)) == am_nosuspend) {
-	    suspend = 0;
+	    ctx->suspend = 0;
 	} else {
-	    BIF_ERROR(p, BADARG);
+	    ERTS_BIF_PREP_ERROR(retval, p, BADARG);
+	    goto done;
 	}
 	l = CDR(list_val(l));
     }
     if(!is_nil(l)) {
-	BIF_ERROR(p, BADARG);
+	ERTS_BIF_PREP_ERROR(retval, p, BADARG);
+	goto done;
     }
 
 #ifdef DEBUG
     ref = NIL;
 #endif
 
-    result = do_send(p, to, msg, suspend, &ref);
+    result = do_send(p, to, msg, &ref, ctx);
     if (result > 0) {
 	ERTS_VBUMP_REDS(p, result);
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
 	    goto yield_return;
-	BIF_RET(am_ok);
+	ERTS_BIF_PREP_RET(retval, am_ok);
+	goto done;
     }
 
     switch (result) {
@@ -2108,68 +2132,127 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
 	/* May need to yield even though we do not bump reds here... */
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
 	    goto yield_return;
-	BIF_RET(am_ok); 
+	ERTS_BIF_PREP_RET(retval, am_ok);
 	break;
     case SEND_TRAP:
 	if (connect) {
-	    BIF_TRAP3(dsend3_trap, p, to, msg, opts); 
+	    ERTS_BIF_PREP_TRAP3(retval, dsend3_trap, p, to, msg, opts);
 	} else {
-	    BIF_RET(am_noconnect);
+	    ERTS_BIF_PREP_RET(retval, am_noconnect);
 	}
 	break;
     case SEND_YIELD:
-	if (suspend) {
-	    ERTS_BIF_YIELD3(bif_export[BIF_send_3], p, to, msg, opts);
+	if (ctx->suspend) {
+	    ERTS_BIF_PREP_YIELD3(retval,
+				 bif_export[BIF_send_3], p, to, msg, opts);
 	} else {
-	    BIF_RET(am_nosuspend);
+	    ERTS_BIF_PREP_RET(retval, am_nosuspend);
 	}
 	break;
     case SEND_YIELD_RETURN:
-	if (!suspend)
-	    BIF_RET(am_nosuspend);
+	if (!ctx->suspend) {
+	    ERTS_BIF_PREP_RET(retval, am_nosuspend);
+	    break;
+	}
     yield_return:
-	ERTS_BIF_YIELD_RETURN(p, am_ok);
+	ERTS_BIF_PREP_YIELD_RETURN(retval, p, am_ok);
+        break;
     case SEND_AWAIT_RESULT:
 	ASSERT(is_internal_ref(ref));
-	BIF_TRAP3(await_port_send_result_trap, p, ref, am_nosuspend, am_ok);
+	ERTS_BIF_PREP_TRAP3(retval, await_port_send_result_trap, p, ref, am_nosuspend, am_ok);
+	break;
     case SEND_BADARG:
-	BIF_ERROR(p, BADARG); 
+	ERTS_BIF_PREP_ERROR(retval, p, BADARG);
 	break;
     case SEND_USER_ERROR:
-	BIF_ERROR(p, EXC_ERROR); 
+	ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR);
 	break;
     case SEND_INTERNAL_ERROR:
-	BIF_ERROR(p, EXC_INTERNAL_ERROR);
+	ERTS_BIF_PREP_ERROR(retval, p, EXC_INTERNAL_ERROR);
+	break;
+    case SEND_YIELD_CONTINUE:
+	BUMP_ALL_REDS(p);
+	erts_set_gc_state(p, 0);
+	ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p,
+			    erts_dsend_export_trap_context(p, ctx));
 	break;
     default:
-	ASSERT(! "Illegal send result"); 
+	erl_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result);
 	break;
     }
-    ASSERT(! "Can not arrive here");
-    BIF_ERROR(p, BADARG);
+
+done:
+    UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P);
+    return retval;
 }
 
+HIPE_WRAPPER_BIF_DISABLE_GC(send, 2)
+
 BIF_RETTYPE send_2(BIF_ALIST_2)
 {
     return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2);
 }
 
+static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1)
+{
+    Binary* bin = ((ProcBin*) binary_val(BIF_ARG_1))->val;
+    ErtsSendContext* ctx = (ErtsSendContext*) ERTS_MAGIC_BIN_DATA(bin);
+    Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR);
+    int result;
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dsend_context_dtor);
+
+    ctx->dss.reds = initial_reds;
+    result = erts_dsig_send(&ctx->dsd, &ctx->dss);
+
+    switch (result) {
+    case ERTS_DSIG_SEND_OK:
+	erts_set_gc_state(BIF_P, 1);
+	BIF_RET(ctx->return_term);
+	break;
+    case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/
+	erts_set_gc_state(BIF_P, 1);
+	if (!ctx->suspend)
+	    BIF_RET(am_nosuspend);
+	ERTS_BIF_YIELD_RETURN(BIF_P, ctx->return_term);
+
+    case ERTS_DSIG_SEND_CONTINUE: { /*SEND_YIELD_CONTINUE*/
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1);
+    }
+    default:
+	erl_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result);
+	break;
+    }
+    ASSERT(! "Can not arrive here");
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 Eterm erl_send(Process *p, Eterm to, Eterm msg)
 {
+    Eterm retval;
     Eterm ref;
     Sint result;
+    DeclareTypedTmpHeap(ErtsSendContext, ctx, p);
+    UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p);
 
 #ifdef DEBUG
     ref = NIL;
 #endif
+    ctx->suspend = !0;
+    ctx->dep_to_deref = NULL;
+    ctx->return_term = msg;
+    ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR);
+    ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT;
 
-    result = do_send(p, to, msg, !0, &ref);
+    result = do_send(p, to, msg, &ref, ctx);
     
     if (result > 0) {
 	ERTS_VBUMP_REDS(p, result);
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
 	    goto yield_return;
-	BIF_RET(msg);
+	ERTS_BIF_PREP_RET(retval, msg);
+	goto done;
     }
 
     switch (result) {
@@ -2177,35 +2260,46 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
 	/* May need to yield even though we do not bump reds here... */
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
 	    goto yield_return;
-	BIF_RET(msg); 
+	ERTS_BIF_PREP_RET(retval, msg);
 	break;
     case SEND_TRAP:
-	BIF_TRAP2(dsend2_trap, p, to, msg); 
+	ERTS_BIF_PREP_TRAP2(retval, dsend2_trap, p, to, msg);
 	break;
     case SEND_YIELD:
-	ERTS_BIF_YIELD2(bif_export[BIF_send_2], p, to, msg);
+	ERTS_BIF_PREP_YIELD2(retval, bif_export[BIF_send_2], p, to, msg);
 	break;
     case SEND_YIELD_RETURN:
     yield_return:
-	ERTS_BIF_YIELD_RETURN(p, msg);
+	ERTS_BIF_PREP_YIELD_RETURN(retval, p, msg);
+        break;
     case SEND_AWAIT_RESULT:
 	ASSERT(is_internal_ref(ref));
-	BIF_TRAP3(await_port_send_result_trap, p, ref, msg, msg);
+	ERTS_BIF_PREP_TRAP3(retval,
+			    await_port_send_result_trap, p, ref, msg, msg);
+	break;
     case SEND_BADARG:
-	BIF_ERROR(p, BADARG); 
+	ERTS_BIF_PREP_ERROR(retval, p, BADARG);
 	break;
     case SEND_USER_ERROR:
-	BIF_ERROR(p, EXC_ERROR); 
+	ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR);
 	break;
     case SEND_INTERNAL_ERROR:
-	BIF_ERROR(p, EXC_INTERNAL_ERROR);
+	ERTS_BIF_PREP_ERROR(retval, p, EXC_INTERNAL_ERROR);
+	break;
+    case SEND_YIELD_CONTINUE:
+	BUMP_ALL_REDS(p);
+	erts_set_gc_state(p, 0);
+	ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p,
+			    erts_dsend_export_trap_context(p, ctx));
 	break;
     default:
-	ASSERT(! "Illegal send result"); 
+	erl_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result);
 	break;
     }
-    ASSERT(! "Can not arrive here");
-    BIF_ERROR(p, BADARG);
+
+done:
+    UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p);
+    return retval;
 }
 
 /**********************************************************************/
@@ -4809,6 +4903,10 @@ void erts_init_bif(void)
 #endif
 		     , &bif_return_trap);
 
+    erts_init_trap_export(&dsend_continue_trap_export,
+			  am_erts_internal, am_dsend_continue_trap, 1,
+			  dsend_continue_trap_1);
+
     flush_monitor_message_trap = erts_export_put(am_erlang,
 						 am_flush_monitor_message,
 						 2);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index ec07ddcd9c..f73374ba71 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -119,7 +119,7 @@ Export* dmonitor_p_trap = NULL;
 /* forward declarations */
 
 static void clear_dist_entry(DistEntry*);
-static int dsig_send(ErtsDSigData *, Eterm, Eterm, int);
+static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy);
 static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm);
 static void init_nodes_monitors(void);
 
@@ -709,6 +709,55 @@ static void clear_dist_entry(DistEntry *dep)
     }
 }
 
+void erts_dsend_context_dtor(Binary* ctx_bin)
+{
+    ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
+    switch (ctx->dss.phase) {
+    case ERTS_DSIG_SEND_PHASE_MSG_SIZE:
+	DESTROY_SAVED_ESTACK(&ctx->dss.u.sc.estack);
+	break;
+    case ERTS_DSIG_SEND_PHASE_MSG_ENCODE:
+	DESTROY_SAVED_WSTACK(&ctx->dss.u.ec.wstack);
+	break;
+    default:;
+    }
+    if (ctx->dss.phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->dss.obuf) {
+	free_dist_obuf(ctx->dss.obuf);
+    }
+    if (ctx->dep_to_deref)
+	erts_deref_dist_entry(ctx->dep_to_deref);
+}
+
+Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx)
+{
+    struct exported_ctx {
+	ErtsSendContext ctx;
+	ErtsAtomCacheMap acm;
+    };
+    Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx),
+					       erts_dsend_context_dtor);
+    struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin);
+    Uint ctl_size = !HALFWORD_HEAP ? 0 : (arityval(ctx->ctl_heap[0]) + 1);
+    Eterm* hp = HAlloc(p, ctl_size + PROC_BIN_SIZE);
+
+    sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext));
+    ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap));
+#if !HALFWORD_HEAP
+    dst->ctx.dss.ctl = make_tuple(dst->ctx.ctl_heap);
+#else
+    /* Must put control tuple in low mem */
+    sys_memcpy(hp, ctx->ctl_heap,  ctl_size*sizeof(Eterm));
+    dst->ctx.dss.ctl = make_tuple(hp);
+    hp += ctl_size;
+#endif
+    if (ctx->dss.acmp) {
+	sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap));
+	dst->ctx.dss.acmp = &dst->acm;
+    }
+    return erts_mk_magic_binary_term(&hp, &MSO(p), ctx_bin);
+}
+
+
 /*
  * The erts_dsig_send_*() functions implemented below, sends asynchronous
  * distributed signals to other Erlang nodes. Before sending a distributed
@@ -731,7 +780,7 @@ erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote)
     int res;
     UseTmpHeapNoproc(4);
 
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send_ctl(dsdp, ctl, 0);
     UnUseTmpHeapNoproc(4);
     return res;
 }
@@ -744,7 +793,7 @@ erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote)
     int res;
 
     UseTmpHeapNoproc(4);
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send_ctl(dsdp, ctl, 0);
     UnUseTmpHeapNoproc(4);
     return res;
 }
@@ -772,7 +821,7 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     erts_smp_de_links_unlock(dsdp->dep);
 #endif
 
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res = dsig_send_ctl(dsdp, ctl, 1);
     UnUseTmpHeapNoproc(6);
     return res;
 }
@@ -793,7 +842,7 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
 		 make_small(DOP_MONITOR_P),
 		 watcher, watched, ref);
 
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send_ctl(dsdp, ctl, 0);
     UnUseTmpHeapNoproc(5);
     return res;
 }
@@ -815,18 +864,17 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
 		 make_small(DOP_DEMONITOR_P),
 		 watcher, watched, ref);
 
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, force);
+    res = dsig_send_ctl(dsdp, ctl, force);
     UnUseTmpHeapNoproc(5);
     return res;
 }
 
 int
-erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
+erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
 {
     Eterm ctl;
-    DeclareTmpHeapNoproc(ctl_heap,5);
     Eterm token = NIL;
-    Process *sender = dsdp->proc;
+    Process *sender = ctx->dsd.proc;
     int res;
 #ifdef USE_VM_PROBES
     Sint tok_label = 0;
@@ -838,8 +886,7 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
     DTRACE_CHARBUF(receiver_name, 64);
 #endif
 
-    UseTmpHeapNoproc(5);
-    if (SEQ_TRACE_TOKEN(sender) != NIL 
+    if (SEQ_TRACE_TOKEN(sender) != NIL
 #ifdef USE_VM_PROBES
 	&& SEQ_TRACE_TOKEN(sender) != am_have_dt_utag 
 #endif
@@ -852,7 +899,7 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
     *node_name = *sender_name = *receiver_name = '\0';
     if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) {
         erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)),
-                      "%T", dsdp->dep->sysname);
+                      "%T", ctx->dsd.dep->sysname);
         erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
                       "%T", sender->common.id);
         erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
@@ -867,26 +914,28 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
 #endif
 
     if (token != NIL)
-	ctl = TUPLE4(&ctl_heap[0],
+	ctl = TUPLE4(&ctx->ctl_heap[0],
 		     make_small(DOP_SEND_TT), am_Cookie, remote, token);
     else
-	ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote);
+	ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote);
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
-    res = dsig_send(dsdp, ctl, message, 0);
-    UnUseTmpHeapNoproc(5);
+    ctx->dss.ctl = ctl;
+    ctx->dss.msg = message;
+    ctx->dss.force_busy = 0;
+    res = erts_dsig_send(&ctx->dsd, &ctx->dss);
     return res;
 }
 
 int
-erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
+erts_dsig_send_reg_msg(Eterm remote_name, Eterm message,
+		       ErtsSendContext* ctx)
 {
     Eterm ctl;
-    DeclareTmpHeapNoproc(ctl_heap,6);
     Eterm token = NIL;
-    Process *sender = dsdp->proc;
+    Process *sender = ctx->dsd.proc;
     int res;
 #ifdef USE_VM_PROBES
     Sint tok_label = 0;
@@ -898,7 +947,6 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
     DTRACE_CHARBUF(receiver_name, 128);
 #endif
 
-    UseTmpHeapNoproc(6);
     if (SEQ_TRACE_TOKEN(sender) != NIL
 #ifdef USE_VM_PROBES
 	&& SEQ_TRACE_TOKEN(sender) != am_have_dt_utag 
@@ -912,7 +960,7 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
     *node_name = *sender_name = *receiver_name = '\0';
     if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) {
         erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)),
-                      "%T", dsdp->dep->sysname);
+                      "%T", ctx->dsd.dep->sysname);
         erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
                       "%T", sender->common.id);
         erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
@@ -927,17 +975,19 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
 #endif
 
     if (token != NIL)
-	ctl = TUPLE5(&ctl_heap[0], make_small(DOP_REG_SEND_TT),
+	ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_REG_SEND_TT),
 		     sender->common.id, am_Cookie, remote_name, token);
     else
-	ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND),
+	ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_REG_SEND),
 		     sender->common.id, am_Cookie, remote_name);
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
-    res = dsig_send(dsdp, ctl, message, 0);
-    UnUseTmpHeapNoproc(6);
+    ctx->dss.ctl = ctl;
+    ctx->dss.msg = message;
+    ctx->dss.force_busy = 0;
+    res = erts_dsig_send(&ctx->dsd, &ctx->dss);
     return res;
 }
 
@@ -994,7 +1044,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
     DTRACE7(process_exit_signal_remote, sender_name, node_name,
             remote_name, reason_str, tok_label, tok_lastcnt, tok_serial);
     /* forced, i.e ignore busy */
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res = dsig_send_ctl(dsdp, ctl, 1);
     UnUseTmpHeapNoproc(6);
     return res;
 }
@@ -1010,7 +1060,7 @@ erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason)
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_EXIT), local, remote, reason);
     /* forced, i.e ignore busy */
-    res =  dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res =  dsig_send_ctl(dsdp, ctl, 1);
     UnUseTmpHeapNoproc(5);
     return res;
 }
@@ -1026,7 +1076,7 @@ erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_EXIT2), local, remote, reason);
 
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send_ctl(dsdp, ctl, 0);
     UnUseTmpHeapNoproc(5);
     return res;
 }
@@ -1043,7 +1093,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
     ctl = TUPLE3(&ctl_heap[0],
 		 make_small(DOP_GROUP_LEADER), leader, remote);
 
-    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send_ctl(dsdp, ctl, 0);
     UnUseTmpHeapNoproc(4);
     return res;
 }
@@ -1693,194 +1743,235 @@ int erts_net_message(Port *prt,
     return -1;
 }
 
-static int
-dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
+static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy)
 {
+    struct erts_dsig_send_context ctx;
+    int ret;
+    ctx.ctl = ctl;
+    ctx.msg = THE_NON_VALUE;
+    ctx.force_busy = force_busy;
+    ctx.phase = ERTS_DSIG_SEND_PHASE_INIT;
+#ifdef DEBUG
+    ctx.reds = 1; /* provoke assert below (no reduction count without msg) */
+#endif
+    ret = erts_dsig_send(dsdp, &ctx);
+    ASSERT(ret != ERTS_DSIG_SEND_CONTINUE);
+    return ret;
+}
+
+int
+erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
+{
+    int retval;
+    Sint initial_reds = ctx->reds;
     Eterm cid;
-    int suspended = 0;
-    int resume = 0;
-    Uint32 pass_through_size;
-    Uint data_size, dhdr_ext_size;
-    ErtsAtomCacheMap *acmp;
-    ErtsDistOutputBuf *obuf;
-    DistEntry *dep = dsdp->dep;
-    Uint32 flags = dep->flags;
-    Process *c_p = dsdp->proc;
 
-    if (!c_p || dsdp->no_suspend)
-	force_busy = 1;
+    while (1) {
+	switch (ctx->phase) {
+	case ERTS_DSIG_SEND_PHASE_INIT:
+	    ctx->flags = dsdp->dep->flags;
+	    ctx->c_p = dsdp->proc;
 
-    ERTS_SMP_LC_ASSERT(!c_p
-		       || (ERTS_PROC_LOCK_MAIN
-			   == erts_proc_lc_my_proc_locks(c_p)));
+	    if (!ctx->c_p || dsdp->no_suspend)
+		ctx->force_busy = 1;
 
-    if (!erts_is_alive)
-	return ERTS_DSIG_SEND_OK;
+	    ERTS_SMP_LC_ASSERT(!ctx->c_p
+			       || (ERTS_PROC_LOCK_MAIN
+				   == erts_proc_lc_my_proc_locks(ctx->c_p)));
 
-    if (flags & DFLAG_DIST_HDR_ATOM_CACHE) {
-	acmp = erts_get_atom_cache_map(c_p);
-	pass_through_size = 0;
-    }
-    else {
-	acmp = NULL;
-	pass_through_size = 1;
-    }
+	    if (!erts_is_alive)
+		return ERTS_DSIG_SEND_OK;
 
-#ifdef ERTS_DIST_MSG_DBG
-    erts_fprintf(stderr, ">>%s CTL: %T\n", pass_through_size ? "P" : " ", ctl);
-    if (is_value(msg))
-	erts_fprintf(stderr, "    MSG: %T\n", msg);
-#endif
+	    if (ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE) {
+		ctx->acmp = erts_get_atom_cache_map(ctx->c_p);
+		ctx->pass_through_size = 0;
+	    }
+	    else {
+		ctx->acmp = NULL;
+		ctx->pass_through_size = 1;
+	    }
 
-    data_size = pass_through_size;
-    erts_reset_atom_cache_map(acmp);
-    data_size += erts_encode_dist_ext_size(ctl, flags, acmp);
-    if (is_value(msg))
-	data_size += erts_encode_dist_ext_size(msg, flags, acmp);
-    erts_finalize_atom_cache_map(acmp, flags);
+    #ifdef ERTS_DIST_MSG_DBG
+	    erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl);
+	    if (is_value(msg))
+		erts_fprintf(stderr, "    MSG: %T\n", msg);
+    #endif
+
+	    ctx->data_size = ctx->pass_through_size;
+	    erts_reset_atom_cache_map(ctx->acmp);
+	    erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size);
+
+	    if (is_value(ctx->msg)) {
+		ctx->u.sc.estack.start = NULL;
+		ctx->u.sc.flags = ctx->flags;
+		ctx->u.sc.level = 0;
+		ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE;
+	    } else {
+		ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC;
+	    }
+	    break;
 
-    dhdr_ext_size = erts_encode_ext_dist_header_size(acmp);
-    data_size += dhdr_ext_size;
+	case ERTS_DSIG_SEND_PHASE_MSG_SIZE:
+	    if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) {
+		retval = ERTS_DSIG_SEND_CONTINUE;
+		goto done;
+	    }
 
-    obuf = alloc_dist_obuf(data_size);
-    obuf->ext_endp = &obuf->data[0] + pass_through_size + dhdr_ext_size;
+	    ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC;
+	case ERTS_DSIG_SEND_PHASE_ALLOC:
+	    erts_finalize_atom_cache_map(ctx->acmp, ctx->flags);
+
+	    ctx->dhdr_ext_size = erts_encode_ext_dist_header_size(ctx->acmp);
+	    ctx->data_size += ctx->dhdr_ext_size;
+
+	    ctx->obuf = alloc_dist_obuf(ctx->data_size);
+	    ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->pass_through_size + ctx->dhdr_ext_size;
+
+	    /* Encode internal version of dist header */
+	    ctx->obuf->extp = erts_encode_ext_dist_header_setup(ctx->obuf->ext_endp, ctx->acmp);
+	    /* Encode control message */
+	    erts_encode_dist_ext(ctx->ctl, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, NULL, NULL);
+	    if (is_value(ctx->msg)) {
+		ctx->u.ec.flags = ctx->flags;
+		ctx->u.ec.level = 0;
+		ctx->u.ec.wstack.wstart = NULL;
+		ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE;
+	    } else {
+		ctx->phase = ERTS_DSIG_SEND_PHASE_FIN;
+	    }
+	    break;
 
-    /* Encode internal version of dist header */
-    obuf->extp = erts_encode_ext_dist_header_setup(obuf->ext_endp, acmp);
-    /* Encode control message */
-    erts_encode_dist_ext(ctl, &obuf->ext_endp, flags, acmp);
-    if (is_value(msg)) {
-	/* Encode message */
-	erts_encode_dist_ext(msg, &obuf->ext_endp, flags, acmp);
-    }
+	case ERTS_DSIG_SEND_PHASE_MSG_ENCODE:
+	    if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, &ctx->u.ec, &ctx->reds)) {
+		retval = ERTS_DSIG_SEND_CONTINUE;
+		goto done;
+	    }
 
-    ASSERT(obuf->extp < obuf->ext_endp);
-    ASSERT(&obuf->data[0] <= obuf->extp - pass_through_size);
-    ASSERT(obuf->ext_endp <= &obuf->data[0] + data_size);
+	    ctx->phase = ERTS_DSIG_SEND_PHASE_FIN;
+	case ERTS_DSIG_SEND_PHASE_FIN: {
+	    DistEntry *dep = dsdp->dep;
+	    int suspended = 0;
+	    int resume = 0;
 
-    data_size = obuf->ext_endp - obuf->extp;
+	    ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp);
+	    ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->pass_through_size);
+	    ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size);
 
-    /*
-     * Signal encoded; now verify that the connection still exists,
-     * and if so enqueue the signal and schedule it for send.
-     */
-    obuf->next = NULL;
-    erts_smp_de_rlock(dep);
-    cid = dep->cid;
-    if (cid != dsdp->cid
-	|| dep->connection_id != dsdp->connection_id
-	|| dep->status & ERTS_DE_SFLG_EXITING) {
-	/* Not the same connection as when we started; drop message... */
-	erts_smp_de_runlock(dep);
-	free_dist_obuf(obuf);
-    }
-    else {
-	ErtsProcList *plp = NULL;
-	erts_smp_mtx_lock(&dep->qlock);
-	dep->qsize += size_obuf(obuf);
-	if (dep->qsize >= erts_dist_buf_busy_limit)
-	    dep->qflgs |= ERTS_DE_QFLG_BUSY;
-	if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) {
-	    erts_smp_mtx_unlock(&dep->qlock);
+	    ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp;
 
-	    plp = erts_proclist_create(c_p);
-	    erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
-	    suspended = 1;
-	    erts_smp_mtx_lock(&dep->qlock);
-	}
+	    /*
+	     * Signal encoded; now verify that the connection still exists,
+	     * and if so enqueue the signal and schedule it for send.
+	     */
+	    ctx->obuf->next = NULL;
+	    erts_smp_de_rlock(dep);
+	    cid = dep->cid;
+	    if (cid != dsdp->cid
+		|| dep->connection_id != dsdp->connection_id
+		|| dep->status & ERTS_DE_SFLG_EXITING) {
+		/* Not the same connection as when we started; drop message... */
+		erts_smp_de_runlock(dep);
+		free_dist_obuf(ctx->obuf);
+	    }
+	    else {
+		ErtsProcList *plp = NULL;
+		erts_smp_mtx_lock(&dep->qlock);
+		dep->qsize += size_obuf(ctx->obuf);
+		if (dep->qsize >= erts_dist_buf_busy_limit)
+		    dep->qflgs |= ERTS_DE_QFLG_BUSY;
+		if (!ctx->force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) {
+		    erts_smp_mtx_unlock(&dep->qlock);
+
+		    plp = erts_proclist_create(ctx->c_p);
+		    erts_suspend(ctx->c_p, ERTS_PROC_LOCK_MAIN, NULL);
+		    suspended = 1;
+		    erts_smp_mtx_lock(&dep->qlock);
+		}
 
-	/* Enqueue obuf on dist entry */
-	if (dep->out_queue.last)
-	    dep->out_queue.last->next = obuf;
-	else
-	    dep->out_queue.first = obuf;
-	dep->out_queue.last = obuf;
+		/* Enqueue obuf on dist entry */
+		if (dep->out_queue.last)
+		    dep->out_queue.last->next = ctx->obuf;
+		else
+		    dep->out_queue.first = ctx->obuf;
+		dep->out_queue.last = ctx->obuf;
+
+		if (!ctx->force_busy) {
+		    if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) {
+			if (suspended)
+			    resume = 1; /* was busy when we started, but isn't now */
+    #ifdef USE_VM_PROBES
+			if (resume && DTRACE_ENABLED(dist_port_not_busy)) {
+			    DTRACE_CHARBUF(port_str, 64);
+			    DTRACE_CHARBUF(remote_str, 64);
+
+			    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+					  "%T", cid);
+			    erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
+					  "%T", dep->sysname);
+			    DTRACE3(dist_port_not_busy, erts_this_node_sysname,
+				    port_str, remote_str);
+			}
+    #endif
+		    }
+		    else {
+			/* Enqueue suspended process on dist entry */
+			ASSERT(plp);
+			erts_proclist_store_last(&dep->suspended, plp);
+		    }
+		}
 
-	if (!force_busy) {
-	    if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) {
-		if (suspended)
-		    resume = 1; /* was busy when we started, but isn't now */
-#ifdef USE_VM_PROBES
-                if (resume && DTRACE_ENABLED(dist_port_not_busy)) {
-                    DTRACE_CHARBUF(port_str, 64);
-                    DTRACE_CHARBUF(remote_str, 64);
-
-                    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
-                                  "%T", cid);
-                    erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
-                                  "%T", dep->sysname);
-                    DTRACE3(dist_port_not_busy, erts_this_node_sysname,
-                            port_str, remote_str);
-                }
-#endif
+		erts_smp_mtx_unlock(&dep->qlock);
+		erts_schedule_dist_command(NULL, dep);
+		erts_smp_de_runlock(dep);
+
+		if (resume) {
+		    erts_resume(ctx->c_p, ERTS_PROC_LOCK_MAIN);
+		    erts_proclist_destroy(plp);
+		    /*
+		     * Note that the calling process still have to yield as if it
+		     * suspended. If not, the calling process could later be
+		     * erroneously scheduled when it shouldn't be.
+		     */
+		}
 	    }
-	    else {
-		/* Enqueue suspended process on dist entry */
-		ASSERT(plp);
-		erts_proclist_store_last(&dep->suspended, plp);
+	    ctx->obuf = NULL;
+
+	    if (suspended) {
+    #ifdef USE_VM_PROBES
+		if (!resume && DTRACE_ENABLED(dist_port_busy)) {
+		    DTRACE_CHARBUF(port_str, 64);
+		    DTRACE_CHARBUF(remote_str, 64);
+		    DTRACE_CHARBUF(pid_str, 16);
+
+		    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid);
+		    erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
+				  "%T", dep->sysname);
+		    erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)),
+				  "%T", ctx->c_p->common.id);
+		    DTRACE4(dist_port_busy, erts_this_node_sysname,
+			    port_str, remote_str, pid_str);
+		}
+    #endif
+		if (!resume && erts_system_monitor_flags.busy_dist_port)
+		    monitor_generic(ctx->c_p, am_busy_dist_port, cid);
+		retval = ERTS_DSIG_SEND_YIELD;
+	    } else {
+		retval = ERTS_DSIG_SEND_OK;
 	    }
+	    goto done;
 	}
-
-	erts_smp_mtx_unlock(&dep->qlock);
-	erts_schedule_dist_command(NULL, dep);
-	erts_smp_de_runlock(dep);
-	
-	if (resume) {
-	    erts_resume(c_p, ERTS_PROC_LOCK_MAIN);
-	    erts_proclist_destroy(plp);
-	    /*
-	     * Note that the calling process still have to yield as if it
-	     * suspended. If not, the calling process could later be
-	     * erroneously scheduled when it shouldn't be.
-	     */
+	default:
+	    erl_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase);
 	}
     }
 
-    if (c_p) {
-	int reds;
-	/* 
-	 * Bump reductions on calling process.
-	 *
-	 * This is the reduction cost: Always a base cost of 8 reductions
-	 * plus 16 reductions per kilobyte generated external data.
-	 */
-
-	data_size >>= (10-4);
-#if defined(ARCH_64) && !HALFWORD_HEAP
-	data_size &= 0x003fffffffffffff;
-#elif defined(ARCH_32) || HALFWORD_HEAP
-	data_size &= 0x003fffff;
-#else
-#       error "Ohh come on ... !?!"
-#endif
-	reds = 8 + ((int) data_size > 1000000 ? 1000000 : (int) data_size);
-	BUMP_REDS(c_p, reds);
-    }
-
-    if (suspended) {
-#ifdef USE_VM_PROBES
-        if (!resume && DTRACE_ENABLED(dist_port_busy)) {
-            DTRACE_CHARBUF(port_str, 64);
-            DTRACE_CHARBUF(remote_str, 64);
-            DTRACE_CHARBUF(pid_str, 16);
-
-            erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid);
-            erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
-                          "%T", dep->sysname);
-            erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)),
-                          "%T", c_p->common.id);
-            DTRACE4(dist_port_busy, erts_this_node_sysname,
-                    port_str, remote_str, pid_str);
-        }
-#endif
-	if (!resume && erts_system_monitor_flags.busy_dist_port)
-	    monitor_generic(c_p, am_busy_dist_port, cid);
-	return ERTS_DSIG_SEND_YIELD;
+done:
+    if (ctx->msg && ctx->c_p) {
+	BUMP_REDS(ctx->c_p, (initial_reds - ctx->reds) / TERM_TO_BINARY_LOOP_FACTOR);
     }
-    return ERTS_DSIG_SEND_OK;
+    return retval;
 }
 
-
 static Uint
 dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
 {
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index f32b999198..2a2ba0c83f 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -22,6 +22,7 @@
 
 #include "erl_process.h"
 #include "erl_node_tables.h"
+#include "zlib.h"
 
 #define DFLAG_PUBLISHED           0x01
 #define DFLAG_ATOM_CACHE          0x02
@@ -264,17 +265,105 @@ erts_destroy_dist_link(ErtsDistLinkData *dldp)
 
 #endif
 
+
+
+/* Define for testing */
+/* #define EXTREME_TTB_TRAPPING 1 */
+
+#ifndef EXTREME_TTB_TRAPPING
+#define TERM_TO_BINARY_LOOP_FACTOR 32
+#else
+#define TERM_TO_BINARY_LOOP_FACTOR 1
+#endif
+
+typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState;
+typedef struct TTBSizeContext_ {
+    Uint flags;
+    int level;
+    Uint result;
+    Eterm obj;
+    ErtsEStack estack;
+} TTBSizeContext;
+
+typedef struct TTBEncodeContext_ {
+    Uint flags;
+    int level;
+    byte* ep;
+    Eterm obj;
+    ErtsWStack wstack;
+    Binary *result_bin;
+} TTBEncodeContext;
+
+typedef struct {
+    Uint real_size;
+    Uint dest_len;
+    byte *dbytes;
+    Binary *result_bin;
+    Binary *destination_bin;
+    z_stream stream;
+} TTBCompressContext;
+
+typedef struct {
+    int alive;
+    TTBState state;
+    union {
+	TTBSizeContext sc;
+	TTBEncodeContext ec;
+	TTBCompressContext cc;
+    } s;
+} TTBContext;
+
+enum erts_dsig_send_phase {
+    ERTS_DSIG_SEND_PHASE_INIT,
+    ERTS_DSIG_SEND_PHASE_MSG_SIZE,
+    ERTS_DSIG_SEND_PHASE_ALLOC,
+    ERTS_DSIG_SEND_PHASE_MSG_ENCODE,
+    ERTS_DSIG_SEND_PHASE_FIN
+};
+
+struct erts_dsig_send_context {
+    enum erts_dsig_send_phase phase;
+    Sint reds;
+
+    Eterm ctl;
+    Eterm msg;
+    int force_busy;
+    Uint32 pass_through_size;
+    Uint data_size, dhdr_ext_size;
+    ErtsAtomCacheMap *acmp;
+    ErtsDistOutputBuf *obuf;
+    Uint32 flags;
+    Process *c_p;
+    union {
+	TTBSizeContext sc;
+	TTBEncodeContext ec;
+    }u;
+};
+
+typedef struct {
+    int suspend;
+
+    Eterm ctl_heap[6];
+    ErtsDSigData dsd;
+    DistEntry* dep_to_deref;
+    struct erts_dsig_send_context dss;
+
+    Eterm return_term;
+}ErtsSendContext;
+
+
 /*
  * erts_dsig_send_* return values.
  */
 #define ERTS_DSIG_SEND_OK	0
 #define ERTS_DSIG_SEND_YIELD	1
+#define ERTS_DSIG_SEND_CONTINUE 2
 
 extern int erts_dsig_send_link(ErtsDSigData *, Eterm, Eterm);
-extern int erts_dsig_send_msg(ErtsDSigData *, Eterm, Eterm);
+extern int erts_dsig_send_msg(Eterm, Eterm, ErtsSendContext*);
 extern int erts_dsig_send_exit_tt(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm);
 extern int erts_dsig_send_unlink(ErtsDSigData *, Eterm, Eterm);
-extern int erts_dsig_send_reg_msg(ErtsDSigData *, Eterm, Eterm);
+extern int erts_dsig_send_reg_msg(Eterm, Eterm, ErtsSendContext*);
 extern int erts_dsig_send_group_leader(ErtsDSigData *, Eterm, Eterm);
 extern int erts_dsig_send_exit(ErtsDSigData *, Eterm, Eterm, Eterm);
 extern int erts_dsig_send_exit2(ErtsDSigData *, Eterm, Eterm, Eterm);
@@ -282,6 +371,10 @@ extern int erts_dsig_send_demonitor(ErtsDSigData *, Eterm, Eterm, Eterm, int);
 extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm);
 extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm);
 
+extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx);
+extern void erts_dsend_context_dtor(Binary*);
+extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx);
+
 extern int erts_dist_command(Port *prt, int reds);
 extern void erts_dist_port_not_busy(Port *prt);
 extern void erts_kill_dist_connection(DistEntry *dep, Uint32);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 8d240355b0..564657a13b 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -498,15 +498,37 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint
     return ep;
 }
 
-Uint erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp)
+int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp,
+			      Uint* szp)
 {
-    Uint sz = 0;
+    Uint sz;
+    if (encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz)) {
+	return -1;
+    } else {
 #ifndef ERTS_DEBUG_USE_DIST_SEP
-    if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+	if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
 #endif
-	sz++ /* VERSION_MAGIC */;
-    sz += encode_size_struct2(acmp, term, flags);
-    return sz;
+	    sz++ /* VERSION_MAGIC */;
+
+	*szp += sz;
+	return 0;
+    }
+}
+
+int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp)
+{
+    Uint sz;
+    if (encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, ctx->flags, &ctx->reds, &sz)) {
+	return -1;
+    } else {
+#ifndef ERTS_DEBUG_USE_DIST_SEP
+	if (!(ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE))
+#endif
+	    sz++ /* VERSION_MAGIC */;
+
+	*szp += sz;
+	return 0;
+    }
 }
 
 Uint erts_encode_ext_size(Eterm term)
@@ -527,19 +549,16 @@ Uint erts_encode_ext_size_ets(Eterm term)
 }
 
 
-void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp)
+int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp,
+			  TTBEncodeContext* ctx, Sint* reds)
 {
-    byte *ep = *ext;
-#ifndef ERTS_DEBUG_USE_DIST_SEP
-    if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
-#endif
-	*ep++ = VERSION_MAGIC;
-    ep = enc_term(acmp, term, ep, flags, NULL);
-    if (!ep)
-	erl_exit(ERTS_ABORT_EXIT,
-		 "%s:%d:erts_encode_dist_ext(): Internal data structure error\n",
-		 __FILE__, __LINE__);
-    *ext = ep;
+    if (!ctx || !ctx->wstack.wstart) {
+    #ifndef ERTS_DEBUG_USE_DIST_SEP
+	if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+    #endif
+	    *(*ext)++ = VERSION_MAGIC;
+    }
+    return enc_term_int(ctx, acmp, term, *ext, flags, NULL, reds, ext);
 }
 
 void erts_encode_ext(Eterm term, byte **ext)
@@ -1740,55 +1759,15 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
     return erts_term_to_binary_simple(p, Term, size, level, flags);
 }
 
-/* Define for testing */
-/* #define EXTREME_TTB_TRAPPING 1 */
+/* Define EXTREME_TTB_TRAPPING for testing in dist.h */
 
 #ifndef EXTREME_TTB_TRAPPING
-#define TERM_TO_BINARY_LOOP_FACTOR 32
 #define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18)
 #else
-#define TERM_TO_BINARY_LOOP_FACTOR 1
 #define TERM_TO_BINARY_COMPRESS_CHUNK 10
 #endif
 
 
-typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState;
-typedef struct TTBSizeContext_ {
-    Uint flags;
-    int level;
-    Uint result;
-    Eterm obj;
-    ErtsEStack estack;
-} TTBSizeContext;
-
-typedef struct TTBEncodeContext_ {
-    Uint flags;
-    int level;
-    byte* ep;
-    Eterm obj;
-    ErtsWStack wstack;
-    Binary *result_bin;
-} TTBEncodeContext;
-
-typedef struct {
-    Uint real_size;
-    Uint dest_len;
-    byte *dbytes;
-    Binary *result_bin;
-    Binary *destination_bin;
-    z_stream stream;
-} TTBCompressContext;
-
-typedef struct {
-    int alive;
-    TTBState state;
-    union {
-	TTBSizeContext sc;
-	TTBEncodeContext ec;
-	TTBCompressContext cc;
-    } s;
-} TTBContext;
-
 static void ttb_context_destructor(Binary *context_bin)
 {
     TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index bf00958eb1..f120e96e3b 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -150,6 +150,7 @@ typedef struct {
     Uint extsize;
 } ErtsBinary2TermState;
 
+
 /* -------------------------------------------------------------------------- */
 
 void erts_init_atom_cache_map(ErtsAtomCacheMap *);
@@ -161,8 +162,12 @@ Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
 Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
 byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *);
 byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32);
-Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *);
-void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *);
+struct erts_dsig_send_context;
+int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp);
+int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp);
+struct TTBEncodeContext_;
+int erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *,
+			  struct TTBEncodeContext_ *, Sint* reds);
 
 Uint erts_encode_ext_size(Eterm);
 Uint erts_encode_ext_size_2(Eterm, unsigned);
-- 
cgit v1.2.3


From ac2eba5ea214152d806a73a8c992911cdf3aae1d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 Aug 2014 17:48:42 +0200
Subject: erts: Yield in term_to_binary while copying large binaries

Applies also to distributed send.
---
 erts/emulator/beam/external.c | 45 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 564657a13b..c6e6a8a954 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2304,8 +2304,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 #define ENC_TERM ((Eterm) 0)
 #define ENC_ONE_CONS ((Eterm) 1)
 #define ENC_PATCH_FUN_SIZE ((Eterm) 2)
-#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3)
-
+#define ENC_BIN_COPY ((Eterm) 3)
+#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 4)
 
 static byte*
 enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
@@ -2341,6 +2341,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    WSTACK_RESTORE(s, &ctx->wstack);
 	    ep = ctx->ep;
 	    obj = ctx->obj;
+	    if (is_non_value(obj)) {
+		goto outer_loop;
+	    }
 	}
     }
 
@@ -2378,6 +2381,26 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		put_int32(ep - size_p, size_p);
 	    }
 	    goto outer_loop;
+	case ENC_BIN_COPY: {
+	    Uint bits = (Uint)obj;
+	    Uint bitoffs = WSTACK_POP(s);
+	    byte* bytes = (byte*) WSTACK_POP(s);
+	    byte* dst = (byte*) WSTACK_POP(s);
+	    if (bits > r * (B2T_MEMCPY_FACTOR * 8)) {
+		Uint n = r * B2T_MEMCPY_FACTOR;
+		WSTACK_PUSH3(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs);
+		WSTACK_PUSH2(s, ENC_BIN_COPY, bits - 8*n);
+		bits = 8*n;
+		copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits);
+		obj = THE_NON_VALUE;
+		r = 0; /* yield */
+		break;
+	    } else {
+		copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits);
+		r -= bits / (B2T_MEMCPY_FACTOR * 8);
+		goto outer_loop;
+	    }
+	}
 	case ENC_LAST_ARRAY_ELEMENT:
 	    /* obj is the tuple */
 	    {
@@ -2405,8 +2428,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 
     L_jump_start:
 
-	if (ctx && --r == 0) {
-	    *reds = r;
+	if (ctx && --r <= 0) {
+	    *reds = 0;
 	    ctx->obj = obj;
 	    ctx->ep = ep;
 	    WSTACK_SAVE(s, &ctx->wstack);
@@ -2621,6 +2644,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		Uint bitoffs;
 		Uint bitsize;
 		byte* bytes;
+		byte* data_dst;
 
 		ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize);
 		if (dflags & DFLAG_INTERNAL_TAGS) {
@@ -2666,7 +2690,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    j = binary_size(obj);
 		    put_int32(j, ep);
 		    ep += 4;
-		    copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j);
+		    data_dst = ep;
 		    ep += j;
 		} else if (dflags & DFLAG_BIT_BINARIES) {
 		    /* Bit-level binary. */
@@ -2676,7 +2700,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    ep += 4;
 		    *ep++ = bitsize;
 		    ep[j] = 0;	/* Zero unused bits at end of binary */
-		    copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize);
+		    data_dst = ep;
 		    ep += j + 1;
 		} else {
 		    /*
@@ -2690,11 +2714,18 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    put_int32((j+1), ep);
 		    ep += 4;
 		    ep[j] = 0;	/* Zero unused bits at end of binary */
-		    copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize);
+		    data_dst = ep;
 		    ep += j+1;
 		    *ep++ = SMALL_INTEGER_EXT;
 		    *ep++ = bitsize;
 		}
+		if (ctx && j > r * B2T_MEMCPY_FACTOR) {
+		    WSTACK_PUSH3(s, (UWord)data_dst, (UWord)bytes, bitoffs);
+		    WSTACK_PUSH2(s, ENC_BIN_COPY, 8*j + bitsize);
+		} else {
+		    copy_binary_to_buffer(data_dst, 0, bytes, bitoffs,
+					  8 * j + bitsize);
+		}
 	    }
 	    break;
 	case EXPORT_DEF:
-- 
cgit v1.2.3


From fb25e93d290138251d0305bd2698ad29575c661c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 Aug 2014 20:17:07 +0200
Subject: erts: Remove unnecessary goto for fun encoding

---
 erts/emulator/beam/external.c | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c6e6a8a954..914214c5b5 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2754,10 +2754,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	case FUN_DEF:
 	    {
 		ErlFunThing* funp = (ErlFunThing *) fun_val(obj);
+		int ei;
 
 		if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) {
-		    int ei;
-
 		    *ep++ = NEW_FUN_EXT;
 		    WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
 		    WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */
@@ -2774,16 +2773,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap);
 		    ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap);
 		    ep = enc_pid(acmp, funp->creator, ep, dflags);
-
-		fun_env:
-		    for (ei = funp->num_free-1; ei > 0; ei--) {
-			WSTACK_PUSH(s, ENC_TERM);
-			WSTACK_PUSH(s, (UWord) funp->env[ei]);
-		    }
-		    if (funp->num_free != 0) {
-			obj = funp->env[0];
-			goto L_jump_start;
-		    }
 		} else {
 		    /*
 		     * Communicating with an obsolete erl_interface or
@@ -2815,7 +2804,14 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    *ep++ = SMALL_TUPLE_EXT;
 		    put_int8(funp->num_free, ep);
 		    ep += 1;
-		    goto fun_env;
+		}
+		for (ei = funp->num_free-1; ei > 0; ei--) {
+		    WSTACK_PUSH(s, ENC_TERM);
+		    WSTACK_PUSH(s, (UWord) funp->env[ei]);
+		}
+		if (funp->num_free != 0) {
+		    obj = funp->env[0];
+		    goto L_jump_start;
 		}
 	    }
 	    break;
-- 
cgit v1.2.3


From c1676f09ea1176116b503de88604816a2da36ac5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 19 Aug 2014 16:40:52 +0200
Subject: erts: Yield in term_to_binary when encoding big maps

---
 erts/emulator/beam/external.c | 38 +++++++++++++++++++++-----------------
 erts/emulator/beam/global.h   | 22 ++++++++++++++++++++++
 2 files changed, 43 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 914214c5b5..fb2fe146f3 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2305,7 +2305,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 #define ENC_ONE_CONS ((Eterm) 1)
 #define ENC_PATCH_FUN_SIZE ((Eterm) 2)
 #define ENC_BIN_COPY ((Eterm) 3)
-#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 4)
+#define ENC_MAP_PAIR ((Eterm) 4)
+#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 5)
 
 static byte*
 enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
@@ -2388,8 +2389,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    byte* dst = (byte*) WSTACK_POP(s);
 	    if (bits > r * (B2T_MEMCPY_FACTOR * 8)) {
 		Uint n = r * B2T_MEMCPY_FACTOR;
-		WSTACK_PUSH3(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs);
-		WSTACK_PUSH2(s, ENC_BIN_COPY, bits - 8*n);
+		WSTACK_PUSH5(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs,
+			     ENC_BIN_COPY, bits - 8*n);
 		bits = 8*n;
 		copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits);
 		obj = THE_NON_VALUE;
@@ -2401,6 +2402,19 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		goto outer_loop;
 	    }
 	}
+	case ENC_MAP_PAIR: {
+	    Uint pairs_left = obj;
+	    Eterm *vptr = (Eterm*) WSTACK_POP(s);
+	    Eterm *kptr = (Eterm*) WSTACK_POP(s);
+
+	    obj = *kptr;
+	    if (--pairs_left > 0) {
+		WSTACK_PUSH4(s, (UWord)(kptr+1), (UWord)(vptr+1),
+			     ENC_MAP_PAIR, pairs_left);
+	    }
+	    WSTACK_PUSH2(s, ENC_TERM, *vptr);
+	    break;
+	}
 	case ENC_LAST_ARRAY_ELEMENT:
 	    /* obj is the tuple */
 	    {
@@ -2595,18 +2609,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    Eterm *kptr = map_get_keys(mp);
 		    Eterm *vptr = map_get_values(mp);
 
-		    for (i = size-1; i >= 1; i--) {
-			WSTACK_PUSH(s, ENC_TERM);
-			WSTACK_PUSH(s, (UWord) vptr[i]);
-			WSTACK_PUSH(s, ENC_TERM);
-			WSTACK_PUSH(s, (UWord) kptr[i]);
-		    }
-
-		    WSTACK_PUSH(s, ENC_TERM);
-		    WSTACK_PUSH(s, (UWord) vptr[0]);
-
-		    obj = kptr[0];
-		    goto L_jump_start;
+		    WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr,
+				 ENC_MAP_PAIR, size);
 		}
 	    }
 	    break;
@@ -2720,8 +2724,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    *ep++ = bitsize;
 		}
 		if (ctx && j > r * B2T_MEMCPY_FACTOR) {
-		    WSTACK_PUSH3(s, (UWord)data_dst, (UWord)bytes, bitoffs);
-		    WSTACK_PUSH2(s, ENC_BIN_COPY, 8*j + bitsize);
+		    WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs,
+				 ENC_BIN_COPY, 8*j + bitsize);
 		} else {
 		    copy_binary_to_buffer(data_dst, 0, bytes, bitoffs,
 					  8 * j + bitsize);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 891046a8b5..02b8f399b5 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -597,6 +597,28 @@ do {						\
     *s.wsp++ = (z);				\
 } while(0)
 
+#define WSTACK_PUSH4(s, A1, A2, A3, A4)		\
+do {						\
+    if (s.wsp > s.wend - 4) {	\
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+    }						\
+    *s.wsp++ = (A1);				\
+    *s.wsp++ = (A2);				\
+    *s.wsp++ = (A3);				\
+    *s.wsp++ = (A4);				\
+} while(0)
+
+#define WSTACK_PUSH5(s, A1, A2, A3, A4, A5)     \
+do {						\
+    if (s.wsp > s.wend - 5) {	                \
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+    }						\
+    *s.wsp++ = (A1);				\
+    *s.wsp++ = (A2);				\
+    *s.wsp++ = (A3);				\
+    *s.wsp++ = (A4);				\
+    *s.wsp++ = (A5);				\
+} while(0)
 #define WSTACK_COUNT(s) (s.wsp - s.wstart)
 #define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
 #define WSTACK_POP(s) (*(--s.wsp))
-- 
cgit v1.2.3


From 5d66bc37e4583074abd00cb05f9a377f39e2081f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 19 Aug 2014 16:45:45 +0200
Subject: erts: Optimize some repeated calls to {E,W}STACK_PUSH

---
 erts/emulator/beam/erl_printf_term.c | 32 ++++++++------------------------
 erts/emulator/beam/external.c        | 17 +++++++----------
 erts/emulator/beam/global.h          | 25 +++++++++++++++++++++++++
 erts/emulator/beam/utils.c           | 11 +++--------
 4 files changed, 43 insertions(+), 42 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index d18760dc43..31acfdc6d6 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -283,13 +283,9 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		    tl = CDR(cons);
 		    if (is_not_nil(tl)) {
 			if (is_list(tl)) {
-			    WSTACK_PUSH(s, tl);
-			    WSTACK_PUSH(s, PRT_ONE_CONS);
-			    WSTACK_PUSH(s, PRT_COMMA);
+			    WSTACK_PUSH3(s, tl, PRT_ONE_CONS, PRT_COMMA);
 			} else {
-			    WSTACK_PUSH(s, tl);
-			    WSTACK_PUSH(s, PRT_TERM);
-			    WSTACK_PUSH(s, PRT_BAR);
+			    WSTACK_PUSH3(s, tl, PRT_TERM, PRT_BAR);
 			}
 		    }
 		}
@@ -299,9 +295,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		break;
 	    default:		/* PRT_LAST_ARRAY_ELEMENT+1 and upwards */
 		obj = *popped.ptr;
-	        WSTACK_PUSH(s, (UWord) (popped.ptr + 1));
-		WSTACK_PUSH(s, val-1);
-		WSTACK_PUSH(s, PRT_COMMA);
+	        WSTACK_PUSH3(s, (UWord) (popped.ptr + 1), val-1, PRT_COMMA);
 		break;
 	    }
 	    break;
@@ -431,8 +425,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
 	    ++nobj;
 	    if (i > 0) {
-		WSTACK_PUSH(s, (UWord) nobj);
-		WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1);
+		WSTACK_PUSH2(s, (UWord) nobj, PRT_LAST_ARRAY_ELEMENT+i-1);
 	    }
 	    break;
 	case FLOAT_DEF: {
@@ -502,19 +495,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		WSTACK_PUSH(s, PRT_CLOSE_TUPLE);
 		if (n > 0) {
 		    n--;
-		    WSTACK_PUSH(s, vs[n]);
-		    WSTACK_PUSH(s, PRT_TERM);
-		    WSTACK_PUSH(s, PRT_ASSOC);
-		    WSTACK_PUSH(s, ks[n]);
-		    WSTACK_PUSH(s, PRT_TERM);
-
+		    WSTACK_PUSH5(s, vs[n], PRT_TERM, PRT_ASSOC, ks[n], PRT_TERM);
 		    while (n--) {
-			WSTACK_PUSH(s, PRT_COMMA);
-			WSTACK_PUSH(s, vs[n]);
-			WSTACK_PUSH(s, PRT_TERM);
-			WSTACK_PUSH(s, PRT_ASSOC);
-			WSTACK_PUSH(s, ks[n]);
-			WSTACK_PUSH(s, PRT_TERM);
+			WSTACK_PUSH6(s, PRT_COMMA, vs[n], PRT_TERM, PRT_ASSOC,
+				     ks[n], PRT_TERM);
 		    }
 		}
 	    }
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index fb2fe146f3..5e1f5c56e1 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2368,8 +2368,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 
 		obj = CAR(cons);
 		tl = CDR(cons);
-		WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
-		WSTACK_PUSH(s, tl);
+		WSTACK_PUSH2(s, (is_list(tl) ? ENC_ONE_CONS : ENC_TERM),
+			     tl);
 	    }
 	    break;
 	case ENC_PATCH_FUN_SIZE:
@@ -2433,9 +2433,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 #else
 		Eterm* ptr = (Eterm *) obj;
 #endif
-		WSTACK_PUSH(s, val-1);
 		obj = *ptr++;
-		WSTACK_PUSH(s, (UWord)ptr);
+		WSTACK_PUSH2(s, val-1, (UWord)ptr);
 	    }
 	    break;
 	}
@@ -2592,8 +2591,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		ep += 4;
 	    }
 	    if (i > 0) {
-		WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1);
-		WSTACK_PUSH(s, (UWord)ptr);
+		WSTACK_PUSH2(s, ENC_LAST_ARRAY_ELEMENT+i-1, (UWord)ptr);
 	    }
 	    break;
 
@@ -2762,8 +2760,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 
 		if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) {
 		    *ep++ = NEW_FUN_EXT;
-		    WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
-		    WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */
+		    WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE,
+				 (UWord) ep); /* Position for patching in size */
 		    ep += 4;
 		    *ep = funp->arity;
 		    ep += 1;
@@ -2810,8 +2808,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    ep += 1;
 		}
 		for (ei = funp->num_free-1; ei > 0; ei--) {
-		    WSTACK_PUSH(s, ENC_TERM);
-		    WSTACK_PUSH(s, (UWord) funp->env[ei]);
+		    WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]);
 		}
 		if (funp->num_free != 0) {
 		    obj = funp->env[0];
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 02b8f399b5..7f1bd36b28 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -479,6 +479,17 @@ do {						\
     *s.sp++ = (z);				\
 } while(0)
 
+#define ESTACK_PUSH4(s, E1, E2, E3, E4)		\
+do {						\
+    if (s.sp > s.end - 4) {			\
+	erl_grow_estack(&s, ESTK_DEF_STACK(s)); \
+    }						\
+    *s.sp++ = (E1);				\
+    *s.sp++ = (E2);				\
+    *s.sp++ = (E3);				\
+    *s.sp++ = (E4);				\
+} while(0)
+
 #define ESTACK_COUNT(s) (s.sp - s.start)
 #define ESTACK_ISEMPTY(s) (s.sp == s.start)
 #define ESTACK_POP(s) (*(--s.sp))
@@ -619,6 +630,20 @@ do {						\
     *s.wsp++ = (A4);				\
     *s.wsp++ = (A5);				\
 } while(0)
+
+#define WSTACK_PUSH6(s, A1, A2, A3, A4, A5, A6) \
+do {						\
+    if (s.wsp > s.wend - 6) {	                \
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+    }						\
+    *s.wsp++ = (A1);				\
+    *s.wsp++ = (A2);				\
+    *s.wsp++ = (A3);				\
+    *s.wsp++ = (A4);				\
+    *s.wsp++ = (A5);				\
+    *s.wsp++ = (A6);				\
+} while(0)
+
 #define WSTACK_COUNT(s) (s.wsp - s.wstart)
 #define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
 #define WSTACK_POP(s) (*(--s.wsp))
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 72092ec7b0..cd62394987 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1228,25 +1228,20 @@ make_hash2(Eterm term)
 		if (size == 0) {
 		    goto hash2_common;
 		}
-		ESTACK_PUSH(s, hash_xor_values);
-		ESTACK_PUSH(s, hash_xor_keys);
-		ESTACK_PUSH(s, hash);
-		ESTACK_PUSH(s, HASH_MAP_TAIL);
+		ESTACK_PUSH4(s, hash_xor_values, hash_xor_keys, hash, HASH_MAP_TAIL);
 		hash = 0;
 		hash_xor_keys = 0;
 		hash_xor_values = 0;
 		for (i = size - 1; i >= 0; i--) {
 		    tmp = vs[i];
-		    ESTACK_PUSH(s, HASH_MAP_VAL);
-		    ESTACK_PUSH(s, tmp);
+		    ESTACK_PUSH2(s, HASH_MAP_VAL, tmp);
 		}
 		/* We do not want to expose the tuple representation.
 		 * Do not push the keys as a tuple.
 		 */
 		for (i = size - 1; i >= 0; i--) {
 		    tmp = ks[i];
-		    ESTACK_PUSH(s, HASH_MAP_KEY);
-		    ESTACK_PUSH(s, tmp);
+		    ESTACK_PUSH2(s, HASH_MAP_KEY, tmp);
 		}
 		goto hash2_common;
 	    }
-- 
cgit v1.2.3


From 05f748bfdaa4297ae45264ae9297cadb5f33966e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 2 Sep 2014 21:16:58 +0200
Subject: erts: Fix bug in term_to_binary that reallocates binary with wrong
 size

---
 erts/emulator/beam/external.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 8d240355b0..9b9b4b2a62 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1925,6 +1925,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		}
 		real_size = endp - bytes;
 		result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size);
+		result_bin->orig_size = real_size;
 		level = context->s.ec.level;
 		BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR);
 		if (level == 0 || real_size < 6) { /* We are done */
@@ -2004,6 +2005,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 			erl_zlib_deflate_finish(&(context->s.cc.stream));
 			result_bin = erts_bin_realloc(context->s.cc.destination_bin,
 						      context->s.cc.dest_len+6);
+			result_bin->orig_size = context->s.cc.dest_len+6;
 			context->s.cc.destination_bin = NULL;
 			pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
 			pb->thing_word = HEADER_PROC_BIN;
-- 
cgit v1.2.3


From b182aa2cfab793e566c92183c361e78f4d6f84cb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 3 Sep 2014 17:13:22 +0200
Subject: erts: Fix bug with enif_make_copy reallocating writable binary

that could invalidate a pointer received from an earlier
call to enif_inspect_binary.

Solution: Emasculate writable binary at enif_inspect_binary. There are
room for optimizations here as we now do an unconditional emasculation
even though enif_make_copy is not called later in the NIF.
---
 erts/emulator/beam/erl_nif.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ff551ea3af..8c4bf73dbe 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -472,6 +472,18 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
 	struct enif_tmp_obj_t* tmp;
 	byte* raw_ptr;
     }u;
+
+    if (is_boxed(bin_term) && *binary_val(bin_term) == HEADER_SUB_BIN) {
+	ErlSubBin* sb = (ErlSubBin*) binary_val(bin_term);
+	if (sb->is_writable) {
+	    ProcBin* pb = (ProcBin*) binary_val(sb->orig);
+	    ASSERT(pb->thing_word == HEADER_PROC_BIN);
+	    if (pb->flags) {
+		erts_emasculate_writable_binary(pb);
+		sb->is_writable = 0;
+	    }
+	}
+    }
     u.tmp = NULL;
     bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator,
 						    sizeof(struct enif_tmp_obj_t));
-- 
cgit v1.2.3


From 21e4cf5b378ed378296a544d0ad8da08cf95700e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 3 Sep 2014 20:22:42 +0200
Subject: erts: Correct conversion of MIN_SMALL numeral to fixnum

list_to_integer and binary_to_integer returned un-normalized bignum for
-134217728 on 32-bit and -576460752303423488 on 64-bit.

Thanks to Jesper Louis Andersen, Mikael Pettersson
and Anthony Ramine for report, initial patch and optimization suggestion.
---
 erts/emulator/beam/bif.c | 13 +++++++------
 erts/emulator/beam/big.c | 11 ++++++-----
 2 files changed, 13 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index fcbeb6cf5c..b835946729 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -2887,9 +2887,6 @@ static int do_list_to_integer(Process *p, Eterm orig_list,
 	     res = big_plus_small(res, m, hp);
 	 }
 
-	 if (is_big(res))  /* check if small */
-	     res = big_plus_small(res, 0, hp); /* includes conversion to small */
-	 
 	 if (neg) {
 	     if (is_small(res))
 		 res = make_small(-signed_val(res));
@@ -2899,8 +2896,12 @@ static int do_list_to_integer(Process *p, Eterm orig_list,
 	     }
 	 }
 
-	 if (is_big(res)) {
-	     hp += (big_arity(res)+1);
+	 if (is_not_small(res)) {
+	     res = big_plus_small(res, 0, hp); /* includes conversion to small */
+
+	     if (is_not_small(res)) {
+		 hp += (big_arity(res)+1);
+	     }
 	 }
 	 HRelease(p,hp_end,hp);
      }
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index e62caa6b22..a8710dd910 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2675,9 +2675,6 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes,
 	res = big_plus_small(res, m, hp);
     }
 
-    if (is_big(res)) /* check if small */
-	res = big_plus_small(res, 0, hp); /* includes conversion to small */
-
     if (neg) {
 	if (is_small(res))
 	    res = make_small(-signed_val(res));
@@ -2687,8 +2684,12 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes,
 	}
     }
 
-    if (is_big(res)) {
-	hp += (big_arity(res) + 1);
+    if (is_not_small(res)) {
+	res = big_plus_small(res, 0, hp); /* includes conversion to small */
+
+	if (is_not_small(res)) {
+	    hp += (big_arity(res) + 1);
+	}
     }
     HRelease(BIF_P, hp_end, hp);
     goto bytebuf_to_integer_1_done;
-- 
cgit v1.2.3


From 2b53f2b1514b0d405ab92c767277483c761ad928 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 4 Sep 2014 20:10:54 +0200
Subject: erts: Refactor binary allocation interface to also initialize Binary

except the reference counter 'refc', as different callers
have different strategies regarding the lifetime of the binary.
---
 erts/emulator/beam/beam_emu.c       |  4 ----
 erts/emulator/beam/binary.c         |  5 -----
 erts/emulator/beam/dist.c           |  2 --
 erts/emulator/beam/erl_bif_binary.c |  2 --
 erts/emulator/beam/erl_binary.h     | 34 +++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_bits.c       | 10 ----------
 erts/emulator/beam/erl_gc.c         |  1 -
 erts/emulator/beam/erl_nif.c        |  3 ---
 erts/emulator/beam/external.c       | 10 ----------
 erts/emulator/beam/io.c             |  7 -------
 10 files changed, 27 insertions(+), 51 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8bfb7d2ad2..32961da271 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3782,8 +3782,6 @@ get_map_elements_fail:
 	  * Allocate the binary struct itself.
 	  */
 	 bptr = erts_bin_nrml_alloc(num_bytes);
-	 bptr->flags = 0;
-	 bptr->orig_size = num_bytes;
 	 erts_refc_init(&bptr->refc, 1);
 	 erts_current_bin = (byte *) bptr->orig_bytes;
 
@@ -3883,8 +3881,6 @@ get_map_elements_fail:
 	  * Allocate the binary struct itself.
 	  */
 	 bptr = erts_bin_nrml_alloc(tmp_arg1);
-	 bptr->flags = 0;
-	 bptr->orig_size = tmp_arg1;
 	 erts_refc_init(&bptr->refc, 1);
 	 erts_current_bin = (byte *) bptr->orig_bytes;
 
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index f50d484576..b014bca108 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -83,8 +83,6 @@ new_binary(Process *p, byte *buf, Uint len)
      * Allocate the binary struct itself.
      */
     bptr = erts_bin_nrml_alloc(len);
-    bptr->flags = 0;
-    bptr->orig_size = len;
     erts_refc_init(&bptr->refc, 1);
     if (buf != NULL) {
 	sys_memcpy(bptr->orig_bytes, buf, len);
@@ -122,8 +120,6 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
      * Allocate the binary struct itself.
      */
     bptr = erts_bin_nrml_alloc(len);
-    bptr->flags = 0;
-    bptr->orig_size = len;
     erts_refc_init(&bptr->refc, 1);
     if (buf != NULL) {
 	sys_memcpy(bptr->orig_bytes, buf, len);
@@ -177,7 +173,6 @@ erts_realloc_binary(Eterm bin, size_t size)
     } else {			/* REFC */
 	ProcBin* pb = (ProcBin *) bval;
 	Binary* newbin = erts_bin_realloc(pb->val, size);
-	newbin->orig_size = size;
 	pb->val = newbin;
 	pb->size = size;
 	pb->bytes = (byte*) newbin->orig_bytes;
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index ec07ddcd9c..dcbbb857da 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -622,9 +622,7 @@ alloc_dist_obuf(Uint size)
     ErtsDistOutputBuf *obuf;
     Uint obuf_size = sizeof(ErtsDistOutputBuf)+sizeof(byte)*(size-1);
     Binary *bin = erts_bin_drv_alloc(obuf_size);
-    bin->flags = BIN_FLAG_DRV;
     erts_refc_init(&bin->refc, 1);
-    bin->orig_size = (SWord) obuf_size;
     obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0];
 #ifdef DEBUG
     obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN;
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 3bf78adce7..85bc2daf5d 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2424,8 +2424,6 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
 	}
 	cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap
 							   if trapping */
-	cbs->result->flags = 0;
-	cbs->result->orig_size = target_size;
 	erts_refc_init(&(cbs->result->refc), 1);
 	t = (byte *) cbs->result->orig_bytes; /* No offset or anything */
 	pos = 0;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 06dfeb1260..8d264d166e 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -231,41 +231,58 @@ erts_free_aligned_binary_bytes(byte* buf)
 #  define CHICKEN_PAD (sizeof(void*) - 1)
 #endif
 
+/* Caller must initialize 'refc'
+*/
 ERTS_GLB_INLINE Binary *
 erts_bin_drv_alloc_fnf(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    void *res;
+    Binary *res;
+
     if (bsize < size) /* overflow */
 	return NULL;
     res = erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
-    return (Binary *) res;
+    if (res) {
+	res->orig_size = size;
+	res->flags = BIN_FLAG_DRV;
+    }
+    return res;
 }
 
+/* Caller must initialize 'refc'
+*/
 ERTS_GLB_INLINE Binary *
 erts_bin_drv_alloc(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    void *res;
+    Binary *res;
+
     if (bsize < size) /* overflow */
 	erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size);
     res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
-    return (Binary *) res;
+    res->orig_size = size;
+    res->flags = BIN_FLAG_DRV;
+    return res;
 }
 
 
+/* Caller must initialize 'refc'
+*/
 ERTS_GLB_INLINE Binary *
 erts_bin_nrml_alloc(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    void *res;
+    Binary *res;
+
     if (bsize < size) /* overflow */
 	erts_alloc_enomem(ERTS_ALC_T_BINARY, size);
     res = erts_alloc(ERTS_ALC_T_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
-    return (Binary *) res;
+    res->orig_size = size;
+    res->flags = 0;
+    return res;
 }
 
 ERTS_GLB_INLINE Binary *
@@ -280,6 +297,8 @@ erts_bin_realloc_fnf(Binary *bp, Uint size)
 	return NULL;
     nbp = erts_realloc_fnf(type, (void *) bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
+    if (nbp)
+	nbp->orig_size = size;
     return nbp;
 }
 
@@ -297,6 +316,7 @@ erts_bin_realloc(Binary *bp, Uint size)
     if (!nbp)
 	erts_realloc_enomem(type, bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
+    nbp->orig_size = size;
     return nbp;
 }
 
@@ -329,4 +349,4 @@ erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif
+#endif /* !__ERL_BINARY_H */
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 73765772c8..71d31c01aa 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1299,7 +1299,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
     if (binp->orig_size < pb->size) {
 	Uint new_size = 2*pb->size;
 	binp = erts_bin_realloc(binp, new_size);
-	binp->orig_size = new_size;
 	pb->val = binp;
 	pb->bytes = (byte *) binp->orig_bytes;
     }
@@ -1371,8 +1370,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	 * Allocate the binary data struct itself.
 	 */
 	bptr = erts_bin_nrml_alloc(bin_size);
-	bptr->flags = 0;
-	bptr->orig_size = bin_size;
 	erts_refc_init(&bptr->refc, 1);
 	erts_current_bin = (byte *) bptr->orig_bytes;
 
@@ -1475,7 +1472,6 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
 	     * is safe to reallocate it.
 	     */
 	    binp = erts_bin_realloc(binp, new_size);
-	    binp->orig_size = new_size;
 	    pb->val = binp;
 	    pb->bytes = (byte *) binp->orig_bytes;
 	} else {
@@ -1488,8 +1484,6 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
 	     * binary and copy the contents of the old binary into it.
 	     */
 	    Binary* bptr = erts_bin_nrml_alloc(new_size);
-	    bptr->flags = 0;
-	    bptr->orig_size = new_size;
 	    erts_refc_init(&bptr->refc, 1);
 	    sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size);
 	    pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER;
@@ -1537,8 +1531,6 @@ erts_bs_init_writable(Process* p, Eterm sz)
      * Allocate the binary data struct itself.
      */
     bptr = erts_bin_nrml_alloc(bin_size);
-    bptr->flags = 0;
-    bptr->orig_size = bin_size;
     erts_refc_init(&bptr->refc, 1);
     
     /*
@@ -1585,9 +1577,7 @@ erts_emasculate_writable_binary(ProcBin* pb)
     /* Our allocators are 8 byte aligned, i.e., shrinking with
        less than 8 bytes will have no real effect */
     if (unused >= 8) {
-	Uint new_size = pb->size;
 	binp = erts_bin_realloc(binp, pb->size);
-	binp->orig_size = new_size;
 	pb->val = binp;
 	pb->bytes = (byte *) binp->orig_bytes;
     }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 0db42d4325..1dc9e8a786 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2400,7 +2400,6 @@ sweep_off_heap(Process *p, int fullsweep)
 		}
 
 		pb->val = erts_bin_realloc(pb->val, new_size);
-		pb->val->orig_size = new_size;
 		pb->bytes = (byte *) pb->val->orig_bytes;
 	    }
 	}
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1caea6dcf8..769d13f860 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -551,9 +551,7 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin)
     if (refbin == NULL) {
 	return 0; /* The NIF must take action */
     }
-    refbin->flags = BIN_FLAG_DRV; /* BUGBUG: Flag? */
     erts_refc_init(&refbin->refc, 1);
-    refbin->orig_size = (SWord) size;
 
     bin->size = size;
     bin->data = (unsigned char*) refbin->orig_bytes;
@@ -573,7 +571,6 @@ int enif_realloc_binary(ErlNifBinary* bin, size_t size)
 	if (!newbin) {
 	    return 0;
 	}    
-	newbin->orig_size = size;
 	bin->ref_bin = newbin;
 	bin->data = (unsigned char*) newbin->orig_bytes;
 	bin->size = size;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9b9b4b2a62..196913a741 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1899,8 +1899,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		}
 
 		result_bin = erts_bin_nrml_alloc(size);
-		result_bin->flags = 0;
-		result_bin->orig_size = size;
 		erts_refc_init(&result_bin->refc, 0);
 		result_bin->orig_bytes[0] = VERSION_MAGIC;
 		/* Next state immediately, no need to export context */
@@ -1925,7 +1923,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		}
 		real_size = endp - bytes;
 		result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size);
-		result_bin->orig_size = real_size;
 		level = context->s.ec.level;
 		BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR);
 		if (level == 0 || real_size < 6) { /* We are done */
@@ -1962,8 +1959,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		context->s.cc.result_bin = result_bin;
 
 		result_bin = erts_bin_nrml_alloc(real_size);
-		result_bin->flags = 0;
-		result_bin->orig_size = real_size;
 		erts_refc_init(&result_bin->refc, 0);
 		result_bin->orig_bytes[0] = VERSION_MAGIC;
 
@@ -2005,7 +2000,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 			erl_zlib_deflate_finish(&(context->s.cc.stream));
 			result_bin = erts_bin_realloc(context->s.cc.destination_bin,
 						      context->s.cc.dest_len+6);
-			result_bin->orig_size = context->s.cc.dest_len+6;
 			context->s.cc.destination_bin = NULL;
 			pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
 			pb->thing_word = HEADER_PROC_BIN;
@@ -3387,8 +3381,6 @@ dec_term_atom_common:
 		} else {
 		    Binary* dbin = erts_bin_nrml_alloc(n);
 		    ProcBin* pb;
-		    dbin->flags = 0;
-		    dbin->orig_size = n;
 		    erts_refc_init(&dbin->refc, 1);
 		    pb = (ProcBin *) hp;
 		    hp += PROC_BIN_SIZE;
@@ -3441,8 +3433,6 @@ dec_term_atom_common:
 		    Binary* dbin = erts_bin_nrml_alloc(n);
 		    ProcBin* pb;
 
-		    dbin->flags = 0;
-		    dbin->orig_size = n;
 		    erts_refc_init(&dbin->refc, 1);
 		    pb = (ProcBin *) hp;
 		    pb->thing_word = HEADER_PROC_BIN;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index ae053fc191..afafa96cdf 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3153,8 +3153,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
 	Binary* bptr;
 
 	bptr = erts_bin_nrml_alloc(len);
-	bptr->flags = 0;
-	bptr->orig_size = len;
 	erts_refc_init(&bptr->refc, 1);
 	sys_memcpy(bptr->orig_bytes, buf, len);
 
@@ -5506,8 +5504,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		ProcBin* pbp;
 		Binary* bp = erts_bin_nrml_alloc(size);
 		ASSERT(bufp);
-		bp->flags = 0;
-		bp->orig_size = (SWord) size;
 		erts_refc_init(&bp->refc, 1);
 		sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
 		pbp = (ProcBin *) hp;
@@ -5999,9 +5995,7 @@ driver_alloc_binary(ErlDrvSizeT size)
     bin = erts_bin_drv_alloc_fnf((Uint) size);
     if (!bin)
 	return NULL; /* The driver write must take action */
-    bin->flags = BIN_FLAG_DRV;
     erts_refc_init(&bin->refc, 1);
-    bin->orig_size = (SWord) size;
     return Binary2ErlDrvBinary(bin);
 }
 
@@ -6031,7 +6025,6 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size)
     if (!newbin)
 	return NULL;
 
-    newbin->orig_size = size;
     return Binary2ErlDrvBinary(newbin);
 }
 
-- 
cgit v1.2.3


From eb3d3e8f726c64f12c00517be87c53dc5de934e2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 5 Sep 2014 14:54:25 +0200
Subject: Use separate allocation type for NIF export

---
 erts/emulator/beam/erl_alloc.types |  2 ++
 erts/emulator/beam/erl_nif.c       | 18 +++++++++++++++---
 erts/emulator/beam/erl_process.h   |  7 ++++---
 3 files changed, 21 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 17ac6316b7..37354b7f8d 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -357,6 +357,7 @@ type	DB_MS_PSDO_PROC	LONG_LIVED_LOW	ETS		db_match_pseudo_proc
 type	SCHDLR_DATA	LONG_LIVED_LOW	SYSTEM		scheduler_data
 type  	LL_TEMP_TERM 	LONG_LIVED_LOW	SYSTEM		ll_temp_term
 
+type	NIF_TRAP_EXPORT	STANDARD_LOW	CODE		nif_trap_export_entry
 type	EXPORT		LONG_LIVED_LOW	CODE		export_entry
 type	MONITOR_SH	STANDARD_LOW	PROCESSES	monitor_sh
 type	NLINK_SH	STANDARD_LOW	PROCESSES	nlink_sh
@@ -375,6 +376,7 @@ type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
 type  	LL_TEMP_TERM 	LONG_LIVED	SYSTEM		ll_temp_term
 
+type	NIF_TRAP_EXPORT	STANDARD	CODE		nif_trap_export_entry
 type	EXPORT		LONG_LIVED	CODE		export_entry
 type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
 type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1414744763..df135c09c5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1566,17 +1566,29 @@ allocate_nif_sched_data(Process* proc, int argc)
 
     argv_extra = argc > 1 ? sizeof(Eterm)*(argc-1) : 0;
     total = sizeof(NifExport) + argv_extra;
-    ep = erts_alloc(ERTS_ALC_T_PSD, total);
+    ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total);
     sys_memset((void*) ep, 0, total);
     ep->alloced_argv_sz = argc;
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
 	ep->exp.addressv[i] = &ep->exp.code[3];
     }
     ep->exp.code[3] = (BeamInstr) em_call_nif;
-    (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, &ep->exp);
+    (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, ep);
     return ep;
 }
 
+static ERTS_INLINE void
+destroy_nif_export(NifExport *nif_export)
+{
+    erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export);
+}
+
+void
+erts_destroy_nif_export(void *nif_export)
+{
+    destroy_nif_export((NifExport *) nif_export);
+}
+
 /*
  * Initialize a NifExport struct. Create it if needed and store it in the
  * proc. The direct_fp function is what will be invoked by op_call_nif, and
@@ -1599,7 +1611,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
 	ep = allocate_nif_sched_data(proc, argc);
     else if (need_save && ep->alloced_argv_sz < argc) {
 	NifExport* new_ep = allocate_nif_sched_data(proc, argc);
-	erts_free(ERTS_ALC_T_PSD, (void*) ep);
+	destroy_nif_export(ep);
 	ep = new_ep;
     }
     ERTS_VBUMP_ALL_REDS(proc);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 31f4a09c94..d12eb92bff 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1362,6 +1362,7 @@ Uint64 erts_ensure_later_proc_interval(Uint64);
 Uint64 erts_step_proc_interval(void);
 
 int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
+void erts_destroy_nif_export(void *); /* see erl_nif.c */
 
 ErtsProcList *erts_proclist_create(Process *);
 void erts_proclist_destroy(ErtsProcList *);
@@ -1814,9 +1815,9 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
     ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))
 
 #define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
-    ((Export *) erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT))
-#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, DSTE) \
-    ((Export *) erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (DSTE)))
+    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
+#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
+    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
 
 
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
-- 
cgit v1.2.3


From 18a38b9e5f5fbf4aa8fb7d349bc493c78626d3f6 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 5 Sep 2014 14:55:25 +0200
Subject: Fix leak of NIF exports

---
 erts/emulator/beam/erl_process.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e96d5c4ccd..716875c326 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11826,6 +11826,7 @@ erts_continue_exit_process(Process *p)
     struct saved_calls *scb;
     process_breakpoint_time_t *pbt;
     erts_aint32_t state;
+    void *nif_export;
 
 #ifdef DEBUG
     int yield_allowed = 1;
@@ -11976,6 +11977,7 @@ erts_continue_exit_process(Process *p)
 	   : NULL);
     scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL);
     pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL);
+    nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, ERTS_PROC_LOCKS_ALL, NULL);
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 #ifdef BM_COUNTERS
@@ -12023,6 +12025,9 @@ erts_continue_exit_process(Process *p)
     if (pbt)
         erts_free(ERTS_ALC_T_BPD, (void *) pbt);
 
+    if (nif_export)
+	erts_destroy_nif_export(nif_export);
+
     delete_process(p);
 
 #ifdef ERTS_SMP
-- 
cgit v1.2.3


From a1c09befb56cf8e73b722fe69ff87db65815bb9d Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 8 Sep 2014 09:19:21 +0200
Subject: erts: Print that we are crashdumping earlier

Change so that we print that the we generating a crash dump
before we actually generate it. So that if a huge crashdump is
being dumped, the user knows that something is going on.
---
 erts/emulator/beam/break.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 08265b590d..5aee85174f 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -754,6 +754,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 	dumpname = "erl_crash.dump";
     else
 	dumpname = &dumpnamebuf[0];
+    
+    erts_fprintf(stderr,"\nCrash dump is being written to: %s...", dumpname);
 
     fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640);
     if (fd < 0) 
@@ -804,7 +806,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 
     erts_fdprintf(fd, "=end\n");
     close(fd);
-    erts_fprintf(stderr,"\nCrash dump was written to: %s\n", dumpname);
+    erts_fprintf(stderr,"done\n");
 }
 
 void
-- 
cgit v1.2.3


From 84144e03cd902813d1c9f9d75a8b6ecc5e58270d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 10 Sep 2014 12:02:22 +0200
Subject: erts: Correct dirty scheduler NIF API for Windows

enif_schedule_nif() put LAST of the unconditional functions to keep the
order which is vital for ABI compatibility on Windows.

The conditional dirty scheduler stuff moved down at the end of the list
to keep them out of the way. We don't want them mess things up then they
become unconditional some day.
---
 erts/emulator/beam/erl_nif_api_funcs.h | 43 ++++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index be39816a64..e99cd36153 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -22,7 +22,7 @@
 #endif
 
 /*
-** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list
+** WARNING: Add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list
 ** to keep compatibility on Windows!!!
 **
 ** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h
@@ -141,12 +141,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg));
 ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg));
 ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent));
-ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
-ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
-#endif
-
 ERL_NIF_API_FUNC_DECL(int, enif_is_map, (ErlNifEnv* env, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term, size_t *size));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_new_map, (ErlNifEnv* env));
@@ -161,12 +155,23 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_is_tail, (ErlNifEnv *env, ErlNifMap
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIterator *iter));
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter));
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value));
-
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
 
 /*
-** Add new entries here to keep compatibility on Windows!!!
+** ADD NEW ENTRIES HERE (before this comment) !!!
 */
+
+
+/*
+ * Conditional EXPERIMENTAL stuff always last.
+ * Must be moved up and made unconditional to support binary backward
+ * compatibility on Windows.
+ */
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
+ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
 #endif
+#endif /* ERL_NIF_API_FUNC_DECL */
 
 /*
 ** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order
@@ -280,19 +285,12 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa
 #  define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64)
 #  define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64)
 #endif
-
 #  define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception)
 #  define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list)
 #  define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number)
 #  define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen)
 #  define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym)
 #  define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice)
-#  define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif)
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-#  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
-#  define enif_have_dirty_schedulers ERL_NIF_API_FUNC_MACRO(enif_have_dirty_schedulers)
-#endif
-
 #  define enif_is_map ERL_NIF_API_FUNC_MACRO(enif_is_map)
 #  define enif_get_map_size ERL_NIF_API_FUNC_MACRO(enif_get_map_size)
 #  define enif_make_new_map ERL_NIF_API_FUNC_MACRO(enif_make_new_map)
@@ -307,11 +305,22 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa
 #  define enif_map_iterator_next ERL_NIF_API_FUNC_MACRO(enif_map_iterator_next)
 #  define enif_map_iterator_prev ERL_NIF_API_FUNC_MACRO(enif_map_iterator_prev)
 #  define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair)
+#  define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif)
 
 /*
-** Add new entries here
+** ADD NEW ENTRIES HERE (before this comment)
 */
+
+/*
+ * Conditional EXPERIMENTAL stuff always last
+ * Must be moved up and made unconditional to support binary backward
+ * compatibility on Windows.
+ */
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+#  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
+#  define enif_have_dirty_schedulers ERL_NIF_API_FUNC_MACRO(enif_have_dirty_schedulers)
 #endif
+#endif  /* ERL_NIF_API_FUNC_MACRO */
 
 
 #if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
-- 
cgit v1.2.3


From 12d3d25535225934cb2190c82c1b5dae005f1cda Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 11 Sep 2014 17:47:32 +0200
Subject: erts: Remove enif_have_dirty_schedulers()

and add 'dirty_scheduler_support' to ErlNifSysInfo
---
 erts/emulator/beam/erl_driver.h        |  2 +-
 erts/emulator/beam/erl_drv_nif.h       |  1 +
 erts/emulator/beam/erl_nif.c           | 10 ----------
 erts/emulator/beam/erl_nif_api_funcs.h |  2 --
 erts/emulator/beam/io.c                | 12 ++++++++++++
 5 files changed, 14 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 5ced8c5ca0..f9938fc66c 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -133,7 +133,7 @@ typedef struct {
 
 #define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
 #define ERL_DRV_EXTENDED_MAJOR_VERSION	3
-#define ERL_DRV_EXTENDED_MINOR_VERSION	0
+#define ERL_DRV_EXTENDED_MINOR_VERSION	1
 
 /*
  * The emulator will refuse to load a driver with a major version
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 3f829ea7ea..4e8c6dc68b 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -35,6 +35,7 @@ typedef struct {
     int scheduler_threads;
     int nif_major_version;
     int nif_minor_version;
+    int dirty_scheduler_support;
 }  ErlDrvSysInfo;
 
 typedef struct {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 44914d3681..ede5f335dc 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1901,16 +1901,6 @@ enif_is_on_dirty_scheduler(ErlNifEnv* env)
     return ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data);
 }
 
-int
-enif_have_dirty_schedulers()
-{
-#ifdef USE_THREADS
-    return 1;
-#else
-    return 0;
-#endif
-}
-
 #endif /* ERL_NIF_DIRTY_SCHEDULER_SUPPORT */
 
 /* Maps */
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index e99cd36153..630cefae93 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -169,7 +169,6 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int
  */
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
-ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
 #endif
 #endif /* ERL_NIF_API_FUNC_DECL */
 
@@ -318,7 +317,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void));
  */
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 #  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
-#  define enif_have_dirty_schedulers ERL_NIF_API_FUNC_MACRO(enif_have_dirty_schedulers)
 #endif
 #endif  /* ERL_NIF_API_FUNC_MACRO */
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index ae053fc191..9ae973e108 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7274,6 +7274,18 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
 	sip->nif_major_version = ERL_NIF_MAJOR_VERSION;
 	sip->nif_minor_version = ERL_NIF_MINOR_VERSION;
     }
+    /*
+     * 'dirty_scheduler_support' is the last field in the 4th version
+     * (driver version 3.1, NIF version 2.7)
+     */
+    if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) {
+#if defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) && defined(USE_THREADS)
+	sip->dirty_scheduler_support = 1;
+#else
+	sip->dirty_scheduler_support = 0;
+#endif
+    }
+
 }
 
 
-- 
cgit v1.2.3


From 9c9878d6d4e99aba195177baef66067da0a2b797 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 16 Sep 2014 15:22:15 +0200
Subject: erts: Add icount build type for opcode counter

Enables ERTS_OPCODE_COUNTER_SUPPORT.
---
 erts/emulator/beam/erl_bif_info.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 6efe9d9550..e3fa9e9cb7 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -115,6 +115,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #ifdef ERTS_ENABLE_LOCK_COUNT
 				     " [lock-counting]"
 #endif
+#ifdef ERTS_OPCODE_COUNTER_SUPPORT
+				     " [instruction-counting]"
+#endif
 #ifdef PURIFY
 				     " [purify-compiled]"
 #endif	
@@ -2300,7 +2303,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	for (i = num_instructions-1; i >= 0; i--) {
 	    res = erts_bld_cons(hpp, hszp,
 				erts_bld_tuple(hpp, hszp, 2,
-					       erts_atom_put(opc[i].name,
+					       erts_atom_put((byte *)opc[i].name,
 							     strlen(opc[i].name),
 							     ERTS_ATOM_ENC_LATIN1,
 							     1),
-- 
cgit v1.2.3


From 16d8a6ce56fd066efa9ecb1d3fa3b3dab6d9613c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 22 Sep 2014 12:00:05 +0200
Subject: erts: Fix ub in list_to_integer and bignum div

---
 erts/emulator/beam/bif.c | 8 +++++---
 erts/emulator/beam/big.c | 7 +++----
 2 files changed, 8 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index a5be8e1529..42dd160e38 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2772,6 +2772,7 @@ static int do_list_to_integer(Process *p, Eterm orig_list,
 			      Eterm *integer, Eterm *rest)
 {
      Sint i = 0;
+     Uint ui = 0;
      int skip = 0;
      int neg = 0;
      int n = 0;
@@ -2825,8 +2826,8 @@ static int do_list_to_integer(Process *p, Eterm orig_list,
 	     unsigned_val(CAR(list_val(lst))) > '9') {
 	     break;
 	 }
-	 i = i * 10;
-	 i = i + unsigned_val(CAR(list_val(lst))) - '0';
+	 ui = ui * 10;
+	 ui = ui + unsigned_val(CAR(list_val(lst))) - '0';
 	 n++;
 	 lst = CDR(list_val(lst));
 	 if (is_nil(lst)) {
@@ -2850,7 +2851,8 @@ static int do_list_to_integer(Process *p, Eterm orig_list,
       */
 
      if (n <= SMALL_DIGITS) {  /* It must be small */
-	 if (neg) i = -i;
+	 if (neg) i = -(Sint)ui;
+         else i = (Sint)ui;
 	 res = make_small(i);
      } else {
 	 lg2 =  (n+1)*230/69+1;
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index a8710dd910..de7d370938 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -274,10 +274,9 @@
 	_b = _b << _s;							\
 	_vn1 = _b >> H_EXP;						\
 	_vn0 = _b & LO_MASK;						\
-        /* Sometimes _s is 0 which triggers undefined behaviour for the \
-           (_a0>>(D_EXP-_s)) shift, but this is ok because the          \
-           & -s will make it all to 0 later anyways. */                 \
-	_un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1)));	\
+        /* If needed to avoid undefined behaviour */                    \
+        if (_s) _un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1))); \
+        else _un32 = _a1;                                               \
 	_un10 = _a0 << _s;						\
 	_un1 = _un10 >> H_EXP;						\
 	_un0 = _un10 & LO_MASK;						\
-- 
cgit v1.2.3


From 6084a42a24fca52a5de2bc487c0cd2be46dcc21f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 26 Aug 2014 17:26:31 +0200
Subject: Introduce support for eager check I/O scheduling

---
 erts/emulator/beam/erl_alloc.types |  1 +
 erts/emulator/beam/erl_bif_info.c  | 38 +++++++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_init.c      | 15 +++++++++++++++
 erts/emulator/beam/erl_port_task.c | 24 ++++++++++++++++++------
 erts/emulator/beam/erl_port_task.h |  2 +-
 erts/emulator/beam/erl_process.c   | 38 +++++++++++++++++++++++---------------
 erts/emulator/beam/erl_process.h   |  1 +
 erts/emulator/beam/sys.h           | 12 ++++++++++++
 erts/emulator/beam/utils.c         |  1 +
 9 files changed, 103 insertions(+), 29 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index bb5eba80be..0bbd3e2dd5 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -393,6 +393,7 @@ type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
 type	DRV_EV_D_STATE	FIXED_SIZE	SYSTEM		driver_event_data_state
 type	DRV_SEL_D_STATE	FIXED_SIZE	SYSTEM		driver_select_data_state
 type	FD_LIST		SHORT_LIVED	SYSTEM		fd_list
+type	ACTIVE_FD_ARR	SHORT_LIVED	SYSTEM		active_fd_array
 type	POLLSET		LONG_LIVED	SYSTEM		pollset
 type	POLLSET_UPDREQ	SHORT_LIVED	SYSTEM		pollset_update_req
 type	POLL_FDS	LONG_LIVED	SYSTEM		poll_fds
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d7f1e2d971..8f4095f236 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2643,6 +2643,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) {
         BIF_RET(make_small(erts_db_get_max_tabs()));
     }
+    else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) {
+	BIF_RET(erts_eager_check_io ? am_true : am_false);
+    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -3232,17 +3235,38 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    BIF_RET(make_small((Uint) words));
 	}
 	else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) {
-	    /* Used by (emulator) */
-	    int res;
+	    /* Used by driver_SUITE (emulator) */
+	    Uint sz, *szp;
+	    Eterm res, *hp, **hpp;
+	    int no_errors;
+	    ErtsCheckIoDebugInfo ciodi = {0};
 #ifdef HAVE_ERTS_CHECK_IO_DEBUG
 	    erts_smp_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN);
-	    res = erts_check_io_debug();
+	    no_errors = erts_check_io_debug(&ciodi);
 	    erts_smp_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN);
 #else
-	    res = 0;
-#endif
-	    ASSERT(res >= 0);
-	    BIF_RET(erts_make_integer((Uint) res, BIF_P));
+	    no_errors = 0;
+#endif
+	    sz = 0;
+	    szp = &sz;
+	    hpp = NULL;
+	    while (1) {
+		res = erts_bld_tuple(hpp, szp, 4,
+				     erts_bld_uint(hpp, szp,
+						   (Uint) no_errors),
+				     erts_bld_uint(hpp, szp,
+						   (Uint) ciodi.no_used_fds),
+				     erts_bld_uint(hpp, szp,
+						   (Uint) ciodi.no_driver_select_structs),
+				     erts_bld_uint(hpp, szp,
+						   (Uint) ciodi.no_driver_event_structs));
+		if (hpp)
+		    break;
+		hp = HAlloc(BIF_P, sz);
+		szp = NULL;
+		hpp = &hp;
+	    }
+	    BIF_RET(res);
 	}
 	else if (ERTS_IS_ATOM_STR("process_info_args", BIF_ARG_1)) {
 	    /* Used by process_SUITE (emulator) */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 8c4fffa75b..c3be23c84a 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -537,6 +537,8 @@ void erts_usage(void)
     erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sct cput   set cpu topology,\n");
     erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
+    erts_fprintf(stderr, "-secio bool enable/disable eager check I/O scheduling,\n");
+    erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sws val    set scheduler wakeup strategy, valid values are:\n");
     erts_fprintf(stderr, "            default|legacy.\n");
     erts_fprintf(stderr, "-swct val   set scheduler wake cleanup threshold, valid values are:\n");
@@ -1487,6 +1489,19 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
+	    else if (has_prefix("ecio", sub_param)) {
+		arg = get_arg(sub_param+4, argv[i+1], &i);
+		if (sys_strcmp("true", arg) == 0)
+		    erts_eager_check_io = 1;
+		else if (sys_strcmp("false", arg) == 0)
+		    erts_eager_check_io = 0;
+		else {
+		    erts_fprintf(stderr,
+				 "bad schedule eager check I/O value '%s'\n",
+				 arg);
+		    erts_usage();
+		}
+	    }
 	    else if (has_prefix("pp", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
 		if (sys_strcmp(arg, "true") == 0)
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 547a42beb2..ff4fdc70aa 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -32,6 +32,7 @@
 #include "global.h"
 #include "erl_port_task.h"
 #include "dist.h"
+#include "erl_check_io.h"
 #include "dtrace-wrapper.h"
 #include <stdarg.h>
 
@@ -542,6 +543,16 @@ reset_handle(ErtsPortTask *ptp)
     }
 }
 
+static ERTS_INLINE void
+reset_executed_io_task_handle(ErtsPortTask *ptp)
+{
+    if (ptp->u.alive.handle) {
+	ASSERT(ptp == handle2task(ptp->u.alive.handle));
+	erts_io_notify_port_task_executed(ptp->u.alive.handle);
+	reset_port_task_handle(ptp->u.alive.handle);
+    }
+}
+
 static ERTS_INLINE void
 set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
 {
@@ -1365,10 +1376,7 @@ erts_port_task_schedule(Eterm id,
     ErtsPortTask *ptp = NULL;
     erts_aint32_t act, add_flags;
 
-    if (pthp && erts_port_task_is_scheduled(pthp)) {
-	ASSERT(0);
-	erts_port_task_abort(pthp);
-    }
+    ERTS_LC_ASSERT(!pthp || !erts_port_task_is_scheduled(pthp));
 
     ASSERT(is_internal_port(id));
 
@@ -1654,8 +1662,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    goto aborted_port_task;
 	}
 
-	reset_handle(ptp);
-
 	if (erts_system_monitor_long_schedule != 0) {
 	    start_time = erts_timestamp_millis();
 	}
@@ -1666,6 +1672,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
 	switch (ptp->type) {
 	case ERTS_PORT_TASK_TIMEOUT:
+	    reset_handle(ptp);
 	    reds = ERTS_PORT_REDS_TIMEOUT;
 	    if (!(state & ERTS_PORT_SFLGS_DEAD)) {
                 DTRACE_DRIVER(driver_timeout, pp);
@@ -1679,6 +1686,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    /* NOTE some windows drivers use ->ready_input for input and output */
 	    (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data,
 					ptp->u.alive.td.io.event);
+	    reset_executed_io_task_handle(ptp);
 	    io_tasks_executed++;
 	    break;
 	case ERTS_PORT_TASK_OUTPUT:
@@ -1687,6 +1695,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
             DTRACE_DRIVER(driver_ready_output, pp);
 	    (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data,
 					 ptp->u.alive.td.io.event);
+	    reset_executed_io_task_handle(ptp);
 	    io_tasks_executed++;
 	    break;
 	case ERTS_PORT_TASK_EVENT:
@@ -1696,10 +1705,12 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data,
 				  ptp->u.alive.td.io.event,
 				  ptp->u.alive.td.io.event_data);
+	    reset_executed_io_task_handle(ptp);
 	    io_tasks_executed++;
 	    break;
 	case ERTS_PORT_TASK_PROC_SIG: {
 	    ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data;
+	    reset_handle(ptp);
 	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
 	    if (!pp->sched.taskq.bpq)
 		reds = ptp->u.alive.td.psig.callback(pp,
@@ -1717,6 +1728,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    break;
 	}
 	case ERTS_PORT_TASK_DIST_CMD:
+	    reset_handle(ptp);
 	    reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds);
 	    break;
 	default:
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 123253a057..e1e06b565e 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -154,7 +154,7 @@ erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
 ERTS_GLB_INLINE int
 erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp)
 {
-    return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL;
+    return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL;
 }
 
 ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1efd070afd..4f3ea92cf5 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -143,6 +143,7 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
+int erts_eager_check_io = 0;
 int erts_sched_compact_load;
 Uint erts_no_schedulers;
 
@@ -1977,19 +1978,28 @@ try_set_sys_scheduling(void)
 #endif
 
 static ERTS_INLINE int
-prepare_for_sys_schedule(void)
+prepare_for_sys_schedule(int non_blocking)
 {
+    if (non_blocking && erts_eager_check_io) {
 #ifdef ERTS_SMP
-    while (!erts_port_task_have_outstanding_io_tasks()
-	   && try_set_sys_scheduling()) {
-	if (!erts_port_task_have_outstanding_io_tasks())
-	    return 1;
-	clear_sys_scheduling();
+	return try_set_sys_scheduling();
+#else
+	return 1;
+#endif
     }
-    return 0;
+    else {
+#ifdef ERTS_SMP
+	while (!erts_port_task_have_outstanding_io_tasks()
+	       && try_set_sys_scheduling()) {
+	    if (!erts_port_task_have_outstanding_io_tasks())
+		return 1;
+	    clear_sys_scheduling();
+	}
+	return 0;
 #else
-    return !erts_port_task_have_outstanding_io_tasks();
+	return !erts_port_task_have_outstanding_io_tasks();
 #endif
+    }
 }
 
 #ifdef ERTS_SMP
@@ -2307,7 +2317,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
      * be waiting in erl_sys_schedule()
      */
 
-    if (!prepare_for_sys_schedule()) {
+    if (!prepare_for_sys_schedule(0)) {
 
 	sched_waiting(esdp->no, rq);
 
@@ -2459,7 +2469,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		 * Got to check that we still got I/O tasks; otherwise
 		 * we have to continue checking for I/O...
 		 */
-		if (!prepare_for_sys_schedule()) {
+		if (!prepare_for_sys_schedule(0)) {
 		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
 		    goto tse_wait;
 		}
@@ -2481,7 +2491,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	     * Got to check that we still got I/O tasks; otherwise
 	     * we have to wait in erl_sys_schedule() after all...
 	     */
-	    if (!prepare_for_sys_schedule()) {
+	    if (!prepare_for_sys_schedule(0)) {
 		/*
 		 * Not allowed to wait in erl_sys_schedule;
 		 * do tse wait instead...
@@ -7022,7 +7032,7 @@ Process *schedule(Process *p, int calls)
 
 	    goto check_activities_to_run;
 	}
-	else if (fcalls > input_reductions && prepare_for_sys_schedule()) {
+	else if (fcalls > input_reductions && prepare_for_sys_schedule(!0)) {
 	    /*
 	     * Schedule system-level activities.
 	     */
@@ -7030,8 +7040,6 @@ Process *schedule(Process *p, int calls)
 	    erts_smp_atomic32_set_relb(&function_calls, 0);
 	    fcalls = 0;
 
-	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
-
 #if 0 /* Not needed since we wont wait in sys schedule */
 	    erts_sys_schedule_interrupt(0);
 #endif
@@ -7063,7 +7071,7 @@ Process *schedule(Process *p, int calls)
 	if (RUNQ_READ_LEN(&rq->ports.info.len)) {
 	    int have_outstanding_io;
 	    have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port);
-	    if ((have_outstanding_io && fcalls > 2*input_reductions)
+	    if ((!erts_eager_check_io && have_outstanding_io && fcalls > 2*input_reductions)
 		|| rq->halt_in_progress) {
 		/*
 		 * If we have performed more than 2*INPUT_REDUCTIONS since
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8d136f6e8b..41936c1bb0 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -97,6 +97,7 @@ struct saved_calls {
 };
 
 extern Export exp_send, exp_receive, exp_timeout;
+extern int erts_eager_check_io;
 extern int erts_sched_compact_load;
 extern Uint erts_no_schedulers;
 extern Uint erts_no_run_queues;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 31252ed78f..e691d5c55c 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -64,8 +64,12 @@
  */
 
 #ifndef ERTS_SYS_FD_TYPE
+#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
 typedef int ErtsSysFdType;
 #else
+#ifndef ERTS_SYS_FD_INVALID
+# error missing ERTS_SYS_FD_INVALID
+#endif
 typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #endif
 
@@ -731,6 +735,14 @@ void init_getenv_state(GETENV_STATE *);
 char * getenv_string(GETENV_STATE *);
 void fini_getenv_state(GETENV_STATE *);
 
+#define HAVE_ERTS_CHECK_IO_DEBUG
+typedef struct {
+    int no_used_fds;
+    int no_driver_select_structs;
+    int no_driver_event_structs;
+} ErtsCheckIoDebugInfo;
+int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);
+
 /* xxxP */
 #define SYS_DEFAULT_FLOAT_DECIMALS 20
 void init_sys_float(void);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 0d75bbcc77..426e43d5d8 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -47,6 +47,7 @@
 #include "erl_sched_spec_pre_alloc.h"
 #include "beam_bp.h"
 #include "erl_ptab.h"
+#include "erl_check_io.h"
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
-- 
cgit v1.2.3


From 404c44d58058da8cfead5dd310fcb24bf3e8d344 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 24 Sep 2014 17:48:28 +0200
Subject: No eager check I/O on OSE

---
 erts/emulator/beam/erl_init.c    | 5 ++++-
 erts/emulator/beam/erl_process.c | 5 +++++
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index ef77e1d7fc..61f8385efc 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1678,9 +1678,12 @@ erl_start(int argc, char **argv)
 	    }
 	    else if (has_prefix("ecio", sub_param)) {
 		arg = get_arg(sub_param+4, argv[i+1], &i);
+#ifndef __OSE__
 		if (sys_strcmp("true", arg) == 0)
 		    erts_eager_check_io = 1;
-		else if (sys_strcmp("false", arg) == 0)
+		else
+#endif
+		if (sys_strcmp("false", arg) == 0)
 		    erts_eager_check_io = 0;
 		else {
 		    erts_fprintf(stderr,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 65abd87c08..e5bb1203c8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -148,7 +148,12 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
+#ifdef __OSE__
+/* Eager check I/O not supported on OSE yet. */
+int erts_eager_check_io = 0;
+#else
 int erts_eager_check_io = 0;
+#endif
 int erts_sched_compact_load;
 int erts_sched_balance_util = 0;
 Uint erts_no_schedulers;
-- 
cgit v1.2.3


From cb604704efd28fafd0f8edce03db00f7fef53909 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 23 Sep 2014 18:06:44 +0200
Subject: Change default to "eager check I/O"

Conflicts:
	erts/emulator/beam/erl_process.c
---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e5bb1203c8..7b272885a7 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -152,7 +152,7 @@ extern BeamInstr beam_continue_exit[];
 /* Eager check I/O not supported on OSE yet. */
 int erts_eager_check_io = 0;
 #else
-int erts_eager_check_io = 0;
+int erts_eager_check_io = 1;
 #endif
 int erts_sched_compact_load;
 int erts_sched_balance_util = 0;
-- 
cgit v1.2.3


From bfc6a9b3325969a7bc1d0c228766537049a4f637 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 30 Sep 2014 16:55:06 +0200
Subject: erts: Fix erlang:port_set_data/2 for non immediate data

hsize field was not set leading to VM crash
---
 erts/emulator/beam/erl_bif_port.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index afb33c1cdb..8a622e5e6e 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -554,6 +554,7 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2)
 	hp = &pdhp->heap[0];
 	pdhp->off_heap.first = NULL;
 	pdhp->off_heap.overhead = 0;
+	pdhp->hsize = hsize;
 	pdhp->data = copy_struct(BIF_ARG_2, hsize, &hp, &pdhp->off_heap);
 	data = (erts_aint_t) pdhp;
 	ASSERT((data & 0x3) == 0);
-- 
cgit v1.2.3


From b7801392325784027e93e3488e7e28d596e6b658 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 30 Sep 2014 17:00:37 +0200
Subject: erts: Fix race between port_set_data, port_get_data and port
 termination

Always update prt->data with atomic xchg-op.
Check for NULL data to detect racing port terminator.
Use NULL, as THE_NON_VALUE can be a valid pointer on debug VM.
---
 erts/emulator/beam/erl_bif_port.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 8a622e5e6e..64bd598ba6 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -493,8 +493,8 @@ void
 erts_cleanup_port_data(Port *prt)
 {
     ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP);
-    cleanup_old_port_data(erts_smp_atomic_read_nob(&prt->data));
-    erts_smp_atomic_set_nob(&prt->data, (erts_aint_t) THE_NON_VALUE);
+    cleanup_old_port_data(erts_smp_atomic_xchg_nob(&prt->data,
+						   (erts_aint_t) NULL));
 }
 
 Uint
@@ -562,8 +562,14 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2)
 
     data = erts_smp_atomic_xchg_wb(&prt->data, data);
 
+    if (data == (erts_aint_t)NULL) {
+	/* Port terminated by racing thread */
+	data = erts_smp_atomic_xchg_wb(&prt->data, data);
+	ASSERT(data != (erts_aint_t)NULL);
+	cleanup_old_port_data(data);
+	BIF_ERROR(BIF_P, BADARG);
+    }
     cleanup_old_port_data(data);
-
     BIF_RET(am_true);
 }
 
@@ -582,6 +588,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
         BIF_ERROR(BIF_P, BADARG);
 
     data = erts_smp_atomic_read_ddrb(&prt->data);
+    if (data == (erts_aint_t)NULL)
+        BIF_ERROR(BIF_P, BADARG);  /* Port terminated by racing thread */
 
     if ((data & 0x3) != 0) {
 	res = (Eterm) (UWord) data;
-- 
cgit v1.2.3


From a22560bae02fcb531538d692cbfcc0e293d57747 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Wed, 1 Oct 2014 18:44:36 -0400
Subject: Fix missing field initializer in ERL_NIF_INIT macro

In the ERL_NIF_INIT macro, add a missing static initializer for the new
options field which was added to the ErlNifEntry struct for 17.3. This fix
prevents compile-time errors in NIFs when the compiler is instructed to
treat missing field initializers as errors.
---
 erts/emulator/beam/erl_nif.h | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 226fc199a1..849024453c 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -241,21 +241,10 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  else
 #    define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
 #  endif
-#  ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-#    define ERL_NIF_INIT_BODY do {					\
-	memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)); \
-	entry.options = ERL_NIF_DIRTY_NIF_OPTION;			\
-     } while(0)
-#  else
-#    define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
-#  endif
+#  define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
 #else 
 #  define ERL_NIF_INIT_GLOB
-#  ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-#    define ERL_NIF_INIT_BODY entry.options = ERL_NIF_DIRTY_NIF_OPTION
-#  else
-#    define ERL_NIF_INIT_BODY
-#  endif
+#  define ERL_NIF_INIT_BODY
 #  ifdef STATIC_ERLANG_NIF
 #    define ERL_NIF_INIT_DECL(MODNAME)  ErlNifEntry* MODNAME ## _nif_init(void)
 #  else
@@ -263,6 +252,11 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  endif
 #endif
 
+#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
+#  define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION
+#else
+#  define ERL_NIF_ENTRY_OPTIONS 0
+#endif
 
 #ifdef __cplusplus
 }
@@ -288,7 +282,8 @@ ERL_NIF_INIT_DECL(NAME)			\
 	sizeof(FUNCS) / sizeof(*FUNCS),	\
 	FUNCS,				\
 	LOAD, RELOAD, UPGRADE, UNLOAD,	\
-	ERL_NIF_VM_VARIANT		\
+	ERL_NIF_VM_VARIANT,		\
+	ERL_NIF_ENTRY_OPTIONS		\
     };                                  \
     ERL_NIF_INIT_BODY;                  \
     return &entry;			\
-- 
cgit v1.2.3


From 21a16f6c19cd8aba39481c89bf2b32c7910f3d32 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 2 Oct 2014 21:40:28 +0200
Subject: erts: Mend port_set_data with non-immed data for halfword VM

---
 erts/emulator/beam/erl_alloc.types | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 37354b7f8d..df33bbc2f7 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -267,7 +267,6 @@ type	CODE_IX_LOCK_Q  SHORT_LIVED	SYSTEM		code_ix_lock_q
 type	PROC_INTERVAL	LONG_LIVED	SYSTEM		process_interval
 type    BUSY_CALLER_TAB	SHORT_LIVED	SYSTEM		busy_caller_table
 type    BUSY_CALLER	SHORT_LIVED	SYSTEM		busy_caller
-type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
 type	PROC_SYS_TSK	SHORT_LIVED	PROCESSES	proc_sys_task
 type	PROC_SYS_TSK_QS	SHORT_LIVED	PROCESSES	proc_sys_task_queues
 
@@ -364,6 +363,7 @@ type	NLINK_SH	STANDARD_LOW	PROCESSES	nlink_sh
 type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	STANDARD_LOW	SYSTEM		gc_info_request
+type	PORT_DATA_HEAP	STANDARD_LOW	SYSTEM		port_data_heap
 
 +else # "fullword"
 
@@ -383,6 +383,7 @@ type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
 type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
+type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
 
 +endif
 
-- 
cgit v1.2.3


From 891cc466c957e91c7770f0a91ba83b65a268c2c1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 3 Oct 2014 11:37:52 +0200
Subject: erts: Fix bug when delayed deallocated carrier is reused by
 cpool_fetch

The delayed dealloc queue destroyes one word but cpool_fetch() is expected
to return healthy carriers.

So we restore that overwritten word with a little bit of hackish code.
---
 erts/emulator/beam/erl_alloc_util.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index a4e164bf51..55052430e1 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1775,6 +1775,18 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	     * data has been overwritten by the queue.
 	     */
 	    Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk);
+
+	     /* Restore word overwritten by the dd-queue as it will be read
+	      * if this carrier is pulled from dc_list by cpool_fetch()
+	      */
+	    ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr);
+	    ERTS_ALC_CPOOL_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*));
+#ifdef MBC_ABLK_OFFSET_BITS
+	    blk->u.carrier = crr;
+#else
+	    blk->carrier = crr;
+#endif
+
 	    ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
 	    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
 	    ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr)
-- 
cgit v1.2.3


From e16f287c28ac93b54772169958c965733aa67e66 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 25 Sep 2014 22:50:24 +0200
Subject: erts: Add pooled_list and traitor_list

---
 erts/emulator/beam/erl_alloc_util.c | 300 +++++++++++++++++++++++++++++++-----
 erts/emulator/beam/erl_alloc_util.h |  13 +-
 2 files changed, 272 insertions(+), 41 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 55052430e1..ba16e5c3a3 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -205,7 +205,7 @@ MBC after deallocating first block:
      ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
      (B)->bhdr = ((Sz) | (F)), \
      (B)->u.carrier = (C))
-      
+
 #  define IS_MBC_FIRST_ABLK(AP,B) \
   ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \
    && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0)
@@ -378,9 +378,8 @@ do {										\
 
 #ifdef ERTS_SMP
 #define SBC_HEADER_SIZE	   						\
-    (UNIT_CEILING(sizeof(Carrier_t)					\
-		  - sizeof(ErtsAlcCPoolData_t)				\
-		  + ABLK_HDR_SZ)					\
+    (UNIT_CEILING(offsetof(Carrier_t, cpool)                            \
+	          + ABLK_HDR_SZ)	                                \
      - ABLK_HDR_SZ)
 #else
 #define SBC_HEADER_SIZE	   						\
@@ -929,6 +928,88 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
 
 #ifdef ERTS_SMP
 
+#ifdef DEBUG
+static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node)
+{
+    ErtsDoubleLink_t* p;
+
+    ASSERT(node != sentinel);
+    for (p = sentinel->next; p != sentinel; p = p->next) {
+	if (p == node)
+	    return 1;
+    }
+    return 0;
+}
+#endif /* DEBUG */
+
+static ERTS_INLINE void
+link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node)
+{
+    ErtsDoubleLink_t* before_me = after_me->next;
+    ASSERT(node != after_me && node != before_me);
+    node->next = before_me;
+    node->prev = after_me;
+    before_me->prev = node;
+    after_me->next = node;
+}
+
+static ERTS_INLINE void
+link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node)
+{
+    ErtsDoubleLink_t* after_me = before_me->prev;
+    ASSERT(node != before_me && node != after_me);
+    node->next = before_me;
+    node->prev = after_me;
+    before_me->prev = node;
+    after_me->next = node;
+}
+
+static ERTS_INLINE void
+unlink_edl(ErtsDoubleLink_t* node)
+{
+    node->next->prev = node->prev;
+    node->prev->next = node->next;
+}
+
+static ERTS_INLINE void
+relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node)
+{
+    if (node != before_me && node != before_me->prev) {
+	unlink_edl(node);
+	link_edl_before(before_me, node);
+    }
+}
+
+static ERTS_INLINE int is_abandoned(Carrier_t *crr)
+{
+    return crr->cpool.abandoned.next != NULL;
+}
+
+static ERTS_INLINE void
+link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr)
+{
+    ASSERT(!is_abandoned(crr));
+
+    link_edl_after(list, &crr->cpool.abandoned);
+
+    ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned);
+    ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned);
+}
+
+static ERTS_INLINE void
+unlink_abandoned_carrier(Carrier_t *crr)
+{
+    ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list,
+		      &crr->cpool.abandoned) ||
+	   is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list,
+		      &crr->cpool.abandoned));
+
+    unlink_edl(&crr->cpool.abandoned);
+
+    crr->cpool.abandoned.next = NULL;
+    crr->cpool.abandoned.prev = NULL;
+}
+
 static ERTS_INLINE void
 clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
 {
@@ -955,7 +1036,7 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
     }
 }
 
-#endif
+#endif /* ERTS_SMP */
 
 #if 0
 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B)			\
@@ -2575,10 +2656,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 #ifdef ERTS_SMP
 
 #define ERTS_ALC_MAX_DEALLOC_CARRIER		10
-#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT	10
+#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT	20
+#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT	10
 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT	100
-#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS		5
-#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET	100
 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS	3
 
 #define ERTS_ALC_CPOOL_PTR_MOD_MRK		(((erts_aint_t) 1) << 0)
@@ -2916,59 +2996,163 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr)
 static Carrier_t *
 cpool_fetch(Allctr_t *allctr, UWord size)
 {
-    int i;
+    int i, i_stop, has_passed_sentinel;
     Carrier_t *crr;
     ErtsAlcCPoolData_t *cpdp;
-    ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+    ErtsAlcCPoolData_t *cpool_entrance;
+    ErtsAlcCPoolData_t *sentinel;
+    ErtsDoubleLink_t* dl;
+    ErtsDoubleLink_t* first_old_traitor;
 
     ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */
 			  || erts_thr_progress_is_managed_thread());
 
-    i = 0;
+    i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT;
+    first_old_traitor = allctr->cpool.traitor_list.next;
+    cpool_entrance = NULL;
 
-    /* First; check our own pending dealloc carrier list... */
-    crr = allctr->cpool.dc_list.last;
-    while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) {
-	if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
-	    unlink_carrier(&allctr->cpool.dc_list, crr);
-#ifdef ERTS_ALC_CPOOL_DEBUG
-	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
-							   ((erts_aint_t) allctr))
-				  == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK));
-#else
-	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
-#endif
-	    return crr;
+    /*
+     * Search my own pooled_list,
+     * i.e my abandoned carriers that were in the pool last time I checked.
+     */
+
+    dl = allctr->cpool.pooled_list.next;
+    while(dl != &allctr->cpool.pooled_list) {
+	erts_aint_t exp, act;
+	crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned));
+
+	ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl));
+	ASSERT(crr->cpool.orig_allctr == allctr);
+	dl = dl->next;
+	exp = erts_smp_atomic_read_rb(&crr->allctr);
+	if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL
+	    && erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
+	    /* Try to fetch it... */
+	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+					     (erts_aint_t) allctr,
+					     exp);
+	    if (act == exp) {
+		cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
+		unlink_abandoned_carrier(crr);
+
+		/* Move sentinel to continue next search from here */
+		relink_edl_before(dl, &allctr->cpool.pooled_list);
+		return crr;
+	    }
+	    exp = act;
+	}
+	if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
+	    if (!cpool_entrance)
+		cpool_entrance = &crr->cpool;
+	}
+	else { /* Not in pool, move to traitor_list */
+	    unlink_abandoned_carrier(crr);
+	    link_abandoned_carrier(&allctr->cpool.traitor_list, crr);
+	}
+	if (--i <= 0) {
+	    /* Move sentinel to continue next search from here */
+	    relink_edl_before(dl, &allctr->cpool.pooled_list);
+	    return NULL;
 	}
-	crr = crr->prev;
-	i++;
     }
 
-    /* ... then the pool ... */
+    /* Now search traitor_list.
+     * i.e carriers employed by other allocators last time I checked.
+     * They might have been abandoned since then.
+     */
+
+    i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ?
+	      0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT);
+    dl = first_old_traitor;
+    while(dl != &allctr->cpool.traitor_list) {
+	erts_aint_t exp, act;
+	crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned));
+	ASSERT(dl != &allctr->cpool.pooled_list);
+	ASSERT(crr->cpool.orig_allctr == allctr);
+	dl = dl->next;
+	exp = erts_smp_atomic_read_rb(&crr->allctr);
+	if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
+	    if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY)
+		&& erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
+		/* Try to fetch it... */
+		act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+						 (erts_aint_t) allctr,
+						 exp);
+		if (act == exp) {
+		    cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
+		    unlink_abandoned_carrier(crr);
+
+		    /* Move sentinel to continue next search from here */
+		    relink_edl_before(dl, &allctr->cpool.traitor_list);
+		    return crr;
+		}
+		exp = act;
+	    }
+	    if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
+		if (!cpool_entrance)
+		    cpool_entrance = &crr->cpool;
+
+		/* Move to pooled_list */
+		unlink_abandoned_carrier(crr);
+		link_abandoned_carrier(&allctr->cpool.pooled_list, crr);
+	    }
+	}
+	if (--i <= i_stop) {
+	    /* Move sentinel to continue next search from here */
+	    relink_edl_before(dl, &allctr->cpool.traitor_list);
+	    if (i > 0)
+		break;
+	    else
+		return NULL;
+	}
+    }
 
     /*
-     * We search in 'prev' direction and begin by passing
-     * one element before trying to fetch. This in order to
-     * avoid contention with threads inserting elements.
+     * Finally search the shared pool and try employ foreign carriers
      */
 
-    cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev));
-    if (cpdp == sentinel)
-	return NULL;
+    sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+    if (cpool_entrance) {
+	/* We saw a pooled carried above, use it as entrance into the pool
+	 */
+	cpdp = cpool_entrance;
+    }
+    else {
+	/* No pooled carried seen above. Start search at cpool sentinel,
+	 * but begin by passing one element before trying to fetch.
+	 * This in order to avoid contention with threads inserting elements.
+	 */
+	cpool_entrance = sentinel;
+	cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev));
+	if (cpdp == sentinel)
+	    return NULL;
+    }
 
-    while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) {
+    has_passed_sentinel = 0;
+    while (1) {
 	erts_aint_t exp;
 	cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
-	if (cpdp == sentinel) {
+	if (cpdp == cpool_entrance) {
+	    if (cpool_entrance == sentinel) {
+		cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
+		if (cpdp == sentinel)
+		    return NULL;
+	    }
+	    i = 0; /* Last one to inspect */
+	}
+	else if (cpdp == sentinel) {
+	    if (has_passed_sentinel) {
+		/* We been here before. cpool_entrance must have been removed */
+		return NULL;
+	    }
 	    cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
 	    if (cpdp == sentinel)
 		return NULL;
-	    i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */
+	    has_passed_sentinel = 1;
 	}
-	crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool));
+	crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool));
 	exp = erts_smp_atomic_read_rb(&crr->allctr);
-	if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY))
-	     == ERTS_CRR_ALCTR_FLG_IN_POOL)
+	if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL)
 	    && (erts_atomic_read_nob(&cpdp->max_size) >= size)) {
 	    erts_aint_t act;
 	    /* Try to fetch it... */
@@ -2977,11 +3161,35 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 					     exp);
 	    if (act == exp) {
 		cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
+		if (crr->cpool.orig_allctr == allctr) {
+		    unlink_abandoned_carrier(crr);
+		}
 		return crr;
 	    }
 	}
-	i++;
+	if (--i <= 0)
+	    return NULL;
     }
+
+    /* Last; check our own pending dealloc carrier list... */
+    crr = allctr->cpool.dc_list.last;
+    while (crr) {
+	if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
+	    unlink_carrier(&allctr->cpool.dc_list, crr);
+#ifdef ERTS_ALC_CPOOL_DEBUG
+	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
+							   ((erts_aint_t) allctr))
+				  == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK));
+#else
+	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
+#endif
+	    return crr;
+	}
+	crr = crr->prev;
+	if (--i <= 0)
+	    return NULL;
+    }
+
     return NULL;
 }
 
@@ -3078,6 +3286,9 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 	return;
     }
 
+    if (is_abandoned(crr))
+	unlink_abandoned_carrier(crr);
+
     if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID
 	|| erts_thr_progress_has_reached(crr->cpool.thr_prgr)) {
 	dealloc_carrier(allctr, crr, 1);
@@ -3124,6 +3335,8 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 	    limit = (csz/100)*allctr->cpool.util_limit;
 	crr->cpool.abandon_limit = limit;
     }
+    crr->cpool.abandoned.next = NULL;
+    crr->cpool.abandoned.prev = NULL;
 }
 
 static void
@@ -3154,6 +3367,9 @@ abandon_carrier(Allctr_t *allctr, Carrier_t *crr)
     STAT_MBC_CPOOL_INSERT(allctr, crr);
 
     unlink_carrier(&allctr->mbc_list, crr);
+    if (crr->cpool.orig_allctr == allctr) {
+	link_abandoned_carrier(&allctr->cpool.pooled_list, crr);
+    }
 
     allctr->remove_mbc(allctr, crr);
 
@@ -5540,6 +5756,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	    allctr->min_block_size = sz;
     }
 
+    allctr->cpool.pooled_list.next  = &allctr->cpool.pooled_list;
+    allctr->cpool.pooled_list.prev  = &allctr->cpool.pooled_list;
+    allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list;
+    allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list;
     allctr->cpool.dc_list.first = NULL;
     allctr->cpool.dc_list.last = NULL;
     allctr->cpool.abandon_limit = 0;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 7be6b1ed9d..eee920e66c 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -268,6 +268,11 @@ typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
 
 #ifdef ERTS_SMP
 
+typedef struct ErtsDoubleLink_t_ {
+    struct ErtsDoubleLink_t_ *next;
+    struct ErtsDoubleLink_t_ *prev;
+}ErtsDoubleLink_t;
+
 typedef struct {
     erts_atomic_t next;
     erts_atomic_t prev;
@@ -277,6 +282,7 @@ typedef struct {
     UWord abandon_limit;
     UWord blocks;
     UWord blocks_size;
+    ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
 } ErtsAlcCPoolData_t;
 
 #endif
@@ -500,7 +506,12 @@ struct Allctr_t_ {
     CarrierList_t	sbc_list;
 #ifdef ERTS_SMP
     struct {
-	CarrierList_t	dc_list;
+	/* pooled_list, traitor list and dc_list contain only
+           carriers _created_ by this allocator */
+	ErtsDoubleLink_t pooled_list;
+	ErtsDoubleLink_t traitor_list;
+	CarrierList_t	 dc_list;
+
 	UWord		abandon_limit;
 	int		disable_abandon;
 	int		check_limit_count;
-- 
cgit v1.2.3


From dd853b15e9fb770a3f04fe4504fbfbb8de89e28d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 6 Oct 2014 22:29:44 +0200
Subject: erts: Fix bug causing mbc removed from cpool to be used at pool
 entrance

Clear both IN_POOL and BUSY flags when empty carrier is removed
is removed from pool to be destroyed.

Earlier it was enough to leave BUSY flag set but now with pooled_list
we must clear IN_POOL to avoid using it as cpool_entrance in cpool_fetch().
---
 erts/emulator/beam/erl_alloc_util.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index ba16e5c3a3..0ea9b62103 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3877,6 +3877,11 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	if (busy_pcrr_pp && *busy_pcrr_pp) {
 	    ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr);
 	    *busy_pcrr_pp = NULL;
+	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
+				  == (((erts_aint_t) allctr)
+				      | ERTS_CRR_ALCTR_FLG_IN_POOL
+				      | ERTS_CRR_ALCTR_FLG_BUSY));
+	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
 	    cpool_delete(allctr, allctr, crr);
 	}
 	else
-- 
cgit v1.2.3


From a5ce823283d36eda4788c55b0b9b645577473983 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 22 Sep 2014 17:21:15 +0200
Subject: erts: Fix SMP for ERTS_OPCODE_COUNTER_SUPPORT

Cleanup macro code.
---
 erts/emulator/beam/beam_emu.c | 29 ++++++++++-------------------
 erts/emulator/beam/erl_init.c |  8 --------
 erts/emulator/beam/erl_vm.h   |  2 --
 3 files changed, 10 insertions(+), 29 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8bfb7d2ad2..f02ace9094 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -241,10 +241,6 @@ BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 void** beam_ops;
 #endif
 
-#ifndef ERTS_SMP /* Not supported with smp emulator */
-extern int count_instructions;
-#endif
-
 #define SWAPIN             \
     HTOP = HEAP_TOP(c_p);  \
     E = c_p->stop
@@ -1163,13 +1159,14 @@ void process_main(void)
 
     Eterm (*arith_func)(Process* p, Eterm* reg, Uint live);
 
-#ifndef NO_JUMP_TABLE
-    static void* opcodes[] = { DEFINE_OPCODES };
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
     static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES };
-#endif
+#else
+#ifndef NO_JUMP_TABLE
+    static void* opcodes[] = { DEFINE_OPCODES };
 #else
     int Go;
+#endif
 #endif
 
     Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
@@ -5145,22 +5142,16 @@ get_map_elements_fail:
 
 #ifndef NO_JUMP_TABLE
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
-
      /* Are tables correctly generated by beam_makeops? */
      ASSERT(sizeof(counting_opcodes) == sizeof(opcodes));
-
-     if (count_instructions) {
 #ifdef DEBUG
-	 counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
+     counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
 #endif
-	 counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI);
-	 beam_ops = counting_opcodes;
-     }
-     else
-#endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
-     {
-	 beam_ops = opcodes;
-     }
+     counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI);
+     beam_ops = counting_opcodes;
+#else /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
+     beam_ops = opcodes;
+#endif /* ERTS_OPCODE_COUNTER_SUPPORT */
 #endif /* NO_JUMP_TABLE */
      
      em_call_error_handler = OpCode(call_error_handler);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 88c4006934..890ef89cc1 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -161,9 +161,6 @@ int H_MIN_SIZE;			/* The minimum heap grain */
 int BIN_VH_MIN_SIZE;		/* The minimum binary virtual*/
 
 Uint32 erts_debug_flags;	/* Debug flags. */
-#ifdef ERTS_OPCODE_COUNTER_SUPPORT
-int count_instructions;
-#endif
 int erts_backtrace_depth;	/* How many functions to show in a backtrace
 				 * in error codes.
 				 */
@@ -1882,11 +1879,6 @@ erl_start(int argc, char **argv)
 	    if (argv[i][2] == 0) { /* -c: documented option */
 		erts_disable_tolerant_timeofday = 1;
 	    }
-#ifdef ERTS_OPCODE_COUNTER_SUPPORT
-	    else if (argv[i][2] == 'i') { /* -ci: undcoumented option*/
-		count_instructions = 1;
-	    }
-#endif
 	    break;
 	case 'W':
 	    arg = get_arg(argv[i]+2, argv[i+1], &i);
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index b7de8208ad..78d98229d8 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -20,8 +20,6 @@
 #ifndef __ERL_VM_H__
 #define __ERL_VM_H__
 
-/* #define ERTS_OPCODE_COUNTER_SUPPORT */
-
 /* FORCE_HEAP_FRAGS:
  * Debug provocation to make HAlloc always create heap fragments (if allowed)
  * even if there is room on heap.
-- 
cgit v1.2.3


From 5ab275d6554f4db877624a152dda90c0e3c4cddc Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 30 Jun 2014 17:14:11 +0200
Subject: erts: Make writing to non-tty fds non-blocking

If the fd_driver is given non-tty fds it will now use an async thread
instead of doing a blocking write. An example of a non-tty fd is the
fd used by stderr, but could be anything handed down to the driver.
---
 erts/emulator/beam/erl_alloc.types | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 17ac6316b7..d23d26a0fc 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -411,6 +411,8 @@ type	CS_PROG_PATH	LONG_LIVED	SYSTEM		cs_prog_path
 type	ENVIRONMENT	TEMPORARY	SYSTEM		environment
 type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
 type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
+type	SYS_BLOCKING	STANDARD	SYSTEM		sys_blocking
+type	SYS_WRITE_BUF	TEMPORARY	SYSTEM		sys_write_buf
 
 +endif
 
-- 
cgit v1.2.3


From 3ca75c7e42ab2c0612ceb5e56f60e7932c40f335 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 20 Oct 2014 16:04:44 +0200
Subject: erts: Fix bug causing mbc to be deleted from cpool before it was
 inserted

Set IN_POOL flag _after_ mbc has been actually inserted.

Earlier it did not matter if IN_POOL was set early as it was not possible
to find a carrier before is was fully inserted. Now when searching
pooled_list and traitor_list we must make sure a found carrier has been
fully inserted into cpool before removing it.
---
 erts/emulator/beam/erl_alloc_util.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 0ea9b62103..e3172dc4fb 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -2835,9 +2835,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
 			(erts_aint_t) CARRIER_SZ(crr));
     erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers);
 
-    erts_smp_atomic_set_nob(&crr->allctr,
-			    ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
-
     /*
      * We search in 'next' direction and begin by passing
      * one element before trying to insert. This in order to
@@ -2896,6 +2893,9 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
     cpool_set_mod_marked(&cpd2p->prev,
 			 (erts_aint_t) &crr->cpool,
 			 (erts_aint_t) cpd1p);
+
+    erts_smp_atomic_set_wb(&crr->allctr,
+			   ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
 }
 
 static void
-- 
cgit v1.2.3


From e45a4a8a3797a637a23ad5d660138c021f3ba1b1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 29 Aug 2014 16:06:20 +0200
Subject: erts: Make hipe_{un}reserve_beam_trap_frame INLINE

---
 erts/emulator/beam/bif.c            |  2 ++
 erts/emulator/beam/bif.h            |  3 +++
 erts/emulator/beam/binary.c         |  2 ++
 erts/emulator/beam/copy.c           |  2 ++
 erts/emulator/beam/erl_bif_binary.c |  2 ++
 erts/emulator/beam/erl_gc.c         |  2 ++
 erts/emulator/beam/erl_gc.h         | 35 ++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_process.h    |  6 +-----
 erts/emulator/beam/external.c       |  2 ++
 erts/emulator/beam/global.h         | 18 +-----------------
 erts/emulator/beam/utils.c          |  3 +++
 11 files changed, 52 insertions(+), 25 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index a5be8e1529..5b069eb30d 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -28,7 +28,9 @@
 #include "global.h"
 #include "erl_process.h"
 #include "error.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "dist.h"
 #include "erl_version.h"
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 72c55ccb55..7b69b39511 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -465,6 +465,8 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
+#ifdef ERL_WANT_HIPE_BIF_WRAPPER__
+
 #ifndef HIPE
 
 #define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)
@@ -509,6 +511,7 @@ BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
 
 #endif
 
+#endif /* ERL_WANT_HIPE_BIF_WRAPPER__ */
 
 #include "erl_bif_table.h"
 
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index f50d484576..a094a13f43 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -26,7 +26,9 @@
 #include "global.h"
 #include "erl_process.h"
 #include "error.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 50548850eb..0010f6a440 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -21,6 +21,8 @@
 #  include "config.h"
 #endif
 
+#define ERL_WANT_GC_INTERNALS__
+
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 3bf78adce7..26afffa09b 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -32,7 +32,9 @@
 #include "global.h"
 #include "erl_process.h"
 #include "error.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 0db42d4325..a48ad0eac2 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -20,6 +20,8 @@
 #  include "config.h"
 #endif
 
+#define ERL_WANT_GC_INTERNALS__
+
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 5203dda263..bf0496c112 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -20,10 +20,12 @@
 #ifndef __ERL_GC_H__
 #define __ERL_GC_H__
 
-#include "erl_map.h"
+#if defined(ERL_WANT_GC_INTERNALS__) || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)
 
 /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */
 
+#include "erl_map.h"
+
 #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
 #  define HARDDEBUG 1
 #endif
@@ -67,8 +69,6 @@ do {                                                                    \
 #define in_area(ptr,start,nbytes) \
  ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
-extern Uint erts_test_long_gc_sleep;
-
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 int within(Eterm *ptr, Process *p);
 #endif
@@ -97,4 +97,33 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
 }
 #endif
 
+#endif /* ERL_GC_C__ || HIPE_GC_C__ */
+
+/*
+ * Global exported
+ */
+
+extern Uint erts_test_long_gc_sleep;
+
+typedef struct {
+  Uint64 reclaimed;
+  Uint64 garbage_cols;
+} ErtsGCInfo;
+
+void erts_gc_info(ErtsGCInfo *gcip);
+void erts_init_gc(void);
+int erts_garbage_collect(struct process*, int, Eterm*, int);
+void erts_garbage_collect_hibernate(struct process* p);
+Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity);
+void erts_garbage_collect_literals(struct process* p, Eterm* literals,
+				   Uint lit_size,
+				   struct erl_off_heap_header* oh);
+Uint erts_next_heap_size(Uint, Uint);
+Eterm erts_heap_sizes(struct process* p);
+
+void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*);
+void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
+void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
+void erts_free_heap_frags(struct process* p);
+
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 27a3a3553b..3d08be25ff 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -58,6 +58,7 @@ typedef struct process Process;
 #include "external.h"
 #include "erl_mseg.h"
 #include "erl_async.h"
+#include "erl_gc.h"
 
 #ifdef HIPE
 #include "hipe_process.h"
@@ -488,11 +489,6 @@ typedef struct {
     } working;
 } ErtsSchedWallTime;
 
-typedef struct {
-  Uint64 reclaimed;
-  Uint64 garbage_cols;
-} ErtsGCInfo;
-
 typedef struct {
     int sched;
     erts_aint32_t aux_work;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9b9b4b2a62..c95479bc0b 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -36,7 +36,9 @@
 #include "erl_process.h"
 #include "error.h"
 #include "external.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "dist.h"
 #include "erl_binary.h"
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 891046a8b5..da9f029a9f 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -41,6 +41,7 @@
 #include "error.h"
 #include "erl_utils.h"
 #include "erl_port.h"
+#include "erl_gc.h"
 
 struct enif_environment_t /* ErlNifEnv */
 {
@@ -809,23 +810,6 @@ void MD5Init(MD5_CTX *);
 void MD5Update(MD5_CTX *, unsigned char *, unsigned int);
 void MD5Final(unsigned char [16], MD5_CTX *);
 
-/* ggc.c */
-
-void erts_gc_info(ErtsGCInfo *gcip);
-void erts_init_gc(void);
-int erts_garbage_collect(Process*, int, Eterm*, int);
-void erts_garbage_collect_hibernate(Process* p);
-Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(Process* p, Eterm* literals,
-				   Uint lit_size,
-				   struct erl_off_heap_header* oh);
-Uint erts_next_heap_size(Uint, Uint);
-Eterm erts_heap_sizes(Process* p);
-
-void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*);
-void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
-void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
-void erts_free_heap_frags(Process* p);
 
 /* io.c */
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f20e6e5665..9b3ee5cc65 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -49,6 +49,9 @@
 #include "beam_bp.h"
 #include "erl_ptab.h"
 #include "erl_check_io.h"
+#ifdef HIPE
+#  include "hipe_mode_switch.h"
+#endif
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
-- 
cgit v1.2.3


From 1e5eb4314301e33a105a0ff1f860d9d290ea2618 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Sun, 26 Oct 2014 21:23:00 -0400
Subject: Fix "-smp disable" for emulator with dirty schedulers

Running "erl -smp disable" on an emulator built with dirty scheduler
support caused problems such as segmentation violations and emulator status
line outputs containing garbage. For example:

    $ erl -smp disable
    Segmentation fault (core dumped)

and:

    $ erl -smp disable
    Erlang/OTP 17 [DEVELOPMENT] [erts-6.2] [source] [64-bit] [ds:10:4297895689:4299948152] [async-threads:280]

This problem also caused the emulator smoke_test_SUITE to hit these same
problems if run in an emulator started with the "-smp disable" option.

Fix this segmentation violation by ensuring that dirty scheduler
information is printed in the status line only when the emulator is
compiled with ERTS_SMP enabled.

With this fix in place, the smoke_test_SUITE now passes when the "-smp
disable" option is used, and the emulator status line prints correctly for
both "-smp enable" and "-smp disable":

    $ erl -smp enable
    Erlang/OTP 17 [DEVELOPMENT] [erts-6.2] [source] [64-bit] [smp:8:8] [ds:8:8:10] [async-threads:10] [kernel-poll:false]

and:

    $ erl -smp disable
    Erlang/OTP 17 [DEVELOPMENT] [erts-6.2] [source] [64-bit] [async-threads:10] [kernel-poll:false]
---
 erts/emulator/beam/erl_bif_info.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 6efe9d9550..e977c2ec5f 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -90,7 +90,7 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     " [smp:%beu:%beu]"
 #endif
 #ifdef USE_THREADS
-#ifdef ERTS_DIRTY_SCHEDULERS
+#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
 				     " [ds:%beu:%beu:%beu]"
 #endif
 				     " [async-threads:%d]"
-- 
cgit v1.2.3


From a2ffd6fd0e7f693847450aa633f7a4a18f9baead Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 27 Oct 2014 18:10:37 +0100
Subject: erts: Fix return value from erts_maps_get to be const

From "warning: assignment discards qualifiers from pointer target type"
---
 erts/emulator/beam/erl_nif.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index dd88f59908..10c7af5a9d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1733,7 +1733,7 @@ int enif_get_map_value(ErlNifEnv* env,
 		       Eterm key,
 		       Eterm *value)
 {
-    Eterm *ret;
+    const Eterm *ret;
     if (is_not_map(map)) {
 	return 0;
     }
-- 
cgit v1.2.3


From 4d73c647b55977d23c4295073945bd8aeb91ff83 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Thu, 14 Nov 2013 17:03:40 +0100
Subject: Use isfinite() instead of finite() when available

OS X Mavericks builds result in a number of warnings about finite() being
deprecated, like these:

beam/erl_arith.c:451:7: warning: 'finite' is deprecated: first deprecated in OS X 10.9 [-Wdeprecated-declarations]
                    ERTS_FP_ERROR(p, f1.fd, goto badarith);
                    ^
sys/unix/erl_unix_sys.h:319:33: note: expanded from macro 'ERTS_FP_ERROR'
                                        ^
sys/unix/erl_unix_sys.h:244:51: note: expanded from macro '__ERTS_FP_ERROR'
                                                  ^
/usr/include/math.h:718:12: note: 'finite' has been explicitly marked deprecated here
extern int finite(double) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_0, __MAC_10_9, __IPHONE_NA, __IPHONE_NA);

Add checks to use isfinite() instead of finite() where available. Verified
on OS X Mavericks 10.9.5 and Ubuntu 12.04.
---
 erts/emulator/beam/sys.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 3d8dd9c6d0..42272a14a3 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -501,7 +501,7 @@ extern volatile int erts_writing_erl_crash_dump;
 #  define NO_ERF
 #  define NO_ERFC
 /* This definition doesn't take NaN into account, but matherr() gets those */
-#  define finite(x) (fabs(x) != HUGE_VAL)
+#  define isfinite(x) (fabs(x) != HUGE_VAL)
 #  define USE_MATHERR
 #  define HAVE_FINITE
 #endif
-- 
cgit v1.2.3


From 5b4cb838afbe9e700139f810a1c4c9b0e91c511c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 29 Oct 2014 17:11:31 +0100
Subject: erts: Fix bug in beam_ranges

Symptom: VM on OSX (darwin11.4.2) with +Meamin running sasl tests,
crashing when init:reboot() does erlang:purge_module(installer).

Problem: Off-by-one bug in beam_ranges:find_range, returning the wrong
range if the 'end' of one module is the 'start' of the next. This is only
possible if using sys_alloc (+Meamin) as our own allocators always put
block headers between allocated payload data.
---
 erts/emulator/beam/beam_ranges.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 0f2d5d0c2a..cb6470638f 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -282,7 +282,7 @@ find_range(BeamInstr* pc)
     while (low < high) {
 	if (pc < mid->start) {
 	    high = mid;
-	} else if (pc > RANGE_END(mid)) {
+	} else if (pc >= RANGE_END(mid)) {
 	    low = mid + 1;
 	} else {
 	    erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid);
-- 
cgit v1.2.3


From 1bc59d68f5d22650fa18aa064ed8e50fc9a6a216 Mon Sep 17 00:00:00 2001
From: Peter Lemenkov <lemenkov@gmail.com>
Date: Sun, 2 Nov 2014 19:49:55 +0300
Subject: Expose NIF version

This patch allows checking for NIF API version in a way similar to
driver version. E.g. by calling erlang:system_info(nif_version).

Signed-off-by: Peter Lemenkov <lemenkov@gmail.com>
---
 erts/emulator/beam/erl_bif_info.c | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 61e4469600..2c00a5860e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -27,6 +27,7 @@
 #include "erl_process.h"
 #include "error.h"
 #include "erl_driver.h"
+#include "erl_nif.h"
 #include "bif.h"
 #include "big.h"
 #include "erl_version.h"
@@ -2459,6 +2460,13 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 			      ERL_DRV_EXTENDED_MINOR_VERSION);
 	hp = HAlloc(BIF_P, 2*n);
 	BIF_RET(buf_to_intlist(&hp, buf, n, NIL));
+    } else if (ERTS_IS_ATOM_STR("nif_version", BIF_ARG_1)) {
+	char buf[42];
+	int n = erts_snprintf(buf, 42, "%d.%d",
+			      ERL_NIF_MAJOR_VERSION,
+			      ERL_NIF_MINOR_VERSION);
+	hp = HAlloc(BIF_P, 2*n);
+	BIF_RET(buf_to_intlist(&hp, buf, n, NIL));
     } else if (ERTS_IS_ATOM_STR("smp_support", BIF_ARG_1)) {
 #ifdef ERTS_SMP
 	BIF_RET(am_true);
-- 
cgit v1.2.3


From d3f0abb4daf5c823f05c3908fdc4f6ee01035937 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 3 Nov 2014 18:04:26 +0100
Subject: erts: Add constant TERM_TO_BINARY_MEMCPY_FACTOR

and do not piggyback on B2T_MEMCPY_FACTOR
---
 erts/emulator/beam/external.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 5e1f5c56e1..831af2be4a 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1766,7 +1766,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
 #else
 #define TERM_TO_BINARY_COMPRESS_CHUNK 10
 #endif
-
+#define TERM_TO_BINARY_MEMCPY_FACTOR 8
 
 static void ttb_context_destructor(Binary *context_bin)
 {
@@ -2387,8 +2387,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    Uint bitoffs = WSTACK_POP(s);
 	    byte* bytes = (byte*) WSTACK_POP(s);
 	    byte* dst = (byte*) WSTACK_POP(s);
-	    if (bits > r * (B2T_MEMCPY_FACTOR * 8)) {
-		Uint n = r * B2T_MEMCPY_FACTOR;
+	    if (bits > r * (TERM_TO_BINARY_MEMCPY_FACTOR * 8)) {
+		Uint n = r * TERM_TO_BINARY_MEMCPY_FACTOR;
 		WSTACK_PUSH5(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs,
 			     ENC_BIN_COPY, bits - 8*n);
 		bits = 8*n;
@@ -2398,7 +2398,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		break;
 	    } else {
 		copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits);
-		r -= bits / (B2T_MEMCPY_FACTOR * 8);
+		r -= bits / (TERM_TO_BINARY_MEMCPY_FACTOR * 8);
 		goto outer_loop;
 	    }
 	}
@@ -2721,7 +2721,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    *ep++ = SMALL_INTEGER_EXT;
 		    *ep++ = bitsize;
 		}
-		if (ctx && j > r * B2T_MEMCPY_FACTOR) {
+		if (ctx && j > r * TERM_TO_BINARY_MEMCPY_FACTOR) {
 		    WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs,
 				 ENC_BIN_COPY, 8*j + bitsize);
 		} else {
-- 
cgit v1.2.3


From 66a184c576d9262045194e95c752a50c74877802 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Tue, 4 Nov 2014 11:46:19 -0500
Subject: Fix gc-related problem with dirty NIFs

Ensure that the return value from a dirty NIF call is made part of the GC
rootset. Add a new regression test to nif_SUITE.

Thanks to Daniel Goertzen for reporting the error and providing a test
case, and to Sverker Eriksson for making test case reproducible and finding
the fix.
---
 erts/emulator/beam/erl_nif.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ede5f335dc..adc3520ebb 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1646,6 +1646,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     ep->m = env->mod_nif;
     ep->fp = indirect_fp;
     proc->freason = TRAP;
+    proc->arity = argc;
     return THE_NON_VALUE;
 }
 
-- 
cgit v1.2.3


From c2fd0dadaeccd42585770fa672acde7bdfae3121 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 4 Nov 2014 18:00:15 +0100
Subject: Do not let non-empty run-queue prevent activation of scheduler

Conflicts:
	erts/emulator/beam/erl_process.c
---
 erts/emulator/beam/erl_process.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1efd070afd..ef59fa8ae0 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2692,11 +2692,11 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
 	return 0;
     wrq = ERTS_RUNQ_IX(ix);
     flags = ERTS_RUNQ_FLGS_GET(wrq);
+    if (activate && !(flags & ERTS_RUNQ_FLG_SUSPENDED)) {
+	if (try_inc_no_active_runqs(ix+1))
+	    (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE);
+    }
     if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) {
-	if (activate) {
-	    if (try_inc_no_active_runqs(ix+1))
-		(void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE);
-	}
 	wake_scheduler(wrq, 0);
 	return 1;
     }
-- 
cgit v1.2.3


From afd79e01d3375d8e8217b258ffb4d9b90541c922 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 13 Nov 2014 19:59:27 +0100
Subject: erts: Optimize ets:lookup and ets:take for bags

by reducing number of iterations through objects with matching key
---
 erts/emulator/beam/erl_db_hash.c | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 633a2272ad..6e8d892561 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -382,7 +382,7 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
 static void shrink(DbTableHash* tb, int nactive);
 static void grow(DbTableHash* tb, int nactive);
 static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
-			   DbTableHash*);
+			   Uint sz, DbTableHash*);
 static int analyze_pattern(DbTableHash *tb, Eterm pattern, 
 			   struct mp_info *mpi);
 
@@ -887,13 +887,17 @@ get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval,
 {
     HashDbTerm* b2 = b1->next;
     Eterm copy;
+    Uint sz = b1->dbterm.size + 2;
 
     if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
         while (b2 && has_key(tb, b2, key, hval)) {
+	    if (b2->hvalue != INVALID_HASH)
+		sz += b2->dbterm.size + 2;
+
             b2 = b2->next;
         }
     }
-    copy = build_term_list(p, b1, b2, tb);
+    copy = build_term_list(p, b1, b2, sz, tb);
     CHECK_TABLES();
     if (bend) {
         *bend = b2;
@@ -1253,7 +1257,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret)
     lck = RLOCK_HASH(tb, slot);
     nactive = NACTIVE(tb);
     if (slot < nactive) {
-	*ret = build_term_list(p, BUCKET(tb, slot), 0, tb);
+	*ret = build_term_list(p, BUCKET(tb, slot), NULL, 0, tb);
 	retval = DB_ERROR_NONE;
     }
     else if (slot == nactive) {
@@ -2536,23 +2540,23 @@ static int free_seg(DbTableHash *tb, int free_records)
 ** Copy terms from ptr1 until ptr2
 ** works for ptr1 == ptr2 == 0  => []
 ** or ptr2 == 0
+** sz is either precalculated heap size or 0 if not known
 */
 static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
-			   DbTableHash* tb)
+			     Uint sz, DbTableHash* tb)
 {
-    int sz = 0;
     HashDbTerm* ptr;
     Eterm list = NIL;
     Eterm copy;
     Eterm *hp, *hend;
 
-    ptr = ptr1;
-    while(ptr != ptr2) {
-
-	if (ptr->hvalue != INVALID_HASH)
-	    sz += ptr->dbterm.size + 2;
-
-	ptr = ptr->next;
+    if (!sz) {
+	ptr = ptr1;
+	while(ptr != ptr2) {
+	    if (ptr->hvalue != INVALID_HASH)
+		sz += ptr->dbterm.size + 2;
+	    ptr = ptr->next;
+	}
     }
 
     hp = HAlloc(p, sz);
-- 
cgit v1.2.3


From 6b435239e4eeb2bc57ef3e8ca8a2de48817b5571 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 25 Nov 2014 14:33:04 +0100
Subject: erts: Fix port data memory allocation bug

for non-immediate port data >= sizeof(Eterm)*2 words.
---
 erts/emulator/beam/erl_bif_port.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 64bd598ba6..7ce950e090 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -472,7 +472,7 @@ cleanup_old_port_data(erts_aint_t data)
 	ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data;
 	size_t size;
 	ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
-	size = sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm) - 1);
+	size = sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm);
 	erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap,
 						(void *) pdhp,
 						&pdhp->later_op,
@@ -508,7 +508,7 @@ erts_port_data_size(Port *prt)
     }
     else {
 	ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data;
-	return (Uint) sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm)-1);
+	return (Uint) sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm);
     }
 }
 
@@ -550,7 +550,7 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2)
 
 	hsize = size_object(BIF_ARG_2);
 	pdhp = erts_alloc(ERTS_ALC_T_PORT_DATA_HEAP,
-			  sizeof(ErtsPortDataHeap) + hsize*(sizeof(Eterm)-1));
+			  sizeof(ErtsPortDataHeap) + (hsize-1)*sizeof(Eterm));
 	hp = &pdhp->heap[0];
 	pdhp->off_heap.first = NULL;
 	pdhp->off_heap.overhead = 0;
-- 
cgit v1.2.3


From 9f24ed8e0ddfa426c79f13c7091c39834c66780f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 26 Nov 2014 18:41:47 +0100
Subject: erts: Use internal stack for ets db_has_variable

---
 erts/emulator/beam/erl_db_util.c | 52 +++++++++++++++++++++-------------------
 1 file changed, 28 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index b9fd3b208e..7eb80e3bb1 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3254,34 +3254,38 @@ int db_is_variable(Eterm obj)
 /* return 1 if obj contains a variable or underscore */
 /* return 0 if obj is fully ground                   */
 
-int db_has_variable(Eterm obj)
-{
-    switch(obj & _TAG_PRIMARY_MASK) {
-    case TAG_PRIMARY_LIST: {
-	while (is_list(obj)) {
-	    if (db_has_variable(CAR(list_val(obj))))
+int db_has_variable(Eterm node) {
+    DECLARE_ESTACK(s);
+
+    ESTACK_PUSH(s,node);
+    while (!ESTACK_ISEMPTY(s)) {
+	node = ESTACK_POP(s);
+	switch(node & _TAG_PRIMARY_MASK) {
+	case TAG_PRIMARY_LIST:
+	    while (is_list(node)) {
+		ESTACK_PUSH(s,CAR(list_val(node)));
+		node = CDR(list_val(node));
+	    }
+	    ESTACK_PUSH(s,node);    /* Non wellformed list or [] */
+	    break;
+	case TAG_PRIMARY_BOXED:
+	    if (is_tuple(node)) {
+		Eterm *tuple = tuple_val(node);
+		int arity = arityval(*tuple);
+		while(arity--) {
+		    ESTACK_PUSH(s,*(++tuple));
+		}
+	    }
+	    break;
+	case TAG_PRIMARY_IMMED1:
+	    if (node == am_Underscore || db_is_variable(node) >= 0) {
+		DESTROY_ESTACK(s);
 		return 1;
-	    obj = CDR(list_val(obj));
-	}
-	return(db_has_variable(obj));  /* Non wellformed list or [] */
-    }
-    case TAG_PRIMARY_BOXED: 
-	if (!is_tuple(obj)) {
-	    return 0;
-	} else {
-	    Eterm *tuple = tuple_val(obj);
-	    int arity = arityval(*tuple++);
-	    while(arity--) {
-		if (db_has_variable(*tuple))
-		    return 1;
-		tuple++;
 	    }
-	    return(0);
+	    break;
 	}
-    case TAG_PRIMARY_IMMED1:
-	if (obj == am_Underscore || db_is_variable(obj) >= 0)
-	    return 1;
     }
+    DESTROY_ESTACK(s);
     return 0;
 }
 
-- 
cgit v1.2.3


From b3e52c026ce4920e1a4e36ef98e5de94666e91ef Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 2 Dec 2014 17:37:13 +0100
Subject: erts: Add compile time assert ERTS_CT_ASSERT

and usage
---
 erts/emulator/beam/beam_emu.c        |  2 +-
 erts/emulator/beam/erl_alloc.c       |  2 +-
 erts/emulator/beam/erl_alloc_util.c  |  6 +++---
 erts/emulator/beam/erl_gc.c          | 18 +++++++++---------
 erts/emulator/beam/erl_instrument.c  |  2 +-
 erts/emulator/beam/erl_mtrace.c      |  2 +-
 erts/emulator/beam/erl_printf_term.c |  2 +-
 erts/emulator/beam/erl_process.c     |  3 +--
 erts/emulator/beam/erl_trace.c       |  2 +-
 erts/emulator/beam/io.c              |  4 ++--
 erts/emulator/beam/sys.h             | 16 ++++++++++++++++
 11 files changed, 37 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index e9f5fd798b..3af7c43abf 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5142,7 +5142,7 @@ get_map_elements_fail:
 #ifndef NO_JUMP_TABLE
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
      /* Are tables correctly generated by beam_makeops? */
-     ASSERT(sizeof(counting_opcodes) == sizeof(opcodes));
+     ERTS_CT_ASSERT(sizeof(counting_opcodes) == sizeof(opcodes));
 #ifdef DEBUG
      counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
 #endif
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 90cd227fae..f2bceff4eb 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3939,7 +3939,7 @@ static Uint
 install_debug_functions(void)
 {
     int i;
-    ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
+    ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
 
     sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
 
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index e3172dc4fb..2f277690e4 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1442,7 +1442,7 @@ get_pref_allctr(void *extra)
 
     pref_ix = ERTS_ALC_GET_THR_IX();
 
-    ASSERT(sizeof(UWord) == sizeof(Allctr_t *));
+    ERTS_CT_ASSERT(sizeof(UWord) == sizeof(Allctr_t *));
     ASSERT(0 <= pref_ix && pref_ix < tspec->size);
 
     return tspec->allctr[pref_ix];
@@ -1861,7 +1861,7 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	      * if this carrier is pulled from dc_list by cpool_fetch()
 	      */
 	    ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr);
-	    ERTS_ALC_CPOOL_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*));
+	    ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*));
 #ifdef MBC_ABLK_OFFSET_BITS
 	    blk->u.carrier = crr;
 #else
@@ -5942,7 +5942,7 @@ erts_alcu_init(AlcUInit_t *init)
 	erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel);
     }
 #endif
-    ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
+    ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
 #if HAVE_ERTS_MSEG
     ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ);
     max_mseg_carriers = init->mmc;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 5f78a7b532..d1a7ee113b 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -175,15 +175,15 @@ erts_init_gc(void)
     int i = 0, ix;
     Sint max_heap_size = 0;
 
-    ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
-    ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word));
-    ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header));
-    ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size));
-    ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size));
-    ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size));
-    ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next));
-    ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
-    ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
+    ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
+    ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word));
+    ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header));
+    ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size));
+    ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size));
+    ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size));
+    ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next));
+    ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
+    ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
 
     erts_test_long_gc_sleep = 0;
 
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index df7c443387..da85b86c87 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -1226,7 +1226,7 @@ erts_instr_init(int stat, int map_stat)
     mem_anchor = NULL;
 
     /* Install instrumentation functions */
-    ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
+    ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
 
     sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
 
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index c8bb126687..fa1bde1c87 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -627,7 +627,7 @@ erts_mtrace_install_wrapper_functions(void)
     if (erts_mtrace_enabled) {
 	int i;
 	/* Install trace functions */
-	ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
+	ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
 
 	sys_memcpy((void *) real_allctrs,
 		   (void *) erts_allctrs,
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 74e38c13df..c982dc2080 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -594,7 +594,7 @@ erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision,
 		 ErlPfEterm* term_base)
 {
     int res;
-    ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm));
+    ERTS_CT_ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm));
 
     res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base);
     if (res < 0)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b0e0cf13f8..08c03e7135 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2366,7 +2366,6 @@ erts_active_schedulers(void)
 
     ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting));
 
-    ASSERT(as >= 0);
     return as;
 }
 
@@ -10484,7 +10483,7 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
     init_arg.run_queue = rq;
     init_arg.state = state;
 
-    ASSERT(((char *) p) == ((char *) &p->common));
+    ERTS_CT_ASSERT(offsetof(Process,common) == 0);
 
     if (!erts_ptab_new_element(&erts_proc,
 			       &p->common,
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index ea5c850a30..eba07667ad 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2225,7 +2225,7 @@ trace_gc(Process *p, Eterm what)
     Eterm* limit;
 #endif
 
-    ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm));
+    ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm));
 
     UseTmpHeap(LOCAL_HEAP_SIZE,p);
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 4d262ff022..d46ac9b336 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -391,7 +391,7 @@ static Port *create_port(char *name,
     /* Set default tracing */
     erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt));
 
-    ASSERT(((char *) prt) == ((char *) &prt->common));
+    ERTS_CT_ASSERT(offsetof(Port,common) == 0);
 
 #if !ERTS_PORT_INIT_INSTR_NEED_ID
     /*
@@ -6698,7 +6698,7 @@ static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
 {
     RefThing *refp;
     ASSERT(is_internal_ref(ref));
-    ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor));
+    ERTS_CT_ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor));
     refp = ref_thing_ptr(ref);
     memset(mon,0,sizeof(ErlDrvMonitor));
     memcpy(mon,refp,sizeof(RefThing));
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c29d4b3777..7661bf2dc7 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -188,6 +188,22 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 #  define ASSERT(e) ((void) 1)
 #endif
 
+/*
+ * Compile time assert
+ * (the actual compiler error msg can be a bit confusing)
+ */
+#if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
+# define ERTS_CT_ASSERT(e) \
+    do { \
+	enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \
+    } while(0)
+#else
+# define ERTS_CT_ASSERT(e) \
+    do { \
+        enum { compile_time_assert__ = 1/(e) }; \
+    } while (0)
+#endif
+
 /*
  * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
  * But MSC doesn't define __STDC__, unless we compile with the -Za
-- 
cgit v1.2.3


From b9a8cebcc30ee1bce8e6b57041bf07cf84630386 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 2 Dec 2014 12:31:44 +0100
Subject: erts: Use linear search for small select_val arrays

For searching a key in an array we use linear search in arrays
up to 10 elements.

Selecting on tuple arity will always use linear search.

Instead of using two different instructions we assume selecting on
different tuple arities are always few in numbers.
---
 erts/emulator/beam/beam_debug.c |  42 +++++++--
 erts/emulator/beam/beam_emu.c   |  55 +++++++++---
 erts/emulator/beam/beam_load.c  | 184 ++++++++++++++++++++++++++++++----------
 erts/emulator/beam/ops.tab      |  22 +++--
 4 files changed, 226 insertions(+), 77 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index a3cd08834f..6bb987985d 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -579,9 +579,29 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     unpacked = ap;
     ap = addr + size;
     switch (op) {
-    case op_i_select_val_rfI:
-    case op_i_select_val_xfI:
-    case op_i_select_val_yfI:
+    case op_i_select_val_lins_rfI:
+    case op_i_select_val_lins_xfI:
+    case op_i_select_val_lins_yfI:
+	{
+	    int n = ap[-1];
+	    int ix = n;
+
+	    while (ix--) {
+		erts_print(to, to_arg, "%T ", (Eterm) ap[0]);
+		ap++;
+		size++;
+	    }
+	    ix = n;
+	    while (ix--) {
+		erts_print(to, to_arg, "f(" HEXF ") ", (Eterm) ap[0]);
+		ap++;
+		size++;
+	    }
+	}
+	break;
+    case op_i_select_val_bins_rfI:
+    case op_i_select_val_bins_xfI:
+    case op_i_select_val_bins_yfI:
 	{
 	    int n = ap[-1];
 
@@ -598,13 +618,19 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_select_tuple_arity_yfI:
 	{
 	    int n = ap[-1];
+	    int ix = n;
 
-	    while (n > 0) {
+	    while (ix--) {
 		Uint arity = arityval(ap[0]);
-		erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]);
-		ap += 2;
-		size += 2;
-		n--;
+		erts_print(to, to_arg, "{%d} ", arity, ap[1]);
+		ap++;
+		size++;
+	    }
+	    ix = n;
+	    while (ix--) {
+		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+		ap++;
+		size++;
 	    }
 	}
 	break;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index e9f5fd798b..292971a387 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2138,19 +2138,18 @@ void process_main(void)
      NextPF(0, next);
  }
 
-
  {
      Eterm select_val2;
 
- OpCase(i_select_tuple_arity2_yfAfAf):
+ OpCase(i_select_tuple_arity2_yfAAff):
      select_val2 = yb(Arg(0));
      goto do_select_tuple_arity2;
 
- OpCase(i_select_tuple_arity2_xfAfAf):
+ OpCase(i_select_tuple_arity2_xfAAff):
      select_val2 = xb(Arg(0));
      goto do_select_tuple_arity2;
 
- OpCase(i_select_tuple_arity2_rfAfAf):
+ OpCase(i_select_tuple_arity2_rfAAff):
      select_val2 = r(0);
      I--;
 
@@ -2161,22 +2160,22 @@ void process_main(void)
      select_val2 = *tuple_val(select_val2);
      goto do_select_val2;
 
- OpCase(i_select_val2_yfcfcf):
+ OpCase(i_select_val2_yfccff):
      select_val2 = yb(Arg(0));
      goto do_select_val2;
 
- OpCase(i_select_val2_xfcfcf):
+ OpCase(i_select_val2_xfccff):
      select_val2 = xb(Arg(0));
      goto do_select_val2;
 
- OpCase(i_select_val2_rfcfcf):
+ OpCase(i_select_val2_rfccff):
      select_val2 = r(0);
      I--;
 
  do_select_val2:
      if (select_val2 == Arg(2)) {
-	 I += 2;
-     } else if (select_val2 == Arg(4)) {
+	 I += 3;
+     } else if (select_val2 == Arg(3)) {
 	 I += 4;
      }
 
@@ -2203,20 +2202,50 @@ void process_main(void)
  do_select_tuple_arity:
      if (is_tuple(select_val)) {
 	 select_val = *tuple_val(select_val);
-	 goto do_binary_search;
+	 goto do_linear_search;
+     }
+     SET_I((BeamInstr *) Arg(1));
+     Goto(*I);
+
+ OpCase(i_select_val_lins_xfI):
+     select_val = xb(Arg(0));
+     goto do_linear_search;
+
+ OpCase(i_select_val_lins_yfI):
+     select_val = yb(Arg(0));
+     goto do_linear_search;
+
+ OpCase(i_select_val_lins_rfI):
+     select_val = r(0);
+     I--;
+
+ do_linear_search: {
+     BeamInstr *vs = &Arg(3);
+     int ix = 0;
+
+     for(;;) {
+	 if (vs[ix+0] >= select_val) { ix += 0; break; }
+	 if (vs[ix+1] >= select_val) { ix += 1; break; }
+	 ix += 2;
+     }
+
+     if (vs[ix] == select_val) {
+	 I += ix + Arg(2) + 2;
      }
+
      SET_I((BeamInstr *) Arg(1));
      Goto(*I);
+   }
 
- OpCase(i_select_val_xfI):
+ OpCase(i_select_val_bins_xfI):
      select_val = xb(Arg(0));
      goto do_binary_search;
 
- OpCase(i_select_val_yfI):
+ OpCase(i_select_val_bins_yfI):
      select_val = yb(Arg(0));
      goto do_binary_search;
      
- OpCase(i_select_val_rfI):
+ OpCase(i_select_val_bins_rfI):
      select_val = r(0);
      I--;
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 07654f6d5c..b9a6536ac6 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3321,9 +3321,10 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 
 {
     GenOp* op;
+    GenOpArg *tmp;
     int arity = Size.val + 3;
     int size = Size.val / 2;
-    int i;
+    int i, j, align = 0;
 
     /*
      * Verify the validity of the list.
@@ -3337,10 +3338,38 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	}
     }
 
+    /*
+     * Use a special-cased instruction if there are only two values.
+     */
+    if (size == 2) {
+	NEW_GENOP(stp, op);
+	op->next = NULL;
+	op->op = genop_i_select_tuple_arity2_6;
+	GENOP_ARITY(op, arity - 1);
+	op->a[0] = S;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val  = Rest[0].val;
+	op->a[3].type = TAG_u;
+	op->a[3].val  = Rest[2].val;
+	op->a[4] = Rest[1];
+	op->a[5] = Rest[3];
+
+	return op;
+    }
+
     /*
      * Generate the generic instruction.
+     * Assumption:
+     *   Few different tuple arities to select on (fewer than 20).
+     *   Use linear scan approach.
      */
 
+    align = 1;
+
+    arity += 2*align;
+    size  += align;
+
     NEW_GENOP(stp, op);
     op->next = NULL;
     op->op = genop_i_select_tuple_arity_3;
@@ -3348,39 +3377,36 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     op->a[0] = S;
     op->a[1] = Fail;
     op->a[2].type = TAG_u;
-    op->a[2].val = Size.val / 2;
-    for (i = 0; i < Size.val; i += 2) {
-	op->a[i+3].type = TAG_v;
-	op->a[i+3].val = make_arityval(Rest[i].val);
-	op->a[i+4] = Rest[i+1];
-    }
+    op->a[2].val = size;
 
-    /*
-     * Sort the values to make them useful for a binary search.
-     */
+    tmp = (GenOpArg *) erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(GenOpArg)*(arity-2*align));
 
-    qsort(op->a+3, size, 2*sizeof(GenOpArg), 
-	   (int (*)(const void *, const void *)) genopargcompare);
-#ifdef DEBUG
-    for (i = 3; i < arity-2; i += 2) {
-	ASSERT(op->a[i].val < op->a[i+2].val);
+    for (i = 3; i < arity - 2*align; i+=2) {
+	tmp[i-3].type = TAG_v;
+	tmp[i-3].val  = make_arityval(Rest[i-3].val);
+	tmp[i-2]      = Rest[i-2];
     }
-#endif
 
     /*
-     * Use a special-cased instruction if there are only two values.
+     * Sort the values to make them useful for a sentinel search
      */
-    if (size == 2) {
-	op->op = genop_i_select_tuple_arity2_6;
-	op->arity--;
-	op->a[2].type = TAG_u;
-	op->a[2].val = arityval(op->a[3].val);
-	op->a[3] = op->a[4];
-	op->a[4].type = TAG_u;
-	op->a[4].val = arityval(op->a[5].val);
-	op->a[5] = op->a[6];
+
+    qsort(tmp, size - align, 2*sizeof(GenOpArg),
+	    (int (*)(const void *, const void *)) genopargcompare);
+
+    j = 3;
+    for (i = 3; i < arity - 2*align; i += 2) {
+	op->a[j]        = tmp[i-3];
+	op->a[j + size] = tmp[i-2];
+	j++;
     }
 
+    erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp);
+
+    op->a[j].type = TAG_u;
+    op->a[j].val  = ~((BeamInstr)0);
+    op->a[j+size] = Fail;
+
     return op;
 }
 
@@ -3602,45 +3628,109 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	       GenOpArg Size, GenOpArg* Rest)
 {
     GenOp* op;
+    GenOpArg *tmp;
     int arity = Size.val + 3;
     int size = Size.val / 2;
-    int i;
+    int i, j, align = 0;
+
+    if (size == 2) {
+
+	/*
+	 * Use a special-cased instruction if there are only two values.
+	 */
+
+	NEW_GENOP(stp, op);
+	op->next = NULL;
+	op->op = genop_i_select_val2_6;
+	GENOP_ARITY(op, arity - 1);
+	op->a[0] = S;
+	op->a[1] = Fail;
+	op->a[2] = Rest[0];
+	op->a[3] = Rest[2];
+	op->a[4] = Rest[1];
+	op->a[5] = Rest[3];
+
+	return op;
+
+    } else if (size > 10) {
+
+	/* binary search instruction */
+
+	NEW_GENOP(stp, op);
+	op->next = NULL;
+	op->op = genop_i_select_val_bins_3;
+	GENOP_ARITY(op, arity);
+	op->a[0] = S;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val = size;
+	for (i = 3; i < arity; i++) {
+	    op->a[i] = Rest[i-3];
+	}
+
+	/*
+	 * Sort the values to make them useful for a binary search.
+	 */
+
+	qsort(op->a+3, size, 2*sizeof(GenOpArg),
+		(int (*)(const void *, const void *)) genopargcompare);
+#ifdef DEBUG
+	for (i = 3; i < arity-2; i += 2) {
+	    ASSERT(op->a[i].val < op->a[i+2].val);
+	}
+#endif
+	return op;
+    }
+
+    /* linear search instruction */
+
+    align = 1;
+
+    arity += 2*align;
+    size  += align;
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    op->op = genop_i_select_val_3;
+    op->op = genop_i_select_val_lins_3;
     GENOP_ARITY(op, arity);
     op->a[0] = S;
     op->a[1] = Fail;
     op->a[2].type = TAG_u;
     op->a[2].val = size;
-    for (i = 3; i < arity; i++) {
-	op->a[i] = Rest[i-3];
+
+    tmp = (GenOpArg *) erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(GenOpArg)*(arity-2*align));
+
+    for (i = 3; i < arity - 2*align; i++) {
+	tmp[i-3] = Rest[i-3];
     }
 
     /*
-     * Sort the values to make them useful for a binary search.
+     * Sort the values to make them useful for a sentinel search
      */
 
-    qsort(op->a+3, size, 2*sizeof(GenOpArg), 
-	  (int (*)(const void *, const void *)) genopargcompare);
-#ifdef DEBUG
-    for (i = 3; i < arity-2; i += 2) {
-	ASSERT(op->a[i].val < op->a[i+2].val);
+    qsort(tmp, size - align, 2*sizeof(GenOpArg),
+	    (int (*)(const void *, const void *)) genopargcompare);
+
+    j = 3;
+    for (i = 3; i < arity - 2*align; i += 2) {
+	op->a[j]      = tmp[i-3];
+	op->a[j+size] = tmp[i-2];
+	j++;
     }
-#endif
 
-    /*
-     * Use a special-cased instruction if there are only two values.
-     */
-    if (size == 2) {
-	op->op = genop_i_select_val2_6;
-	op->arity--;
-	op->a[2] = op->a[3];
-	op->a[3] = op->a[4];
-	op->a[4] = op->a[5];
-	op->a[5] = op->a[6];
+    erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp);
+
+    /* add sentinel */
+
+    op->a[j].type = TAG_u;
+    op->a[j].val  = ~((BeamInstr)0);
+    op->a[j+size] = Fail;
+
+#ifdef DEBUG
+    for (i = 0; i < size; i++) {
+	ASSERT(op->a[i+3].val <= op->a[i+4].val);
     }
+#endif
 
     return op;
 }
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 68fcc177ae..d3649080dc 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -166,22 +166,26 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
 select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-i_select_val r f I
-i_select_val x f I
-i_select_val y f I
+i_select_val_bins r f I
+i_select_val_bins x f I
+i_select_val_bins y f I
 
-i_select_val2 r f c f c f
-i_select_val2 x f c f c f
-i_select_val2 y f c f c f
+i_select_val_lins r f I
+i_select_val_lins x f I
+i_select_val_lins y f I
 
-i_select_tuple_arity2 r f A f A f
-i_select_tuple_arity2 x f A f A f
-i_select_tuple_arity2 y f A f A f
+i_select_val2 r f c c f f
+i_select_val2 x f c c f f
+i_select_val2 y f c c f f
 
 i_select_tuple_arity r f I
 i_select_tuple_arity x f I
 i_select_tuple_arity y f I
 
+i_select_tuple_arity2 r f A A f f
+i_select_tuple_arity2 x f A A f f
+i_select_tuple_arity2 y f A A f f
+
 i_jump_on_val_zero r f I
 i_jump_on_val_zero x f I
 i_jump_on_val_zero y f I
-- 
cgit v1.2.3


From 6e3057ddddb77835664f5264a7da62452dc3d9c1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 11 Sep 2014 18:26:00 +0200
Subject: erts: Allow cpu_timestamp tracing for Linux

---
 erts/emulator/beam/erl_bif_trace.c | 4 ++--
 erts/emulator/beam/erl_time.h      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 06fbbea123..f5e582b1c5 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -651,7 +651,7 @@ Eterm trace_3(BIF_ALIST_3)
 	    if (pid_spec == am_all) {
 		if (on) {
 		    if (!erts_cpu_timestamp) {
-#ifdef HAVE_CLOCK_GETTIME
+#ifdef HAVE_CLOCK_GETTIME_CPU_TIME
 			/* 
 			   Perhaps clock_gettime was found during config
 			   on a different machine than this. We check
@@ -678,7 +678,7 @@ Eterm trace_3(BIF_ALIST_3)
 			if (erts_start_now_cpu() < 0) {
 			    goto error;
 			}
-#endif /* HAVE_CLOCK_GETTIME */
+#endif /* HAVE_CLOCK_GETTIME_CPU_TIME */
 			erts_cpu_timestamp = !0;
 		    }
 		}
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 4bbdcaa3e3..7ed1a395ad 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -107,7 +107,7 @@ ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed)
 
 /* time_sup */
 
-#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME))
+#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
 #  ifndef HAVE_ERTS_NOW_CPU
 #    define HAVE_ERTS_NOW_CPU
 #    ifdef HAVE_GETHRVTIME
-- 
cgit v1.2.3


From 31b54a211eac951e3a0f2d1f5fe221d8296988ba Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 3 Dec 2014 22:45:35 +0100
Subject: Introduce a 64-bit atomic ops API

The 64-bit atomic ops API is implemented by
* native word size atomic ops on 64-bit architectures, and
* native double word size atomic ops on 32-bit architectures
  when available. When native double word size atomic is not
  available, the fallback using modification counters is
  used.
---
 erts/emulator/beam/erl_smp.h     | 222 +++++++++++++
 erts/emulator/beam/erl_threads.h | 679 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 901 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index c38ef47d87..6c40edeb3e 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -60,6 +60,7 @@ typedef erts_tsd_key_t erts_smp_tsd_key_t;
 #define erts_smp_dw_atomic_t erts_dw_atomic_t
 #define erts_smp_atomic_t erts_atomic_t
 #define erts_smp_atomic32_t erts_atomic32_t
+#define erts_smp_atomic64_t erts_atomic64_t
 typedef erts_spinlock_t erts_smp_spinlock_t;
 typedef erts_rwlock_t erts_smp_rwlock_t;
 void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
@@ -95,6 +96,7 @@ typedef int erts_smp_tsd_key_t;
 #define erts_smp_dw_atomic_t erts_no_dw_atomic_t
 #define erts_smp_atomic_t erts_no_atomic_t
 #define erts_smp_atomic32_t erts_no_atomic32_t
+#define erts_smp_atomic64_t erts_no_atomic64_t
 #if __GNUC__ > 2
 typedef struct { } erts_smp_spinlock_t;
 typedef struct { } erts_smp_rwlock_t;
@@ -489,6 +491,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
 #define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty
 
+/* 64-bit atomics */
+
+#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
+#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
+#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
+#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
+#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
+#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
+#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
+#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
+#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
+#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
+#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
+#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
+#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
+#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob
+
+#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
+#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
+#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
+#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
+#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
+#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
+#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
+#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
+#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
+#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
+#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
+#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
+#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
+#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb
+
+#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
+#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
+#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
+#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
+#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
+#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
+#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
+#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
+#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
+#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
+#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
+#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
+#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
+#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb
+
+#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
+#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
+#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
+#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
+#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
+#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
+#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
+#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
+#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
+#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
+#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
+#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
+#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
+#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb
+
+#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
+#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
+#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
+#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
+#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
+#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
+#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
+#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
+#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
+#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
+#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
+#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
+#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
+#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb
+
+#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
+#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
+#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
+#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
+#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
+#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
+#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
+#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
+#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
+#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
+#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
+#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
+#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
+#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb
+
+#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
+#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
+#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
+#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
+#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
+#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
+#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
+#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
+#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
+#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
+#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
+#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
+#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
+#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb
+
+#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
+#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty
+
 #else /* !ERTS_SMP */
 
 /* Double word size atomics */
@@ -751,6 +863,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_set_dirty erts_no_atomic32_set
 #define erts_smp_atomic32_read_dirty erts_no_atomic32_read
 
+/* 64-bit atomics */
+
+#define erts_smp_atomic64_init_nob erts_no_atomic64_set
+#define erts_smp_atomic64_set_nob erts_no_atomic64_set
+#define erts_smp_atomic64_read_nob erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_nob erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_mb erts_no_atomic64_set
+#define erts_smp_atomic64_set_mb erts_no_atomic64_set
+#define erts_smp_atomic64_read_mb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_mb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
+#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
+#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_relb erts_no_atomic64_set
+#define erts_smp_atomic64_set_relb erts_no_atomic64_set
+#define erts_smp_atomic64_read_relb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_relb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
+#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
+#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_rb erts_no_atomic64_set
+#define erts_smp_atomic64_set_rb erts_no_atomic64_set
+#define erts_smp_atomic64_read_rb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_rb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_wb erts_no_atomic64_set
+#define erts_smp_atomic64_set_wb erts_no_atomic64_set
+#define erts_smp_atomic64_read_wb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_wb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
+#define erts_smp_atomic64_read_dirty erts_no_atomic64_read
+
 #endif /* !ERTS_SMP */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 80026104db..7214f3ea33 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -344,6 +344,16 @@ typedef ethr_ts_event erts_tse_t;
 #define erts_aint32_t ethr_sint32_t 
 #define erts_atomic32_t ethr_atomic32_t
 
+#if defined(ARCH_32)
+#  define erts_atomic64_t ethr_dw_atomic_t
+#  define erts_aint64_t ethr_sint64_t
+#elif defined(ARCH_64)
+#  define erts_atomic64_t ethr_atomic_t
+#  define erts_aint64_t ethr_sint_t
+#else
+#  error "Not supported architecture"
+#endif
+
 #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
 #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
 
@@ -414,10 +424,12 @@ typedef int erts_tse_t;
 typedef struct { SWord sint[2]; } erts_dw_aint_t;
 typedef SWord erts_aint_t;
 typedef Sint32 erts_aint32_t;
+typedef Sint64 erts_aint64_t;
 
 #define erts_dw_atomic_t erts_dw_aint_t
 #define erts_atomic_t erts_aint_t
 #define erts_atomic32_t erts_aint32_t
+#define erts_atomic64_t erts_aint64_t
 
 #if __GNUC__ > 2
 typedef struct { } erts_spinlock_t;
@@ -446,6 +458,7 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t;
 #define erts_no_dw_atomic_t erts_dw_aint_t
 #define erts_no_atomic_t erts_aint_t
 #define erts_no_atomic32_t erts_aint32_t
+#define erts_no_atomic64_t erts_aint64_t
 
 #define ERTS_AINT_NULL ((erts_aint_t) NULL)
 
@@ -570,6 +583,29 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp
 ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
 							 erts_aint32_t mask,
 							 erts_aint32_t set);
+ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var,
+					  erts_aint64_t i);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp,
+							erts_aint64_t i);
+ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp,
+					  erts_aint64_t i);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var,
+							erts_aint64_t mask);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var,
+							 erts_aint64_t mask);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp,
+						    erts_aint64_t new);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
+						       erts_aint64_t new,
+						       erts_aint64_t expected);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
+							 erts_aint64_t mask,
+							 erts_aint64_t set);
 
 ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
 					      char *name,
@@ -1200,6 +1236,441 @@ erts_atomic32_read_dirty(erts_atomic32_t *var)
 
 #endif
 
+/* 64-bit atomics */
+
+#if defined(ARCH_64)
+
+#define erts_atomic64_init_nob ethr_atomic_init
+#define erts_atomic64_set_nob ethr_atomic_set
+#define erts_atomic64_read_nob ethr_atomic_read
+#define erts_atomic64_inc_read_nob ethr_atomic_inc_read
+#define erts_atomic64_dec_read_nob ethr_atomic_dec_read
+#define erts_atomic64_inc_nob ethr_atomic_inc
+#define erts_atomic64_dec_nob ethr_atomic_dec
+#define erts_atomic64_add_read_nob ethr_atomic_add_read
+#define erts_atomic64_add_nob ethr_atomic_add
+#define erts_atomic64_read_bor_nob ethr_atomic_read_bor
+#define erts_atomic64_read_band_nob ethr_atomic_read_band
+#define erts_atomic64_xchg_nob ethr_atomic_xchg
+#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg
+#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob
+
+#define erts_atomic64_init_mb ethr_atomic_init_mb
+#define erts_atomic64_set_mb ethr_atomic_set_mb
+#define erts_atomic64_read_mb ethr_atomic_read_mb
+#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb
+#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb
+#define erts_atomic64_inc_mb ethr_atomic_inc_mb
+#define erts_atomic64_dec_mb ethr_atomic_dec_mb
+#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb
+#define erts_atomic64_add_mb ethr_atomic_add_mb
+#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb
+#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb
+#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb
+#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb
+#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb
+
+#define erts_atomic64_init_acqb ethr_atomic_init_acqb
+#define erts_atomic64_set_acqb ethr_atomic_set_acqb
+#define erts_atomic64_read_acqb ethr_atomic_read_acqb
+#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb
+#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb
+#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb
+#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb
+#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb
+#define erts_atomic64_add_acqb ethr_atomic_add_acqb
+#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb
+#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb
+#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb
+#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
+#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb
+
+#define erts_atomic64_init_relb ethr_atomic_init_relb
+#define erts_atomic64_set_relb ethr_atomic_set_relb
+#define erts_atomic64_read_relb ethr_atomic_read_relb
+#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb
+#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb
+#define erts_atomic64_inc_relb ethr_atomic_inc_relb
+#define erts_atomic64_dec_relb ethr_atomic_dec_relb
+#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb
+#define erts_atomic64_add_relb ethr_atomic_add_relb
+#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb
+#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb
+#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb
+#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb
+#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb
+
+#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb
+#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb
+#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb
+#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb
+#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb
+#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb
+#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb
+#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb
+#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb
+#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb
+#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb
+#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb
+#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb
+
+#define erts_atomic64_init_rb ethr_atomic_init_rb
+#define erts_atomic64_set_rb ethr_atomic_set_rb
+#define erts_atomic64_read_rb ethr_atomic_read_rb
+#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb
+#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb
+#define erts_atomic64_inc_rb ethr_atomic_inc_rb
+#define erts_atomic64_dec_rb ethr_atomic_dec_rb
+#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb
+#define erts_atomic64_add_rb ethr_atomic_add_rb
+#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb
+#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb
+#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb
+#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb
+#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb
+
+#define erts_atomic64_init_wb ethr_atomic_init_wb
+#define erts_atomic64_set_wb ethr_atomic_set_wb
+#define erts_atomic64_read_wb ethr_atomic_read_wb
+#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb
+#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb
+#define erts_atomic64_inc_wb ethr_atomic_inc_wb
+#define erts_atomic64_dec_wb ethr_atomic_dec_wb
+#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb
+#define erts_atomic64_add_wb ethr_atomic_add_wb
+#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb
+#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb
+#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb
+#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb
+#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb
+
+#define erts_atomic64_set_dirty erts_atomic_set_dirty
+#define erts_atomic64_read_dirty erts_atomic_read_dirty
+
+#elif defined(ARCH_32)
+
+#undef ERTS_ATOMIC64_OPS_DECL__
+
+#define ERTS_ATOMIC64_OPS_DECL__(BARRIER)				\
+ERTS_GLB_INLINE void							\
+erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val);			\
+ERTS_GLB_INLINE void							\
+erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var);		\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var);		\
+ERTS_GLB_INLINE void							\
+erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var);			\
+ERTS_GLB_INLINE void							\
+erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val);			\
+ERTS_GLB_INLINE void							\
+erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
+				  erts_aint64_t new,			\
+				  erts_aint64_t exp);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t mask,			\
+				    erts_aint64_t set)
+
+ERTS_ATOMIC64_OPS_DECL__(nob);
+ERTS_ATOMIC64_OPS_DECL__(mb);
+ERTS_ATOMIC64_OPS_DECL__(acqb);
+ERTS_ATOMIC64_OPS_DECL__(relb);
+ERTS_ATOMIC64_OPS_DECL__(ddrb);
+ERTS_ATOMIC64_OPS_DECL__(rb);
+ERTS_ATOMIC64_OPS_DECL__(wb);
+
+#undef ERTS_ATOMIC64_OPS_DECL__
+
+ERTS_GLB_INLINE void
+erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val);
+ERTS_GLB_INLINE erts_aint64_t
+erts_atomic64_read_dirty(erts_atomic64_t *var);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+/*
+ * The ethr_dw_atomic_*_nob() functions below
+ * are here to make it possible for the
+ * ERTS_ATOMIC64_OPS_IMPL__() to map erts
+ * barriers to ethread barriers...
+ */
+static ERTS_INLINE void
+ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var,
+			ethr_dw_sint_t *val)
+{
+    ethr_dw_atomic_init(var, val);
+}
+
+static ERTS_INLINE void
+ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var,
+		       ethr_dw_sint_t *val)
+{
+    ethr_dw_atomic_set(var, val);
+}
+
+static ERTS_INLINE void
+ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var,
+			ethr_dw_sint_t *val)
+{
+    ethr_dw_atomic_read(var, val);
+}
+
+static ERTS_INLINE int
+ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var,
+			   ethr_dw_sint_t *new,
+			   ethr_dw_sint_t *xchg)
+{
+    return ethr_dw_atomic_cmpxchg(var, new, xchg);
+}
+
+#undef ERTS_ATOMIC64_OPS_IMPL__
+#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
+#undef ERTS_DW_SINT_TO_AINT64__
+#undef ERTS_AINT64_TO_DW_SINT__
+
+#ifdef ETHR_SU_DW_NAINT_T__
+#define ERTS_DW_SINT_TO_AINT64__(DW)					\
+    ((erts_aint64_t) DW.dw_sint)
+#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
+    (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64)
+#else /* !ETHR_SU_DW_NAINT_T__ */
+#define ERTS_DW_SINT_TO_AINT64__(DW)					\
+    ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32)		\
+     | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD])		\
+	& ((erts_aint64_t) 0xffffffff)))
+#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
+    do {								\
+	DW.sint[ETHR_DW_SINT_LOW_WORD] =				\
+	    (ethr_sint_t) (AINT64 & 0xffffffff);			\
+	DW.sint[ETHR_DW_SINT_HIGH_WORD] = 				\
+	    (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff);		\
+    } while (0)
+#endif /* !ETHR_SU_DW_NAINT_T__ */
+
+#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, 			\
+					AVarP, XchgVar, NewVar, 	\
+					ModificationCode)		\
+do {									\
+    ethr_dw_sint_t dw_xchg__, dw_new__;					\
+    ethr_dw_atomic_read(AVarP, &dw_xchg__);				\
+    do {								\
+	XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__);			\
+	{								\
+	    ModificationCode;						\
+	}								\
+	ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar);			\
+    } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__));		\
+} while (0)
+
+#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER)				\
+    									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val)			\
+{									\
+    ethr_dw_sint_t dw;							\
+    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
+    ethr_dw_atomic_init_ ## BARRIER(var, &dw);				\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val)			\
+{									\
+    ethr_dw_sint_t dw;							\
+    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
+    ethr_dw_atomic_set_ ## BARRIER(var, &dw);				\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var)			\
+{									\
+    ethr_dw_sint_t dw;							\
+    ethr_dw_atomic_read_ ## BARRIER(var, &dw);				\
+    return ERTS_DW_SINT_TO_AINT64__(dw);				\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var)		\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + 1);			\
+    return new;								\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var)		\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg - 1);			\
+    return new;								\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + 1);			\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg - 1);			\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + val);			\
+    return new;								\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + val);			\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg | val);			\
+    return xchg;							\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg & val);			\
+    return xchg;							\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = val);				\
+    return xchg;							\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
+				  erts_aint64_t new,			\
+				  erts_aint64_t exp)			\
+{									\
+    ethr_dw_sint_t dw_xchg, dw_new;					\
+    ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp);				\
+    ERTS_AINT64_TO_DW_SINT__(dw_new, new);				\
+    if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg))	\
+	return exp;							\
+    return ERTS_DW_SINT_TO_AINT64__(dw_xchg);				\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t mask,			\
+				    erts_aint64_t set)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    {					\
+					new = xchg & ~mask;		\
+					new |= mask & set;		\
+				    });					\
+    return xchg;							\
+}
+
+ERTS_ATOMIC64_OPS_IMPL__(nob)
+ERTS_ATOMIC64_OPS_IMPL__(mb)
+ERTS_ATOMIC64_OPS_IMPL__(acqb)
+ERTS_ATOMIC64_OPS_IMPL__(relb)
+ERTS_ATOMIC64_OPS_IMPL__(ddrb)
+ERTS_ATOMIC64_OPS_IMPL__(rb)
+ERTS_ATOMIC64_OPS_IMPL__(wb)
+
+#undef ERTS_ATOMIC64_OPS_IMPL__
+#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
+
+ERTS_GLB_INLINE void
+erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val)
+{
+    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
+    ethr_dw_sint_t dw;
+    ERTS_AINT64_TO_DW_SINT__(dw, val);
+    sint[0] = dw.sint[0];
+    sint[1] = dw.sint[1];
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_atomic64_read_dirty(erts_atomic64_t *var)
+{
+    ethr_sint_t *sint;
+    ethr_dw_sint_t dw;
+    sint = ethr_dw_atomic_addr(var);
+    dw.sint[0] = sint[0];
+    dw.sint[1] = sint[1];
+    return ERTS_DW_SINT_TO_AINT64__(dw);
+}
+
+#undef ERTS_DW_SINT_TO_AINT64__
+#undef ERTS_AINT64_TO_DW_SINT__
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ARCH_32 */
+
 #else /* !USE_THREADS */
 
 /* Double word size atomics */
@@ -1462,6 +1933,116 @@ erts_atomic32_read_dirty(erts_atomic32_t *var)
 #define erts_atomic32_set_dirty erts_no_atomic32_set
 #define erts_atomic32_read_dirty erts_no_atomic32_read
 
+/* 64-bit atomics */
+
+#define erts_atomic64_init_nob erts_no_atomic64_set
+#define erts_atomic64_set_nob erts_no_atomic64_set
+#define erts_atomic64_read_nob erts_no_atomic64_read
+#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read
+#define erts_atomic64_inc_nob erts_no_atomic64_inc
+#define erts_atomic64_dec_nob erts_no_atomic64_dec
+#define erts_atomic64_add_read_nob erts_no_atomic64_add_read
+#define erts_atomic64_add_nob erts_no_atomic64_add
+#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_nob erts_no_atomic64_read_band
+#define erts_atomic64_xchg_nob erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_mb erts_no_atomic64_set
+#define erts_atomic64_set_mb erts_no_atomic64_set
+#define erts_atomic64_read_mb erts_no_atomic64_read
+#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_mb erts_no_atomic64_inc
+#define erts_atomic64_dec_mb erts_no_atomic64_dec
+#define erts_atomic64_add_read_mb erts_no_atomic64_add_read
+#define erts_atomic64_add_mb erts_no_atomic64_add
+#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_mb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_mb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_acqb erts_no_atomic64_set
+#define erts_atomic64_set_acqb erts_no_atomic64_set
+#define erts_atomic64_read_acqb erts_no_atomic64_read
+#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_acqb erts_no_atomic64_inc
+#define erts_atomic64_dec_acqb erts_no_atomic64_dec
+#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read
+#define erts_atomic64_add_acqb erts_no_atomic64_add
+#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_relb erts_no_atomic64_set
+#define erts_atomic64_set_relb erts_no_atomic64_set
+#define erts_atomic64_read_relb erts_no_atomic64_read
+#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_relb erts_no_atomic64_inc
+#define erts_atomic64_dec_relb erts_no_atomic64_dec
+#define erts_atomic64_add_read_relb erts_no_atomic64_add_read
+#define erts_atomic64_add_relb erts_no_atomic64_add
+#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_relb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_relb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_ddrb erts_no_atomic64_set
+#define erts_atomic64_set_ddrb erts_no_atomic64_set
+#define erts_atomic64_read_ddrb erts_no_atomic64_read
+#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_ddrb erts_no_atomic64_inc
+#define erts_atomic64_dec_ddrb erts_no_atomic64_dec
+#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read
+#define erts_atomic64_add_ddrb erts_no_atomic64_add
+#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_rb erts_no_atomic64_set
+#define erts_atomic64_set_rb erts_no_atomic64_set
+#define erts_atomic64_read_rb erts_no_atomic64_read
+#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_rb erts_no_atomic64_inc
+#define erts_atomic64_dec_rb erts_no_atomic64_dec
+#define erts_atomic64_add_read_rb erts_no_atomic64_add_read
+#define erts_atomic64_add_rb erts_no_atomic64_add
+#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_rb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_rb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_wb erts_no_atomic64_set
+#define erts_atomic64_set_wb erts_no_atomic64_set
+#define erts_atomic64_read_wb erts_no_atomic64_read
+#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_wb erts_no_atomic64_inc
+#define erts_atomic64_dec_wb erts_no_atomic64_dec
+#define erts_atomic64_add_read_wb erts_no_atomic64_add_read
+#define erts_atomic64_add_wb erts_no_atomic64_add
+#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_wb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_wb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset
+
+#define erts_atomic64_set_dirty erts_no_atomic64_set
+#define erts_atomic64_read_dirty erts_no_atomic64_read
+
 #endif /* !USE_THREADS */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -2383,6 +2964,104 @@ erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
     return old;
 }
 
+/* atomic64 */
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i)
+{
+    *var = i;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read(erts_no_atomic64_t *var)
+{
+    return *var;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_inc_read(erts_no_atomic64_t *incp)
+{
+    return ++(*incp);
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_dec_read(erts_no_atomic64_t *decp)
+{
+    return --(*decp);
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_inc(erts_no_atomic64_t *incp)
+{
+    ++(*incp);
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_dec(erts_no_atomic64_t *decp)
+{
+    --(*decp);
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i)
+{
+    return *addp += i;
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i)
+{
+    *addp += i;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask)
+{
+    erts_aint64_t old;
+    old = *var;
+    *var |= mask;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask)
+{
+    erts_aint64_t old;
+    old = *var;
+    *var &= mask;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new)
+{
+    erts_aint64_t old = *xchgp;
+    *xchgp = new;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
+			 erts_aint64_t new,
+			 erts_aint64_t expected)
+{
+    erts_aint64_t old = *xchgp;
+    if (old == expected)
+        *xchgp = new;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
+			   erts_aint64_t mask,
+			   erts_aint64_t set)
+{
+    erts_aint64_t old = *var;
+    *var &= ~mask;
+    *var |= (mask & set);
+    return old;
+}
+
 /* spinlock */
 
 ERTS_GLB_INLINE void
-- 
cgit v1.2.3


From 843cf9120d2845be2c3304fc8dcb4c16b49bbfca Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 3 Dec 2014 22:50:28 +0100
Subject: Use the new 64-bit atomic ops API

---
 erts/emulator/beam/erl_process.c      |  54 +--------------
 erts/emulator/beam/erl_process.h      |   8 +--
 erts/emulator/beam/erl_ptab.c         | 100 ++-------------------------
 erts/emulator/beam/erl_ptab.h         |   6 +-
 erts/emulator/beam/erl_thr_progress.c |  52 +-------------
 erts/emulator/beam/erl_thr_progress.h |  60 ++--------------
 erts/emulator/beam/erl_utils.h        |  43 +-----------
 erts/emulator/beam/utils.c            | 124 ++--------------------------------
 8 files changed, 29 insertions(+), 418 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f84677dea4..ea63d20dfa 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -716,72 +716,24 @@ sched_wall_time_ts(void)
 
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
 
-#ifdef ARCH_64
-
-static ERTS_INLINE Uint64
-aschedtime_read(ErtsAtomicSchedTime *var)
-{
-    return (Uint64) erts_atomic_read_nob((erts_atomic_t *) var);
-}
-
-static ERTS_INLINE void
-aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val)
-{
-    erts_atomic_set_nob((erts_atomic_t *) var, (erts_aint_t) val);
-}
-
-static ERTS_INLINE void
-aschedtime_init(ErtsAtomicSchedTime *var)
-{
-    erts_atomic_init_nob((erts_atomic_t *) var, (erts_aint_t) 0);
-}
-
-#elif defined(ARCH_32)
-
 static ERTS_INLINE Uint64
 aschedtime_read(ErtsAtomicSchedTime *var)
 {
-    erts_dw_aint_t dw;
-    erts_dw_atomic_read_nob((erts_dw_atomic_t *) var, &dw);
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (Uint64) dw.dw_sint;
-#else
-    {
-	Uint64 res;
-	res = (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_HIGH_WORD]);
-	res <<= 32;
-	res |= (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_LOW_WORD]);
-	return res;
-    }
-#endif    
+    return (Uint64) erts_atomic64_read_nob((erts_atomic64_t *) var);
 }
 
 static ERTS_INLINE void
 aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val)
 {
-    erts_dw_aint_t dw;
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw.dw_sint = (ETHR_SU_DW_NAINT_T__) val;
-#else
-    dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
-    dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
-#endif
-    erts_dw_atomic_set_nob((erts_dw_atomic_t *) var, &dw);
+    erts_atomic64_set_nob((erts_atomic64_t *) var, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 aschedtime_init(ErtsAtomicSchedTime *var)
 {
-    erts_dw_aint_t dw;
-    dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) 0;
-    dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) 0;
-    erts_dw_atomic_init_nob((erts_dw_atomic_t *) var, &dw);
+    erts_atomic64_init_nob((erts_atomic64_t *) var, (erts_aint64_t) 0);
 }
 
-#else
-#  error :-/
-#endif
-
 #define ERTS_GET_AVG_MAX_UNLOCKED_TRY 50
 #define ERTS_SCHED_AVG_UTIL_WRITE_MARKER (~((Uint64) 0))
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 27a3a3553b..f50b217d4a 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -351,13 +351,7 @@ typedef struct {
 #undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
 
-#ifdef ARCH_64
-typedef erts_atomic_t ErtsAtomicSchedTime;
-#elif defined(ARCH_32)
-typedef erts_dw_atomic_t ErtsAtomicSchedTime;
-#else
-# error :-/
-#endif
+typedef erts_atomic64_t ErtsAtomicSchedTime;
 
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
 typedef struct {
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index eabf016081..02943ee683 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -280,124 +280,38 @@ struct ErtsPTabListBifData_ {
 
 };
 
-#ifdef ARCH_32
-
-static ERTS_INLINE Uint64
-dw_aint_to_uint64(erts_dw_aint_t *dw)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (Uint64) dw->dw_sint;
-#else
-    Uint64 res;
-    res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
-    res <<= 32;
-    res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
-    return res;
-#endif
-}
-
-static void
-unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
-#else
-    dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
-    dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
-#endif
-}
-
 static ERTS_INLINE void
 last_data_init_nob(ErtsPTab *ptab, Uint64 val)
 {
-    erts_dw_aint_t dw;
-    unint64_to_dw_aint(&dw, val);
-    erts_smp_dw_atomic_init_nob(&ptab->vola.tile.last_data, &dw);
+    erts_smp_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 last_data_set_relb(ErtsPTab *ptab, Uint64 val)
 {
-    erts_dw_aint_t dw;
-    unint64_to_dw_aint(&dw, val);
-    erts_smp_dw_atomic_set_relb(&ptab->vola.tile.last_data, &dw);
+    erts_smp_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE Uint64
 last_data_read_nob(ErtsPTab *ptab)
 {
-    erts_dw_aint_t dw;
-    erts_smp_dw_atomic_read_nob(&ptab->vola.tile.last_data, &dw);
-    return dw_aint_to_uint64(&dw);
+    return (Uint64) erts_smp_atomic64_read_nob(&ptab->vola.tile.last_data);
 }
 
 static ERTS_INLINE Uint64
 last_data_read_acqb(ErtsPTab *ptab)
 {
-    erts_dw_aint_t dw;
-    erts_smp_dw_atomic_read_acqb(&ptab->vola.tile.last_data, &dw);
-    return dw_aint_to_uint64(&dw);
+    return (Uint64) erts_smp_atomic64_read_acqb(&ptab->vola.tile.last_data);
 }
 
 static ERTS_INLINE Uint64
 last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp)
 {
-    erts_dw_aint_t dw_new, dw_xchg;
-
-    unint64_to_dw_aint(&dw_new, new);
-    unint64_to_dw_aint(&dw_xchg, exp);
-
-    if (erts_smp_dw_atomic_cmpxchg_relb(&ptab->vola.tile.last_data,
-					&dw_new,
-					&dw_xchg))
-	return exp;
-    else
-	return dw_aint_to_uint64(&dw_xchg);
-}
-
-#elif defined(ARCH_64)
-
-union {
-    erts_smp_atomic_t pid_data;
-    char align[ERTS_CACHE_LINE_SIZE];
-} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
-
-static ERTS_INLINE void
-last_data_init_nob(ErtsPTab *ptab, Uint64 val)
-{
-    erts_smp_atomic_init_nob(&ptab->vola.tile.last_data, (erts_aint_t) val);
+    return (Uint64) erts_smp_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data,
+						   (erts_aint64_t) new,
+						   (erts_aint64_t) exp);
 }
 
-static ERTS_INLINE void
-last_data_set_relb(ErtsPTab *ptab, Uint64 val)
-{
-    erts_smp_atomic_set_relb(&ptab->vola.tile.last_data, (erts_aint_t) val);
-}
-
-static ERTS_INLINE Uint64
-last_data_read_nob(ErtsPTab *ptab)
-{
-    return (Uint64) erts_smp_atomic_read_nob(&ptab->vola.tile.last_data);
-}
-
-static ERTS_INLINE Uint64
-last_data_read_acqb(ErtsPTab *ptab)
-{
-    return (Uint64) erts_smp_atomic_read_acqb(&ptab->vola.tile.last_data);
-}
-
-static ERTS_INLINE Uint64
-last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp)
-{
-    return (Uint64) erts_smp_atomic_cmpxchg_relb(&ptab->vola.tile.last_data,
-						 (erts_aint_t) new,
-						 (erts_aint_t) exp);
-}
-
-#else
-#  error "Not 64-bit, nor 32-bit architecture..."
-#endif
-
 static ERTS_INLINE int
 last_data_cmp(Uint64 ld1, Uint64 ld2)
 {
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index e3e05f14af..876241159b 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -88,11 +88,7 @@ typedef struct {
 } ErtsPTabListData;
 
 typedef struct {
-#ifdef ARCH_32
-    erts_smp_dw_atomic_t last_data;
-#else
-    erts_smp_atomic_t last_data;
-#endif
+    erts_smp_atomic64_t last_data;
     erts_smp_atomic32_t count;
     erts_smp_atomic32_t aid_ix;
     erts_smp_atomic32_t fid_ix;
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 545a0343d0..664c479eb6 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -115,70 +115,24 @@
 #undef read_nob
 #define read_nob erts_thr_prgr_read_nob__
 
-#ifdef ARCH_64
-
 static ERTS_INLINE void
 set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    erts_atomic_set_mb(atmc, val);
+    erts_atomic64_set_mb(atmc, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    erts_atomic_set_nob(atmc, val);
+    erts_atomic64_set_nob(atmc, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    erts_atomic_init_nob(atmc, val);
-}
-
-#else
-
-#undef dw_aint_to_val
-#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__
-
-static void
-val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
-#else
-    dw_aint->sint[ERTS_DW_AINT_LOW_WORD]
-	= (erts_aint_t) (val & 0xffffffff);
-    dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]
-	= (erts_aint_t) ((val >> 32) & 0xffffffff);
-#endif
+    erts_atomic64_init_nob(atmc, (erts_aint64_t) val);
 }
 
-static ERTS_INLINE void
-set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
-{
-    erts_dw_aint_t dw_aint;
-    val_to_dw_aint(&dw_aint, val);
-    erts_dw_atomic_set_mb(atmc, &dw_aint);
-}
-
-static ERTS_INLINE void
-set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
-{
-    erts_dw_aint_t dw_aint;
-    val_to_dw_aint(&dw_aint, val);
-    erts_dw_atomic_set_nob(atmc, &dw_aint);
-}
-
-static ERTS_INLINE void
-init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
-{
-    erts_dw_aint_t dw_aint;
-    val_to_dw_aint(&dw_aint, val);
-    erts_dw_atomic_init_nob(atmc, &dw_aint);
-}
-
-#endif
-
 /* #define ERTS_THR_PROGRESS_STATE_DEBUG */
 
 #ifdef ERTS_THR_PROGRESS_STATE_DEBUG
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 5f392944c2..03ddbd467c 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -115,11 +115,7 @@ struct ErtsThrPrgrLaterOp_ {
 
 extern erts_tsd_key_t erts_thr_prgr_data_key__;
 
-#ifdef ARCH_64
-#  define ERTS_THR_PRGR_ATOMIC erts_atomic_t
-#else /* ARCH_32 */
-#  define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
-#endif
+#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t
 
 typedef struct {
     void *arg;
@@ -158,10 +154,6 @@ void erts_thr_progress_unmanaged_continue__(int umrefc_ix);
 
 void erts_thr_progress_dbg_print_state(void);
 
-#ifdef ARCH_32
-#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
-ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint);
-#endif
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
@@ -184,68 +176,24 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-#ifdef ARCH_64
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
-}
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc);
-}
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc);
-}
-
-#else /* ARCH_32 */
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (ErtsThrPrgrVal) dw_aint->dw_sint;
-#else
-    ErtsThrPrgrVal res;
-    res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]);
-    res <<= 32;
-    res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]);
-    return res;
-#endif
-}
-
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    erts_dw_aint_t dw_aint;
-    erts_dw_atomic_read_nob(atmc, &dw_aint);
-    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    erts_dw_aint_t dw_aint;
-    erts_dw_atomic_read_acqb(atmc, &dw_aint);
-    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    erts_dw_aint_t dw_aint;
-    erts_dw_atomic_read_mb(atmc, &dw_aint);
-    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
 }
 
-#endif
-
 ERTS_GLB_INLINE int
 erts_thr_progress_is_managed_thread(void)
 {
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 0807649ea1..c32f8fd61c 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -32,11 +32,7 @@ typedef struct {
 #endif
     union {
 	Uint64 not_atomic;
-#ifdef ARCH_64
-	erts_atomic_t atomic;
-#else
-	erts_dw_atomic_t atomic;
-#endif
+	erts_atomic64_t atomic;
     } counter;
 } erts_interval_t;
 
@@ -50,9 +46,6 @@ Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64);
 Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64);
 Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64);
 Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64);
-#ifdef ARCH_32
-ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *);
-#endif
 ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *);
 ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *);
 ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *);
@@ -62,46 +55,16 @@ ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-#ifdef ARCH_32
-
-ERTS_GLB_INLINE Uint64
-erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (Uint64) dw->dw_sint;
-#else
-    Uint64 res;
-    res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
-    res <<= 32;
-    res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
-    return res;
-#endif
-}
-
-#endif
-
 ERTS_GLB_INLINE Uint64
 erts_current_interval_nob__(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
-#else
-    erts_dw_aint_t dw;
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &dw);
-    return erts_interval_dw_aint_to_val__(&dw);
-#endif
+    return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
 }
 
 ERTS_GLB_INLINE Uint64
 erts_current_interval_acqb__(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
-#else
-    erts_dw_aint_t dw;
-    erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw);
-    return erts_interval_dw_aint_to_val__(&dw);
-#endif
+    return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
 }
 
 ERTS_GLB_INLINE Uint64
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f20e6e5665..c505c44905 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -4216,19 +4216,7 @@ void erts_silence_warn_unused_result(long unused)
 void
 erts_interval_init(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    erts_atomic_init_nob(&icp->counter.atomic, 0);
-#else
-    erts_dw_aint_t dw;
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw.dw_sint = 0;
-#else
-    dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0;
-    dw.sint[ERTS_DW_AINT_LOW_WORD] = 0;
-#endif
-    erts_dw_atomic_init_nob(&icp->counter.atomic, &dw);
-
-#endif
+    erts_atomic64_init_nob(&icp->counter.atomic, 0);
 #ifdef DEBUG
     icp->smp_api = 0;
 #endif
@@ -4250,55 +4238,13 @@ erts_smp_interval_init(erts_interval_t *icp)
 static ERTS_INLINE Uint64
 step_interval_nob(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic);
 }
 
 static ERTS_INLINE Uint64
 step_interval_relb(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_relb(&icp->counter.atomic);
 }
 
 
@@ -4306,38 +4252,10 @@ static ERTS_INLINE Uint64
 ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
 {
     Uint64 curr_ic;
-#ifdef ARCH_64
-    curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
+    curr_ic = (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
     if (curr_ic > ic)
 	return curr_ic;
-    return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
-    curr_ic = erts_interval_dw_aint_to_val__(&exp);
-    if (curr_ic > ic)
-	return curr_ic;
-
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-	curr_ic = erts_interval_dw_aint_to_val__(&exp);
-	if (curr_ic > ic)
-	    return curr_ic;
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic);
 }
 
 
@@ -4345,38 +4263,10 @@ static ERTS_INLINE Uint64
 ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 {
     Uint64 curr_ic;
-#ifdef ARCH_64
-    curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
+    curr_ic = (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
     if (curr_ic > ic)
 	return curr_ic;
-    return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp);
-    curr_ic = erts_interval_dw_aint_to_val__(&exp);
-    if (curr_ic > ic)
-	return curr_ic;
-
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-	curr_ic = erts_interval_dw_aint_to_val__(&exp);
-	if (curr_ic > ic)
-	    return curr_ic;
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_acqb(&icp->counter.atomic);
 }
 
 Uint64
-- 
cgit v1.2.3


From 3aa7023f2e4f5454faddd663f00ee4c935f9b8f6 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 4 Dec 2014 11:09:28 +0100
Subject: ets: Increase data available in crash dumps and ets:info

OTP-12376
---
 erts/emulator/beam/erl_db.c      | 15 +++++++++--
 erts/emulator/beam/erl_db_hash.c | 55 +++++++++++++++++++++++++---------------
 erts/emulator/beam/erl_db_hash.h |  1 +
 3 files changed, 48 insertions(+), 23 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 8f246ffa07..0d75dbbc04 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2643,7 +2643,9 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 BIF_RETTYPE ets_info_1(BIF_ALIST_1)
 {
     static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table,
-	am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed};
+                             am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed,
+                             am_write_concurrency,
+                             am_read_concurrency};
     Eterm results[sizeof(fields)/sizeof(Eterm)];
     DbTable* tb;
     Eterm res;
@@ -3670,6 +3672,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    ret = am_protected;
 	else if (tb->common.status & DB_PUBLIC)
 	    ret = am_public;
+    } else if (What == am_write_concurrency) {
+        ret = tb->common.status & DB_FINE_LOCKED ? am_true : am_false;
+    } else if (What == am_read_concurrency) {
+        ret = tb->common.status & DB_FREQ_READ ? am_true : am_false;
     } else if (What == am_name) {
 	ret = tb->common.the_name;
     } else if (What == am_keypos) {
@@ -3752,7 +3758,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 			 avg, std_dev_real, std_dev_exp,
 			 make_small(stats.min_chain_len),
 			 make_small(stats.max_chain_len),
-			 make_small(db_kept_items_hash(&tb->hash)));
+			 make_small(stats.kept_items));
 	}
 	else {
 	    ret = am_false;
@@ -3774,6 +3780,11 @@ static void print_table(int to, void *to_arg, int show,  DbTable* tb)
 			+ sizeof(Uint)
 			- 1)
 		       / sizeof(Uint)));
+    erts_print(to, to_arg, "Type: %T\n", table_info(NULL, tb, am_type));
+    erts_print(to, to_arg, "Protection: %T\n", table_info(NULL, tb, am_protection));
+    erts_print(to, to_arg, "Compressed: %T\n", table_info(NULL, tb, am_compressed));
+    erts_print(to, to_arg, "Write Concurrency: %T\n", table_info(NULL, tb, am_write_concurrency));
+    erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency));
 }
 
 void db_info(int to, void *to_arg, int show)    /* Called by break handler */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 06dac8f161..82d1f894ae 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -646,25 +646,6 @@ restart:
     /* ToDo: Maybe try grow/shrink the table as well */
 }
 
-/* Only used by tests
-*/
-Uint db_kept_items_hash(DbTableHash *tb)
-{
-    Uint kept_items = 0;
-    Uint ix = 0;
-    erts_smp_rwmtx_t* lck = RLOCK_HASH(tb,ix);
-    HashDbTerm* b;
-    do {
-	for (b = BUCKET(tb, ix); b != NULL; b = b->next) {
-	    if (b->hvalue == INVALID_HASH) {
-		++kept_items;
-	    }
-	}	
-	ix = next_slot(tb, ix, &lck);
-    }while (ix);
-    return kept_items;
-}
-
 int db_create_hash(Process *p, DbTable *tbl)
 {
     DbTableHash *tb = &tbl->hash;
@@ -2104,10 +2085,38 @@ int db_mark_all_deleted_hash(DbTable *tbl)
 static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl)
 {
     DbTableHash *tb = &tbl->hash;
+    DbHashStats stats;
     int i;
 
     erts_print(to, to_arg, "Buckets: %d\n", NACTIVE(tb));
-    
+
+#ifdef ERTS_SMP
+    i = tbl->common.is_thread_safe;
+    /* If crash dumping we set table to thread safe in order to
+       avoid taking any locks */
+    if (ERTS_IS_CRASH_DUMPING)
+        tbl->common.is_thread_safe = 1;
+
+    db_calc_stats_hash(&tbl->hash, &stats);
+
+    tbl->common.is_thread_safe = i;
+#else
+    db_calc_stats_hash(&tbl->hash, &stats);
+#endif
+
+    erts_print(to, to_arg, "Chain Length Avg: %f\n", stats.avg_chain_len);
+    erts_print(to, to_arg, "Chain Length Max: %d\n", stats.max_chain_len);
+    erts_print(to, to_arg, "Chain Length Min: %d\n", stats.min_chain_len);
+    erts_print(to, to_arg, "Chain Length Std Dev: %f\n",
+               stats.std_dev_chain_len);
+    erts_print(to, to_arg, "Chain Length Expected Std Dev: %f\n",
+               stats.std_dev_expected);
+
+    if (IS_FIXED(tb))
+        erts_print(to, to_arg, "Fixed: %d\n", stats.kept_items);
+    else
+        erts_print(to, to_arg, "Fixed: false\n");
+
     if (show) {
 	for (i = 0; i < NACTIVE(tb); i++) {
 	    HashDbTerm* list = BUCKET(tb,i);
@@ -2833,6 +2842,7 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats)
     erts_smp_rwmtx_t* lck;
     int sum = 0;
     int sq_sum = 0;
+    int kept_items = 0;
     int ix;
     int len;
     
@@ -2844,6 +2854,8 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats)
 	len = 0;
 	for (b = BUCKET(tb,ix); b!=NULL; b=b->next) {
 	    len++;
+            if (b->hvalue == INVALID_HASH)
+                ++kept_items;
 	}
 	sum += len;
 	sq_sum += len*len;
@@ -2855,7 +2867,8 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats)
     stats->std_dev_chain_len = sqrt((sq_sum - stats->avg_chain_len*sum) / NACTIVE(tb));
     /* Expected	standard deviation from a good uniform hash function, 
        ie binomial distribution (not taking the linear hashing into acount) */
-    stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb)));	
+    stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb)));
+    stats->kept_items = kept_items;
 }
 #ifdef HARDDEBUG
 
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index e68081a5b1..f12cd363b0 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -104,6 +104,7 @@ typedef struct {
     float std_dev_expected;
     int max_chain_len;
     int min_chain_len;
+    int kept_items;
 }DbHashStats;
 
 void db_calc_stats_hash(DbTableHash* tb, DbHashStats*);
-- 
cgit v1.2.3


From f03bce6a77ff5c7885a3b200fe879210299194bb Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 4 Dec 2014 11:00:22 +0100
Subject: erts: Add support for thread names

---
 erts/emulator/beam/erl_async.c      | 13 +++----------
 erts/emulator/beam/erl_drv_thread.c |  7 ++-----
 erts/emulator/beam/erl_process.c    | 31 +++++++------------------------
 erts/emulator/beam/erl_threads.h    | 11 +++++++++++
 erts/emulator/beam/erl_trace.c      |  5 +----
 5 files changed, 24 insertions(+), 43 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index decae6b2ca..bc06d41720 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -176,7 +176,7 @@ erts_init_async(void)
 	ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
 #endif
 	erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
-	char *ptr;
+	char *ptr, thr_name[16];
 	size_t tot_size = 0;
 	int i;
 
@@ -227,23 +227,16 @@ erts_init_async(void)
 	thr_opts.suggested_stack_size
 	    = erts_async_thread_suggested_stack_size;
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-	thr_opts.name = malloc(sizeof(char)*(strlen("async_XXXX")+1));
-#endif
+	thr_opts.name = thr_name;
 
 	for (i = 0; i < erts_async_max_threads; i++) {
 	    ErtsAsyncQ *aq = async_q(i);
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-            sprintf(thr_opts.name, "async_%d", i+1);
-#endif
+            erts_snprintf(thr_opts.name, 16, "async_%d", i+1);
 
 	    erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts);
 	}
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-	free(thr_opts.name);
-#endif
 	/* Wait for async threads to initialize... */
 
 	erts_mtx_lock(&async->init.data.mtx);
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 147249f751..31b05d22af 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -601,17 +601,14 @@ erl_drv_thread_create(char *name,
 #ifdef USE_THREADS
     int res;
     struct ErlDrvTid_ *dtid;
-    ethr_thr_opts ethr_opts;
+    ethr_thr_opts ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
     ethr_thr_opts *use_opts;
-    ethr_thr_opts def_ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
 
     if (!opts)
 	use_opts = NULL;
     else {
-	sys_memcpy((void *) &ethr_opts,
-		   (void *) &def_ethr_opts,
-		   sizeof(ethr_thr_opts));
 	ethr_opts.suggested_stack_size = opts->suggested_stack_size;
+        ethr_opts.name = name;
 	use_opts = &ethr_opts;
     }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b0e0cf13f8..31dda21b47 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7883,23 +7883,17 @@ erts_start_schedulers(void)
     Uint actual;
     Uint wanted = erts_no_schedulers;
     Uint wanted_no_schedulers = erts_no_schedulers;
+    char name[16];
     ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER;
 
     opts.detached = 1;
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-    opts.name = malloc(80);
-    if (!opts.name) {
-        ERTS_INTERNAL_ERROR("malloc failed to allocate memory!");
-    }
-#endif
+    opts.name = name;
 
 #ifdef ERTS_SMP
     if (erts_runq_supervision_interval) {
 	opts.suggested_stack_size = 16;
-#ifdef ETHR_HAVE_THREAD_NAMES
-        sprintf(opts.name, "runq_supervisor");
-#endif
+        erts_snprintf(opts.name, 16, "runq_supervisor");
 	erts_atomic_init_nob(&runq_supervisor_sleeping, 0);
 	if (0 != ethr_event_init(&runq_supervision_event))
 	    erl_exit(1, "Failed to create run-queue supervision event\n");
@@ -7926,9 +7920,7 @@ erts_start_schedulers(void)
 
 	ASSERT(actual == esdp->no - 1);
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-	sprintf(opts.name, "scheduler_%d", actual + 1);
-#endif
+	erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1);
 
 #ifdef __OSE__
         /* This should be done in the bind strategy */
@@ -7950,18 +7942,14 @@ erts_start_schedulers(void)
 	int ix;
 	for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
-#ifdef ETHR_HAVE_THREAD_NAMES
-            sprintf(opts.name,"dirty_cpu_scheduler_%d", ix + 1);
-#endif
+	    erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1);
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
 		erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix);
 	}
 	for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
-#ifdef ETHR_HAVE_THREAD_NAMES
-            sprintf(opts.name,"dirty_io_scheduler_%d", ix + 1);
-#endif
+	    erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1);
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
 		erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix);
@@ -7972,9 +7960,7 @@ erts_start_schedulers(void)
 
     ERTS_THR_MEMORY_BARRIER;
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-    sprintf(opts.name, "aux");
-#endif
+    erts_snprintf(opts.name, 16, "aux");
 
 #ifdef __OSE__
     opts.coreNo = 0;
@@ -8000,9 +7986,6 @@ erts_start_schedulers(void)
 	erts_send_error_to_logger_nogl(dsbufp);
     }
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-    free(opts.name);
-#endif
 }
 
 #endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 80026104db..1ab5418741 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -463,6 +463,7 @@ ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid);
 ERTS_GLB_INLINE void erts_thr_exit(void *res);
 ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
+ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
 ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
 				     int enable_lcnt);
@@ -1548,6 +1549,16 @@ erts_thr_self(void)
 #endif
 }
 
+ERTS_GLB_INLINE int
+erts_thr_getname(erts_tid_t tid, char *buf, size_t len)
+{
+#ifdef USE_THREADS
+    return ethr_getname(tid, buf, len);
+#else
+    return -1;
+#endif
+}
+
 
 ERTS_GLB_INLINE int
 erts_equal_tids(erts_tid_t x, erts_tid_t y)
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index ea5c850a30..15648d6056 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3492,16 +3492,13 @@ init_sys_msg_dispatcher(void)
     thr_opts.coreNo   = 0;
 #endif
     thr_opts.detached = 1;
+    thr_opts.name = "sys_msg_dispatcher";
     init_smq_element_alloc();
     sys_message_queue = NULL;
     sys_message_queue_end = NULL;
     erts_smp_cnd_init(&smq_cnd);
     erts_smp_mtx_init(&smq_mtx, "sys_msg_q");
 
-#ifdef ETHR_HAVE_THREAD_NAMES
-    thr_opts.name = "sys_msg_dispatcher";
-#endif
-
     erts_smp_thr_create(&sys_msg_dispatcher_tid,
 			sys_msg_dispatcher_func,
 			NULL,
-- 
cgit v1.2.3


From b8a2313263d0f120dacbe82ced5c99545bbd15a3 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 4 Dec 2014 11:04:23 +0100
Subject: erts: Introduce thread suspend functions

These functions allow any thread to suspend any other thread
immediately and then resume all threads. This is useful when
doing a crash dump in order to get a more accurate picture
of what state the system is in.
---
 erts/emulator/beam/erl_threads.h | 15 +++++++++++++++
 erts/emulator/beam/sys.h         |  5 +++++
 2 files changed, 20 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 1ab5418741..463271252d 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -621,11 +621,17 @@ ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
 ERTS_GLB_INLINE int erts_thr_get_main_status(void);
 ERTS_GLB_INLINE void erts_thr_yield(void);
 
+
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 #define ERTS_THR_HAVE_SIG_FUNCS 1
 ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
 				      sigset_t *oset);
 ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
+
+#ifdef USE_THREADS
+ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig);
+#endif
+
 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
 
 #ifdef USE_THREADS
@@ -2848,6 +2854,15 @@ ERTS_GLB_INLINE void erts_thr_yield(void)
 
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 
+ERTS_GLB_INLINE void
+erts_thr_kill(erts_tid_t tid, int sig) {
+#ifdef USE_THREADS
+  int res = ethr_kill((ethr_tid)tid, sig);
+  if (res)
+    erts_thr_fatal_error(res, "killing thread");
+#endif
+}
+
 ERTS_GLB_INLINE void
 erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
 {
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c29d4b3777..bdc5237815 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -784,6 +784,11 @@ int erts_sys_unsetenv(char *key);
 char *erts_read_env(char *key);
 void erts_free_read_env(void *value);
 
+#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
+extern void sys_thr_resume(erts_tid_t tid);
+extern void sys_thr_suspend(erts_tid_t tid);
+#endif
+
 /* utils.c */
 
 /* Options to sys_alloc_opt */
-- 
cgit v1.2.3


From ffd0153e6dffcc29cf79d0191860047dba0438bb Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 4 Dec 2014 11:09:06 +0100
Subject: erts: Improve crash dumps

This commit improves crash dumps in several ways:
  *  Suspends schedulers to get a current snapshot
  *  Dumps information about scheduler
  *  Dumps stack trace of current running process
     (including Garbing processes)
---
 erts/emulator/beam/break.c            | 128 +++++++++++++++-----
 erts/emulator/beam/erl_bif_info.c     |   7 ++
 erts/emulator/beam/erl_process.c      | 215 +++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_process.h      |  13 ++
 erts/emulator/beam/erl_process_dump.c | 174 ++++++++++++++++++++++++++-
 erts/emulator/beam/erl_thr_progress.c |  42 +++----
 erts/emulator/beam/erl_thr_progress.h |   4 +-
 erts/emulator/beam/sys.h              |   2 +
 8 files changed, 526 insertions(+), 59 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 5aee85174f..41765dad12 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -210,25 +210,12 @@ print_process_info(int to, void *to_arg, Process *p)
     erts_print(to, to_arg, "State: ");
 
     state = erts_smp_atomic32_read_acqb(&p->state);
-    if (state & ERTS_PSFLG_FREE)
-	erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */
-    else if (state & ERTS_PSFLG_EXITING)
-	erts_print(to, to_arg, "Exiting\n");
-    else if (state & ERTS_PSFLG_GC) {
-	garbing = 1;
-	running = 1;
-	erts_print(to, to_arg, "Garbing\n");
-    }
-    else if (state & ERTS_PSFLG_SUSPENDED)
-	erts_print(to, to_arg, "Suspended\n");
-    else if (state & ERTS_PSFLG_RUNNING) {
-	running = 1;
-	erts_print(to, to_arg, "Running\n");
-    }
-    else if (state & ERTS_PSFLG_ACTIVE)
-	erts_print(to, to_arg, "Scheduled\n");
-    else
-	erts_print(to, to_arg, "Waiting\n");
+    erts_dump_process_state(to, to_arg, state);
+    if (state & ERTS_PSFLG_GC) {
+        garbing = 1;
+        running = 1;
+    } else if (state & ERTS_PSFLG_RUNNING)
+        running = 1;
 
     /*
      * If the process is registered as a global process, display the
@@ -352,6 +339,10 @@ print_process_info(int to, void *to_arg, Process *p)
 #endif
 	    erts_stack_dump(to, to_arg, p);
     }
+
+    /* Display all states */
+    erts_print(to, to_arg, "Internal State: ");
+    erts_dump_extended_process_state(to, to_arg, state);
 }
 
 static void
@@ -671,6 +662,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 {
 #ifdef ERTS_SMP
     ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
+    int bc;
 #endif
     int fd;
     size_t envsz;
@@ -681,27 +673,39 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     char* dumpname;
     int secs;
     int env_erl_crash_dump_seconds_set = 1;
+    int i;
 
     if (ERTS_SOMEONE_IS_CRASH_DUMPING)
 	return;
 
 #ifdef ERTS_SMP
+    /* Order all managed threads to block, this has to be done
+       first to guarantee that this is the only thread to generate
+       crash dump. */
+    bc = erts_thr_progress_fatal_error_block(&tpd_buf);
+
+#ifdef ERTS_THR_HAVE_SIG_FUNCS
     /*
-     * Wait for all managed threads to block. If all threads haven't blocked
-     * after a minute, we go anyway and hope for the best...
-     *
-     * We do not release system again. We expect an exit() or abort() after
-     * dump has been written.
+     * We suspend all scheduler threads so that we can dump some
+     * data about the currently running processes and scheduler data.
+     * We have to be very very careful when doing this as the schedulers
+     * could be anywhere.
      */
-    erts_thr_progress_fatal_error_block(60000, &tpd_buf);
-    /* Either worked or not... */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid;
+        if (!erts_equal_tids(tid,erts_thr_self()))
+            sys_thr_suspend(tid);
+    }
+
+#endif
 
     /* Allow us to pass certain places without locking... */
     erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1);
     erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1);
-#else
+
+#else /* !ERTS_SMP */
     erts_writing_erl_crash_dump = 1;
-#endif
+#endif /* ERTS_SMP */
 
     envsz = sizeof(env);
     /* ERL_CRASH_DUMP_SECONDS not set
@@ -758,9 +762,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_fprintf(stderr,"\nCrash dump is being written to: %s...", dumpname);
 
     fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640);
-    if (fd < 0) 
+    if (fd < 0)
 	return; /* Can't create the crash dump, skip it */
-    
     time(&now);
     erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now));
 
@@ -774,9 +777,74 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_fdprintf(fd, "System version: ");
     erts_print_system_version(fd, NULL, NULL);
     erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE);
+
     erts_fdprintf(fd, "Taints: ");
     erts_print_nif_taints(fd, NULL);
     erts_fdprintf(fd, "Atoms: %d\n", atom_table_size());
+
+#ifdef USE_THREADS
+    /* We want to note which thread it was that called erl_exit */
+    if (erts_get_scheduler_data()) {
+        erts_fdprintf(fd, "Calling Thread: scheduler:%d\n",
+                      erts_get_scheduler_data()->no);
+    } else {
+        if (!erts_thr_getname(erts_thr_self(), dumpnamebuf, MAXPATHLEN))
+            erts_fdprintf(fd, "Calling Thread: %s\n", dumpnamebuf);
+        else
+            erts_fdprintf(fd, "Calling Thread: %p\n", erts_thr_self());
+    }
+#else
+    erts_fdprintf(fd, "Calling Thread: scheduler:1\n");
+#endif
+
+#if defined(ERTS_HAVE_TRY_CATCH)
+
+    /*
+     * erts_print_scheduler_info is not guaranteed to be safe to call
+     * here for all schedulers as we may have suspended a scheduler
+     * in the middle of updating the STACK_TOP and STACK_START
+     * variables and thus when scanning the stack we could get
+     * segmentation faults. We protect against this very unlikely
+     * scenario by using the ERTS_SYS_TRY_CATCH.
+     */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        ERTS_SYS_TRY_CATCH(
+            erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)),
+            erts_fdprintf(fd, "** crashed **\n"));
+    }
+#endif
+
+#ifdef ERTS_SMP
+
+#if defined(ERTS_THR_HAVE_SIG_FUNCS)
+
+    /* We resume all schedulers so that we are in a known safe state
+       when we write the rest of the crash dump */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid;
+        if (!erts_equal_tids(tid,erts_thr_self()))
+	    sys_thr_resume(tid);
+    }
+#endif
+
+    /*
+     * Wait for all managed threads to block. If all threads haven't blocked
+     * after a minute, we go anyway and hope for the best...
+     *
+     * We do not release system again. We expect an exit() or abort() after
+     * dump has been written.
+     */
+    erts_thr_progress_fatal_error_wait(60000);
+    /* Either worked or not... */
+#endif
+
+#ifndef ERTS_HAVE_TRY_CATCH
+    /* This is safe to call here, as all schedulers are blocked */
+    for (i = 0; i < erts_no_schedulers; i++) {
+        erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i));
+    }
+#endif
+    
     info(fd, NULL); /* General system info */
     if (erts_ptab_initialized(&erts_proc))
 	process_info(fd, NULL); /* Info about each process and port */
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d2ee5e4224..d750e34be3 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3900,6 +3900,13 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		}
 	    }
 	}
+        else if (ERTS_IS_ATOM_STR("broken_halt", BIF_ARG_1)) {
+            /* Ugly ugly code used by bif_SUITE:erlang_halt/1 */
+#if defined(ERTS_HAVE_TRY_CATCH)
+	    erts_get_scheduler_data()->run_queue = NULL;
+#endif
+            erl_exit(ERTS_DUMP_EXIT, "%T", BIF_ARG_2);
+        }
     }
 
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 31dda21b47..1ec931e312 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -457,8 +457,7 @@ do {									\
 
 static void exec_misc_ops(ErtsRunQueue *);
 static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
-static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
-			      int yreg);
+static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg);
 
 static void aux_work_timeout(void *unused);
 static void aux_work_timeout_early_init(int no_schedulers);
@@ -12186,7 +12185,7 @@ erts_stack_dump(int to, void *to_arg, Process *p)
     }
     erts_program_counter_info(to, to_arg, p);
     for (sp = p->stop; sp < STACK_START(p); sp++) {
-        yreg = stack_element_dump(to, to_arg, p, sp, yreg);
+        yreg = stack_element_dump(to, to_arg, sp, yreg);
     }
 }
 
@@ -12243,7 +12242,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 }
 
 static int
-stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
+stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
 {
     Eterm x = *sp;
 
@@ -12271,6 +12270,214 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
     return yreg;
 }
 
+/*
+ * Print scheduler information
+ */
+void
+erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
+    int i;
+    erts_aint32_t flg;
+    Process *p;
+
+    erts_print(to, to_arg, "=scheduler:%u\n", esdp->no);
+
+#ifdef ERTS_SMP
+    flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags);
+    erts_print(to, to_arg, "Scheduler Sleep Info Flags: ");
+    for (i = 0; i < ERTS_SSI_FLGS_MAX && flg; i++) {
+        erts_aint32_t chk = (1 << i);
+        if (flg & chk) {
+            switch (chk) {
+            case ERTS_SSI_FLG_SLEEPING:
+                erts_print(to, to_arg, "SLEEPING"); break;
+            case ERTS_SSI_FLG_POLL_SLEEPING:
+                erts_print(to, to_arg, "POLL_SLEEPING"); break;
+            case ERTS_SSI_FLG_TSE_SLEEPING:
+                erts_print(to, to_arg, "TSE_SLEEPING"); break;
+            case ERTS_SSI_FLG_WAITING:
+                erts_print(to, to_arg, "WAITING"); break;
+            case ERTS_SSI_FLG_SUSPENDED:
+                erts_print(to, to_arg, "SUSPENDED"); break;
+            default:
+                erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
+            }
+            if (flg > chk)
+                erts_print(to, to_arg, " | ");
+            flg -= chk;
+        }
+    }
+    erts_print(to, to_arg, "\n");
+#endif
+
+    flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work);
+    erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: ");
+    for (i = 0; i < ERTS_SSI_AUX_WORK_MAX && flg; i++) {
+        erts_aint32_t chk = (1 << i);
+        if (flg & chk) {
+            switch (chk) {
+            case ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP:
+                erts_print(to, to_arg, "DELAYED_AW_WAKEUP"); break;
+            case ERTS_SSI_AUX_WORK_DD:
+                erts_print(to, to_arg, "DELAYED_DEALLOC"); break;
+            case ERTS_SSI_AUX_WORK_DD_THR_PRGR:
+                erts_print(to, to_arg, "DELAYED_DEALLOC_THR_PRGR"); break;
+            case ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC:
+                erts_print(to, to_arg, "FIX_ALLOC_DEALLOC"); break;
+            case ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM:
+                erts_print(to, to_arg, "FIX_ALLOC_LOWER_LIM"); break;
+            case ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP:
+                erts_print(to, to_arg, "THR_PRGR_LATER_OP"); break;
+            case ERTS_SSI_AUX_WORK_ASYNC_READY:
+                erts_print(to, to_arg, "ASYNC_READY"); break;
+            case ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN:
+                erts_print(to, to_arg, "ASYNC_READY_CLEAN"); break;
+            case ERTS_SSI_AUX_WORK_MISC_THR_PRGR:
+                erts_print(to, to_arg, "MISC_THR_PRGR"); break;
+            case ERTS_SSI_AUX_WORK_MISC:
+                erts_print(to, to_arg, "MISC"); break;
+            case ERTS_SSI_AUX_WORK_CHECK_CHILDREN:
+                erts_print(to, to_arg, "CHECK_CHILDREN"); break;
+            case ERTS_SSI_AUX_WORK_SET_TMO:
+                erts_print(to, to_arg, "SET_TMO"); break;
+            case ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK:
+                erts_print(to, to_arg, "MSEG_CACHE_CHECK"); break;
+            case ERTS_SSI_AUX_WORK_REAP_PORTS:
+                erts_print(to, to_arg, "REAP_PORTS"); break;
+            default:
+                erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
+            }
+            if (flg > chk)
+                erts_print(to, to_arg, " | ");
+            flg -= chk;
+        }
+    }
+    erts_print(to, to_arg, "\n");
+
+    erts_print(to, to_arg, "Current Port: ");
+    if (esdp->current_port)
+        erts_print(to, to_arg, "%T", esdp->current_port->common.id);
+    erts_print(to, to_arg, "\n");
+
+    p = esdp->current_process;
+    erts_print(to, to_arg, "Current Process: ");
+    if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) {
+      flg = erts_smp_atomic32_read_dirty(&p->state);
+      erts_print(to, to_arg, "%T\n", p->common.id);
+      
+      erts_print(to, to_arg, "Current Process State: ");
+      erts_dump_process_state(to, to_arg, flg);
+      
+      erts_print(to, to_arg, "Current Process Internal State: ");
+      erts_dump_extended_process_state(to, to_arg, flg);
+      
+      erts_print(to, to_arg, "Current Process Program counter: %p (", p->i);
+      print_function_from_pc(to, to_arg, p->i);
+      erts_print(to, to_arg, ")\n");
+      erts_print(to, to_arg, "Current Process CP: %p (", p->cp);
+      print_function_from_pc(to, to_arg, p->cp);
+      erts_print(to, to_arg, ")\n");
+      
+      /* Getting this stacktrace can segfault if we are very very
+	 unlucky if called while a process is being garbage collected.
+	 Therefore we only call this on other schedulers if we either
+	 have protection against segfaults, or we know that the process
+	 is not garbage collecting. It *should* always be safe to call
+	 on a process owned by us, even if it is currently being garbage
+	 collected.
+      */
+      erts_print(to, to_arg, "Current Process Limited Stack Trace:\n");
+      erts_limited_stack_trace(to, to_arg, p);
+    } else
+      erts_print(to, to_arg, "\n");
+
+    for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) {
+        erts_print(to, to_arg, "Run Queue ");
+        switch (i) {
+        case PRIORITY_MAX:
+            erts_print(to, to_arg, "Max ");
+            break;
+        case PRIORITY_HIGH:
+            erts_print(to, to_arg, "High ");
+            break;
+        case PRIORITY_NORMAL:
+            erts_print(to, to_arg, "Normal ");
+            break;
+        case PRIORITY_LOW:
+            erts_print(to, to_arg, "Low ");
+            break;
+        default:
+            erts_print(to, to_arg, "Unknown ");
+            break;
+        }
+        erts_print(to, to_arg, "Length: %d\n",
+                   erts_smp_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len));
+    }
+    erts_print(to, to_arg, "Run Queue Port Length: %d\n",
+               erts_smp_atomic32_read_dirty(&esdp->run_queue->ports.info.len));
+
+    flg = erts_smp_atomic32_read_dirty(&esdp->run_queue->flags);
+    erts_print(to, to_arg, "Run Queue Flags: ");
+    for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) {
+        erts_aint32_t chk = (1 << i);
+        if (flg & chk) {
+            switch (chk) {
+            case (1 << PRIORITY_MAX):
+                erts_print(to, to_arg, "NONEMPTY_MAX"); break;
+            case (1 << PRIORITY_HIGH):
+                erts_print(to, to_arg, "NONEMPTY_HIGH"); break;
+            case (1 << PRIORITY_NORMAL):
+                erts_print(to, to_arg, "NONEMPTY_NORMAL"); break;
+            case (1 << PRIORITY_LOW):
+                erts_print(to, to_arg, "NONEMPTY_LOW"); break;
+            case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)):
+                erts_print(to, to_arg, "EMIGRATE_MAX"); break;
+            case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)):
+                erts_print(to, to_arg, "EMIGRATE_HIGH"); break;
+            case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)):
+                erts_print(to, to_arg, "EMIGRATE_NORMAL"); break;
+            case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)):
+                erts_print(to, to_arg, "EMIGRATE_LOW"); break;
+            case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)):
+                erts_print(to, to_arg, "IMMIGRATE_MAX"); break;
+            case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)):
+                erts_print(to, to_arg, "IMMIGRATE_HIGH"); break;
+            case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)):
+                erts_print(to, to_arg, "IMMIGRATE_NORMAL"); break;
+            case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)):
+                erts_print(to, to_arg, "IMMIGRATE_LOW"); break;
+            case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EVACUATE_SHFT)):
+                erts_print(to, to_arg, "EVACUATE_MAX"); break;
+            case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EVACUATE_SHFT)):
+                erts_print(to, to_arg, "EVACUATE_HIGH"); break;
+            case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EVACUATE_SHFT)):
+                erts_print(to, to_arg, "EVACUATE_NORMAL"); break;
+            case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EVACUATE_SHFT)):
+                erts_print(to, to_arg, "EVACUATE_LOW"); break;
+            case ERTS_RUNQ_FLG_OUT_OF_WORK:
+                erts_print(to, to_arg, "OUT_OF_WORK"); break;
+            case ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK:
+                erts_print(to, to_arg, "HALFTIME_OUT_OF_WORK"); break;
+            case ERTS_RUNQ_FLG_SUSPENDED:
+                erts_print(to, to_arg, "SUSPENDED"); break;
+            case ERTS_RUNQ_FLG_CHK_CPU_BIND:
+                erts_print(to, to_arg, "CHK_CPU_BIND"); break;
+            case ERTS_RUNQ_FLG_INACTIVE:
+                erts_print(to, to_arg, "INACTIVE"); break;
+            case ERTS_RUNQ_FLG_NONEMPTY:
+                erts_print(to, to_arg, "NONEMPTY"); break;
+            case ERTS_RUNQ_FLG_PROTECTED:
+                erts_print(to, to_arg, "PROTECTED"); break;
+            default:
+                erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
+            }
+            if (flg > chk)
+                erts_print(to, to_arg, " | ");
+            flg -= chk;
+        }
+    }
+    erts_print(to, to_arg, "\n");
+}
+
 /*
  * A nice system halt closing all open port goes as follows:
  * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 3d08be25ff..a12b577656 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -170,6 +170,8 @@ extern int erts_sched_thread_suggested_stack_size;
 #define ERTS_RUNQ_FLG_PROTECTED \
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
 
+#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 7)
+
 #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
   (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
    | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
@@ -252,6 +254,8 @@ typedef enum {
 #define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
 #define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)
 
+#define ERTS_SSI_FLGS_MAX                                       5
+
 #define ERTS_SSI_FLGS_SLEEP_TYPE			\
  (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)
 
@@ -283,6 +287,8 @@ typedef enum {
 #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 12)
 #define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 13)
 
+#define ERTS_SSI_AUX_WORK_MAX                                           14
+
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -1081,6 +1087,9 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
 #define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
 #define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
+#else
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
 #endif
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
@@ -1616,7 +1625,11 @@ void erts_cleanup_empty_process(Process* p);
 void erts_debug_verify_clean_empty_process(Process* p);
 #endif
 void erts_stack_dump(int to, void *to_arg, Process *);
+void erts_limited_stack_trace(int to, void *to_arg, Process *);
 void erts_program_counter_info(int to, void *to_arg, Process *);
+void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
+void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
+void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);
 
 Eterm erts_get_process_priority(Process *p);
 Eterm erts_set_process_priority(Process *p, Eterm prio);
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 2f3cf23b00..36bb6b2f0e 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -43,8 +43,9 @@ static void dump_process_info(int to, void *to_arg, Process *p);
 static void dump_element(int to, void *to_arg, Eterm x);
 static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep);
 static void dump_element_nl(int to, void *to_arg, Eterm x);
-static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
+static int stack_element_dump(int to, void *to_arg, Eterm* sp,
 			      int yreg);
+static void stack_trace_dump(int to, void *to_arg, Eterm* sp);
 static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
 static void heap_dump(int to, void *to_arg, Eterm x);
 static void dump_binaries(int to, void *to_arg, Binary* root);
@@ -148,7 +149,7 @@ dump_process_info(int to, void *to_arg, Process *p)
     if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) {
 	erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id);
 	for (sp = p->stop; sp < STACK_START(p); sp++) {
-	    yreg = stack_element_dump(to, to_arg, p, sp, yreg);
+	    yreg = stack_element_dump(to, to_arg, sp, yreg);
 	}
 
 	erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id);
@@ -243,9 +244,65 @@ dump_element_nl(int to, void *to_arg, Eterm x)
     erts_putc(to, to_arg, '\n');
 }
 
+static void
+stack_trace_dump(int to, void *to_arg, Eterm *sp) {
+    Eterm x = *sp;
+    if (is_CP(x)) {
+        erts_print(to, to_arg, "%p:", sp);
+        erts_print(to, to_arg, "SReturn addr 0x%X (", cp_val(x));
+        print_function_from_pc(to, to_arg, cp_val(x));
+        erts_print(to, to_arg, ")\n");
+    }
+}
+
+void
+erts_limited_stack_trace(int to, void *to_arg, Process *p)
+{
+    Eterm* sp;
+
+
+    if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) {
+	return;
+    }
+
+    if (STACK_START(p) < STACK_TOP(p)) {
+        return;
+    }
+
+    if ((STACK_START(p) - STACK_TOP(p)) < 512) {
+        if (erts_sys_is_area_readable((char*)STACK_TOP(p),
+                                      (char*)STACK_START(p)))
+            for (sp = STACK_TOP(p); sp < STACK_START(p); sp++)
+                stack_trace_dump(to, to_arg, sp);
+        else
+            erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n",
+                       STACK_TOP(p), STACK_START(p));
+    } else {
+        sp = STACK_TOP(p);
+        if (erts_sys_is_area_readable((char*)STACK_TOP(p),
+                                      (char*)(STACK_TOP(p) + 25)))
+            for (; sp < (STACK_TOP(p) + 256); sp++)
+                stack_trace_dump(to, to_arg, sp);
+        else
+            erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n",
+                       STACK_TOP(p), STACK_TOP(p) + 256);
+
+        erts_print(to, to_arg, "%p: skipping %d frames\n",
+                   sp, STACK_START(p) - STACK_TOP(p) - 512);
+
+        if (erts_sys_is_area_readable((char*)(STACK_START(p) - 256),
+                                      (char*)STACK_START(p)))
+            for (sp = STACK_START(p) - 256; sp < STACK_START(p); sp++)
+                stack_trace_dump(to, to_arg, sp);
+        else
+            erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n",
+                       STACK_START(p) - 256, STACK_START(p));
+    }
+
+}
 
 static int
-stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
+stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
 {
     Eterm x = *sp;
 
@@ -508,3 +565,114 @@ dump_externally(int to, void *to_arg, Eterm term)
 	erts_print(to, to_arg, "%02X", *s++);
     }
 }
+
+void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) {
+    if (psflg & ERTS_PSFLG_FREE)
+	erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */
+    else if (psflg & ERTS_PSFLG_EXITING)
+	erts_print(to, to_arg, "Exiting\n");
+    else if (psflg & ERTS_PSFLG_GC) {
+	erts_print(to, to_arg, "Garbing\n");
+    }
+    else if (psflg & ERTS_PSFLG_SUSPENDED)
+	erts_print(to, to_arg, "Suspended\n");
+    else if (psflg & ERTS_PSFLG_RUNNING) {
+	erts_print(to, to_arg, "Running\n");
+    }
+    else if (psflg & ERTS_PSFLG_ACTIVE)
+	erts_print(to, to_arg, "Scheduled\n");
+    else
+	erts_print(to, to_arg, "Waiting\n");
+}
+
+void
+erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) {
+
+    int i;
+
+    switch (ERTS_PSFLGS_GET_ACT_PRIO(psflg)) {
+    case PRIORITY_MAX: erts_print(to, to_arg, "ACT_PRIO_MAX | "); break;
+    case PRIORITY_HIGH: erts_print(to, to_arg, "ACT_PRIO_HIGH | "); break;
+    case PRIORITY_NORMAL: erts_print(to, to_arg, "ACT_PRIO_NORMAL | "); break;
+    case PRIORITY_LOW: erts_print(to, to_arg, "ACT_PRIO_LOW | "); break;
+    }
+    switch (ERTS_PSFLGS_GET_USR_PRIO(psflg)) {
+    case PRIORITY_MAX: erts_print(to, to_arg, "USR_PRIO_MAX | "); break;
+    case PRIORITY_HIGH: erts_print(to, to_arg, "USR_PRIO_HIGH | "); break;
+    case PRIORITY_NORMAL: erts_print(to, to_arg, "USR_PRIO_NORMAL | "); break;
+    case PRIORITY_LOW: erts_print(to, to_arg, "USR_PRIO_LOW | "); break;
+    }
+    switch (ERTS_PSFLGS_GET_PRQ_PRIO(psflg)) {
+    case PRIORITY_MAX: erts_print(to, to_arg, "PRQ_PRIO_MAX"); break;
+    case PRIORITY_HIGH: erts_print(to, to_arg, "PRQ_PRIO_HIGH"); break;
+    case PRIORITY_NORMAL: erts_print(to, to_arg, "PRQ_PRIO_NORMAL"); break;
+    case PRIORITY_LOW: erts_print(to, to_arg, "PRQ_PRIO_LOW"); break;
+    }
+
+    psflg &= ~(ERTS_PSFLGS_ACT_PRIO_MASK |
+               ERTS_PSFLGS_USR_PRIO_MASK |
+               ERTS_PSFLGS_PRQ_PRIO_MASK);
+
+    if (psflg)
+        erts_print(to, to_arg, " | ");
+
+    for (i = 0; i < ERTS_PSFLG_MAX && psflg; i++) {
+        erts_aint32_t chk = (1 << i);
+        if (psflg & chk) {
+            switch (chk) {
+            case ERTS_PSFLG_IN_PRQ_MAX:
+                erts_print(to, to_arg, "IN_PRQ_MAX"); break;
+            case ERTS_PSFLG_IN_PRQ_HIGH:
+                erts_print(to, to_arg, "IN_PRQ_HIGH"); break;
+            case ERTS_PSFLG_IN_PRQ_NORMAL:
+                erts_print(to, to_arg, "IN_PRQ_NORMAL"); break;
+            case ERTS_PSFLG_IN_PRQ_LOW:
+                erts_print(to, to_arg, "IN_PRQ_LOW"); break;
+            case ERTS_PSFLG_FREE:
+                erts_print(to, to_arg, "FREE"); break;
+            case ERTS_PSFLG_EXITING:
+                erts_print(to, to_arg, "EXITING"); break;
+            case ERTS_PSFLG_PENDING_EXIT:
+                erts_print(to, to_arg, "PENDING_EXIT"); break;
+            case ERTS_PSFLG_ACTIVE:
+                erts_print(to, to_arg, "ACTIVE"); break;
+            case ERTS_PSFLG_IN_RUNQ:
+                erts_print(to, to_arg, "IN_RUNQ"); break;
+            case ERTS_PSFLG_RUNNING:
+                erts_print(to, to_arg, "RUNNING"); break;
+            case ERTS_PSFLG_SUSPENDED:
+                erts_print(to, to_arg, "SUSPENDED"); break;
+            case ERTS_PSFLG_GC:
+                erts_print(to, to_arg, "GC"); break;
+            case ERTS_PSFLG_BOUND:
+                erts_print(to, to_arg, "BOUND"); break;
+            case ERTS_PSFLG_TRAP_EXIT:
+                erts_print(to, to_arg, "TRAP_EXIT"); break;
+            case ERTS_PSFLG_ACTIVE_SYS:
+                erts_print(to, to_arg, "ACTIVE_SYS"); break;
+            case ERTS_PSFLG_RUNNING_SYS:
+                erts_print(to, to_arg, "RUNNING_SYS"); break;
+            case ERTS_PSFLG_PROXY:
+                erts_print(to, to_arg, "PROXY"); break;
+            case ERTS_PSFLG_DELAYED_SYS:
+                erts_print(to, to_arg, "DELAYED_SYS"); break;
+#ifdef ERTS_DIRTY_SCHEDULERS
+            case ERTS_PSFLG_DIRTY_CPU_PROC:
+                erts_print(to, to_arg, "DIRTY_CPU_PROC"); break;
+            case ERTS_PSFLG_DIRTY_IO_PROC:
+                erts_print(to, to_arg, "DIRTY_IO_PROC"); break;
+            case ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q:
+                erts_print(to, to_arg, "DIRTY_CPU_PROC_IN_Q"); break;
+            case ERTS_PSFLG_DIRTY_IO_PROC_IN_Q:
+                erts_print(to, to_arg, "DIRTY_IO_PROC_IN_Q"); break;
+#endif
+            default:
+                erts_print(to, to_arg, "UNKNOWN(%d)", chk); break;
+            }
+            if (psflg > chk)
+                erts_print(to, to_arg, " | ");
+            psflg -= chk;
+        }
+    }
+    erts_print(to, to_arg, "\n");
+}
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 545a0343d0..3dca2aba10 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1381,25 +1381,10 @@ erts_thr_progress_block(void)
     thr_progress_block(tmp_thr_prgr_data(NULL), 1);
 }
 
-void
-erts_thr_progress_fatal_error_block(SWord timeout,
-				    ErtsThrPrgrData *tmp_tpd_bufp)
+int
+erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp)
 {
     ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
-    erts_aint32_t bc;
-    SWord time_left = timeout;
-    SysTimeval to;
-
-    /*
-     * Counting poll intervals may give us a too long timeout
-     * if cpu is busy. If we got tolerant time of day we use it
-     * to prevent this.
-     */
-    if (!erts_disable_tolerant_timeofday) {
-	erts_get_timeval(&to);
-	to.tv_sec += timeout / 1000;
-	to.tv_sec += timeout % 1000;
-    }
 
     if (!tpd) {
 	/*
@@ -1412,9 +1397,26 @@ erts_thr_progress_fatal_error_block(SWord timeout,
 	init_tmp_thr_prgr_data(tpd);
     }
 
-    bc = thr_progress_block(tpd, 0);
-    if (bc == 0)
-	return; /* Succefully blocked all managed threads */
+    /* Returns number of threads that have not yes been blocked */
+    return thr_progress_block(tpd, 0);
+}
+
+void
+erts_thr_progress_fatal_error_wait(SWord timeout) {
+    erts_aint32_t bc;
+    SWord time_left = timeout;
+    SysTimeval to;
+
+    /*
+     * Counting poll intervals may give us a too long timeout
+     * if cpu is busy. If we got tolerant time of day we use it
+     * to prevent this.
+     */
+    if (!erts_disable_tolerant_timeofday) {
+        erts_get_timeval(&to);
+        to.tv_sec += timeout / 1000;
+        to.tv_sec += timeout % 1000;
+    }
 
     while (1) {
 	if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0)
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 5f392944c2..85b4eeb0eb 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -83,8 +83,8 @@ typedef struct {
     ErtsThrPrgrLeaderState leader_state;
 } ErtsThrPrgrData;
 
-void erts_thr_progress_fatal_error_block(SWord timeout,
-					 ErtsThrPrgrData *tmp_tpd_bufp);
+int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
+void erts_thr_progress_fatal_error_wait(SWord timeout);
 
 #endif /* ERTS_SMP */
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index bdc5237815..9227ce74a2 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -756,6 +756,8 @@ typedef struct {
 } ErtsCheckIoDebugInfo;
 int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);
 
+int erts_sys_is_area_readable(char *start, char *stop);
+
 /* xxxP */
 #define SYS_DEFAULT_FLOAT_DECIMALS 20
 void init_sys_float(void);
-- 
cgit v1.2.3


From d373dc49ae4c8a2d27dde23564bcaf2642cc46e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 12 Jan 2015 17:30:45 +0100
Subject: erts: Extend driver interface with emergency_close

The intention of this callback is to close all sockets associated to
a port. It is closed only on crashdumps.

This will currently only be used for the epmd port.
---
 erts/emulator/beam/erl_driver.h | 5 ++++-
 erts/emulator/beam/global.h     | 1 +
 erts/emulator/beam/io.c         | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index f9938fc66c..e498ac70ec 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -133,7 +133,7 @@ typedef struct {
 
 #define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
 #define ERL_DRV_EXTENDED_MAJOR_VERSION	3
-#define ERL_DRV_EXTENDED_MINOR_VERSION	1
+#define ERL_DRV_EXTENDED_MINOR_VERSION	2
 
 /*
  * The emulator will refuse to load a driver with a major version
@@ -361,6 +361,9 @@ typedef struct erl_drv_entry {
     	                        /* Called on behalf of driver_select when
 				   it is safe to release 'event'. A typical
 				   unix driver would call close(event) */
+    void (*emergency_close)(ErlDrvData drv_data);
+                                /* called when the port is closed abruptly.
+				   specifically when erl_crash_dump is called. */
     /* When adding entries here, dont forget to pad in obsolete/driver.h */
 } ErlDrvEntry;
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 891046a8b5..d25daaf7b8 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -160,6 +160,7 @@ struct erts_driver_t_ {
     void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data); /* Might be NULL */ 
     void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
     void (*stop_select)(ErlDrvEvent event, void*); /* Might be NULL */
+    void (*emergency_close)(ErlDrvData drv_data);  /* Might be NULL */
 };
 
 extern erts_driver_t *driver_list;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 9ae973e108..4ae8fafb2c 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7396,6 +7396,7 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->timeout = de->timeout ? de->timeout : no_timeout_callback;
     drv->ready_async = de->ready_async;
     drv->process_exit = de->process_exit;
+    drv->emergency_close = de->emergency_close;
     if (de->stop_select)
 	drv->stop_select = de->stop_select;
     else
-- 
cgit v1.2.3


From fdf09e81de5e7f1ecfe71f98b56c411073badae8 Mon Sep 17 00:00:00 2001
From: Olivier Girondel <olivier@biniou.info>
Date: Sun, 4 Jan 2015 04:40:55 +0100
Subject: Add math:log2/1

---
 erts/emulator/beam/bif.tab    |  1 +
 erts/emulator/beam/erl_math.c | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index aea3224342..1d0d214e77 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -194,6 +194,7 @@ bif math:erf/1
 bif math:erfc/1
 bif math:exp/1
 bif math:log/1
+bif math:log2/1
 bif math:log10/1
 bif math:sqrt/1
 bif math:atan2/2
diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c
index 16d4fdc09c..9b864628db 100644
--- a/erts/emulator/beam/erl_math.c
+++ b/erts/emulator/beam/erl_math.c
@@ -207,6 +207,24 @@ BIF_RETTYPE math_log_1(BIF_ALIST_1)
     return math_call_1(BIF_P, log, BIF_ARG_1);
 }
 
+#ifdef HAVE_LOG2
+static double
+log2_wrapper(double x)
+{
+    return log2(x);
+}
+#else
+static double
+log2_wrapper(double x)
+{
+    return log(x) / 0.6931471805599453; /* log(2.0); */
+}
+#endif
+
+BIF_RETTYPE math_log2_1(BIF_ALIST_1)
+{
+    return math_call_1(BIF_P, log2_wrapper, BIF_ARG_1);
+}
 
 BIF_RETTYPE math_log10_1(BIF_ALIST_1)
 {
-- 
cgit v1.2.3


From 86d605807356f79f3484c36cba632ff88d4e2417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 13 Jan 2015 12:13:43 +0100
Subject: erts: Use emergency close to close epmd

Closes all open socket before writing crashdump to file.
---
 erts/emulator/beam/global.h |  1 +
 erts/emulator/beam/io.c     | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d25daaf7b8..32a2dc43e8 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -853,6 +853,7 @@ Uint erts_port_ioq_size(Port *pp);
 void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int);
 
 Port *erts_get_heart_port(void);
+void erts_emergency_close_ports(void);
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
 void erts_lcnt_enable_io_lock_count(int enable);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 4ae8fafb2c..3316654790 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7569,3 +7569,23 @@ Port *erts_get_heart_port(void)
 
     return NULL;
 }
+
+void erts_emergency_close_ports(void)
+{
+    int ix, max = erts_ptab_max(&erts_port);
+
+    for (ix = 0; ix < max; ix++) {
+	Port *port = erts_pix2port(ix);
+
+	if (!port)
+	    continue;
+	/* only examine undead or alive ports */
+	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD)
+	    continue;
+
+	/* emergency close socket */
+	if (port->drv_ptr->emergency_close) {
+	    port->drv_ptr->emergency_close((ErlDrvData) port->drv_data);
+	}
+    }
+}
-- 
cgit v1.2.3


From 1520bca342e8433089841d2c79d3d862d06479e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 13 Jan 2015 17:04:00 +0100
Subject: erts: Don't lookup invalid port for crashdump handling

---
 erts/emulator/beam/io.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3316654790..2ac27f024e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7558,7 +7558,7 @@ Port *erts_get_heart_port(void)
 	if (!port)
 	    continue;
 	/* only examine undead or alive ports */
-	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD)
+	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
 	    continue;
 	/* immediate atom compare */
 	reg = port->common.u.alive.reg;
@@ -7580,7 +7580,7 @@ void erts_emergency_close_ports(void)
 	if (!port)
 	    continue;
 	/* only examine undead or alive ports */
-	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD)
+	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
 	    continue;
 
 	/* emergency close socket */
-- 
cgit v1.2.3


From 86e4d8ead48501f00466d685c157eaa0921d96a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 15 Jan 2015 14:40:50 +0100
Subject: erts: Check driver version before assigning callback

---
 erts/emulator/beam/io.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 2ac27f024e..012a7d1a4b 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7349,6 +7349,8 @@ no_stop_select_callback(ErlDrvEvent event, void* private)
     erts_send_error_to_logger_nogl(dsbufp);
 }
 
+#define IS_DRIVER_VERSION_GE(DE,MAJOR,MINOR) \
+    ((DE)->major_version >= (MAJOR) && (DE)->minor_version >= (MINOR))
 
 static int
 init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
@@ -7396,7 +7398,7 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->timeout = de->timeout ? de->timeout : no_timeout_callback;
     drv->ready_async = de->ready_async;
     drv->process_exit = de->process_exit;
-    drv->emergency_close = de->emergency_close;
+    drv->emergency_close = IS_DRIVER_VERSION_GE(de,3,2) ? de->emergency_close : NULL;
     if (de->stop_select)
 	drv->stop_select = de->stop_select;
     else
@@ -7415,6 +7417,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     }
 }
 
+#undef IS_DRIVER_VERSION_GE
+
 void
 erts_destroy_driver(erts_driver_t *drv)
 {
-- 
cgit v1.2.3


From 7e147a05683c709128b6777d0c360fcde067f567 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Wed, 4 Feb 2015 20:27:37 +0100
Subject: don't create oversize bignums in binary matching

Bignums are artifically restricted in size.  Arithmetic and logical
operations check the sizes of resulting bignums, and turn oversize
results into system_limit exceptions.

However, this check is not performed when bignums are constructed by
binary matching.  The consequence is that such matchings can construct
oversize bignums that satisfy is_integer/1 yet don't work.  Performing
arithmetic such as Term - 0 fails with a system_limit exception.  Worse,
performing a logical operation such as Term band Term results in [].
The latter occurs because the size checking (e.g. in erts_band()) is
a simple ASSERT(is_not_nil(...)) on the result of the bignum operation,
which internally is [] (NIL) in the case of oversize results.  However,
ASSERT is a no-op in release builds, so the error goes unnoticed and []
is returned as the result of the band/2.

This patch addresses this by preventing oversize bignums from entering
the VM via binary matching:

- the internal bytes_to_big() procedure is augmented to return NIL for
  oversize results, just like big_norm()
- callers of bytes_to_big() are augmented to check for NIL returns and
  signal errors in those cases
- erts_bs_get_integer_2() can only fail with badmatch, so that is the
  Erlang-level result of oversize bignums from binary matches
- big_SUITE.erl is extended with a test case that fails without this
  fix (no error signalled) and passes with it (badmatch occurs)

Credit goes to Nico Kruber for the initial bug report.
---
 erts/emulator/beam/beam_load.c | 3 ++-
 erts/emulator/beam/big.c       | 2 ++
 erts/emulator/beam/erl_bits.c  | 4 +++-
 erts/emulator/beam/external.c  | 2 ++
 4 files changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index cfc6146b0a..41c1b5d2c2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4971,7 +4971,8 @@ get_tag_and_value(LoaderState* stp, Uint len_code,
 
     arity = count/sizeof(Eterm);
     *result = new_literal(stp, &hp, arity+1);
-    (void) bytes_to_big(bigbuf, count, neg, hp);
+    if (is_nil(bytes_to_big(bigbuf, count, neg, hp)))
+	goto load_error;
 
     if (bigbuf != default_buf) {
 	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf);
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index de7d370938..d1e46e3063 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1900,6 +1900,8 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
 	*rwp = d;
 	rwp++;
     }
+    if (rsz > BIG_ARITY_MAX)
+	return NIL;
     if (xsgn) {
       *r = make_neg_bignum_header(rsz);
     }
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 73765772c8..642f56a15e 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -403,7 +403,9 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
 	words_needed = 1+WSIZE(bytes);
 	hp = HeapOnlyAlloc(p, words_needed);
 	res = bytes_to_big(LSB, bytes, sgn, hp); 
-	if (is_small(res)) {
+	if (is_nil(res)) {
+	    res = THE_NON_VALUE;
+	} else if (is_small(res)) {
 	    p->htop = hp;
 	} else if ((actual = bignum_header_arity(*hp)+1) < words_needed) {
 	    p->htop = hp + actual;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9b9b4b2a62..45d1f7514e 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3056,6 +3056,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
 		    big = make_small(0);
 		} else {
 		    big = bytes_to_big(first, n, neg, hp);
+		    if (is_nil(big))
+			goto error;
 		    if (is_big(big)) {
 			hp += big_arity(big) + 1;
 		    }
-- 
cgit v1.2.3


From 7f82fdee75c2c3c3c5eaf259e2671737163be32b Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Sat, 14 Feb 2015 16:18:10 +0100
Subject: don't leave a heap hole in erts_bs_get_integer_2

Reset p->htop in the oversize bignum error case to avoid leaving a
hole in the heap, which would crash the debug emulator.
---
 erts/emulator/beam/erl_bits.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 642f56a15e..53c21c40e1 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -404,6 +404,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
 	hp = HeapOnlyAlloc(p, words_needed);
 	res = bytes_to_big(LSB, bytes, sgn, hp); 
 	if (is_nil(res)) {
+	    p->htop = hp;
 	    res = THE_NON_VALUE;
 	} else if (is_small(res)) {
 	    p->htop = hp;
-- 
cgit v1.2.3


From cfa9229ba1eb8851caecfc92ca1bc9813fb1d567 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Sun, 8 Feb 2015 21:19:35 +0100
Subject: remove perfctr support

Perfctr is a Linux kernel extension that allows programmatic access
to the performance monitoring counters found in most current CPUs.
However, development of perfctr ceased after 2010, and it cannot be
used with Linux kernels newer than 2.6.32.

Therefore the perfctr support code in the Erlang VM is effectively
dead code, so this patch removes it.
---
 erts/emulator/beam/benchmark.c | 65 +-----------------------------------------
 erts/emulator/beam/benchmark.h | 39 ++-----------------------
 2 files changed, 3 insertions(+), 101 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c
index 8613131176..b16fe6b271 100644
--- a/erts/emulator/beam/benchmark.c
+++ b/erts/emulator/beam/benchmark.c
@@ -37,37 +37,9 @@ unsigned long long major_gc;
 
 #ifdef BM_TIMERS
 
-#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR
-
-#include "libperfctr.h"
-struct vperfctr *system_clock;
-double cpu_khz;
-BM_NEW_TIMER(start);
-
-static double get_hrvtime(void)
-{
-    unsigned long long ticks;
-    double milli_seconds;
-
-    ticks = vperfctr_read_tsc(system_clock);
-    milli_seconds = (double)ticks / cpu_khz;
-    return milli_seconds;
-}
-
-static void stop_hrvtime(void)
-{
-    if(system_clock)
-    {
-	vperfctr_stop(system_clock);
-	vperfctr_close(system_clock);
-	system_clock = NULL;
-    }
-}
-
-#else /* not perfctr, asuming Solaris */
+/* assuming Solaris */
 #include <time.h>
 BM_TIMER_T system_clock;
-#endif
 
 unsigned long local_pause_times[MAX_PAUSE_TIME];
 unsigned long pause_times[MAX_PAUSE_TIME];
@@ -117,40 +89,6 @@ unsigned long long message_sizes[1000];
 void init_benchmarking()
 {
 #ifdef BM_TIMERS
-#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR
-    /* pass `--with-perfctr=/path/to/perfctr' when configuring */
-    struct perfctr_info info;
-    struct vperfctr_control control;
-    int i;
-
-    system_clock = vperfctr_open();
-    if (system_clock != NULL)
-    {
-        if (vperfctr_info(system_clock,&info) >= 0)
-        {
-            cpu_khz = (double)info.cpu_khz;
-            if (info.cpu_features & PERFCTR_FEATURE_RDTSC)
-            {
-                memset(&control,0,sizeof control);
-                control.cpu_control.tsc_on = 1;
-            }
-        }
-        if (vperfctr_control(system_clock,&control) < 0)
-        {
-            vperfctr_close(system_clock);
-            system_clock = NULL;
-        }
-    }
-
-    for (i = 0; i < 1000; i++)
-    {
-        BM_START_TIMER(system);
-        BM_STOP_TIMER(system);
-    }
-
-    timer_time = system_time / 1000;
-    start_time = 0;
-#else
     int i;
     for (i = 0; i < 1000; i++)
     {
@@ -158,7 +96,6 @@ void init_benchmarking()
         BM_STOP_TIMER(system);
     }
     timer_time = system_time / 1000;
-#endif
 
     for (i = 0; i < MAX_PAUSE_TIME; i++) {
         local_pause_times[i] = 0;
diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h
index 766edaac42..904564a96b 100644
--- a/erts/emulator/beam/benchmark.h
+++ b/erts/emulator/beam/benchmark.h
@@ -37,10 +37,7 @@
 
 /* BM_TIMERS keeps track of the time spent in diferent parts of the
  * system. It only measures accual active time, not time spent in idle
- * mode. These timers requires hardware support.  For Linux, use the
- * package perfctr from user.it.uu.se/~mikpe/linux/perfctr. If this
- * package is not specified when configuring the system
- * (--with-perfctr=PATH), the Solaris hrtime_t will be used.
+ * mode. Currently, the Solaris hrtime_t will be used.
  * To add new timers look below.
  */
 #define BM_TIMERS
@@ -142,38 +139,7 @@ extern unsigned long long major_gc;
  * meassure (send time in shared heap for instance).
  */
 
-#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR
-#include "libperfctr.h"
-
-#define BM_TIMER_T double
-
-extern struct vperfctr *system_clock;
-extern double cpu_khz;
-extern BM_TIMER_T start_time;
-
-#define BM_START_TIMER(t) start_time =                                        \
-                          (BM_TIMER_T)vperfctr_read_tsc(system_clock) /       \
-                          cpu_khz;
-
-#define BM_STOP_TIMER(t) do {                                                 \
-    BM_TIMER_T tmp = ((BM_TIMER_T)vperfctr_read_tsc(system_clock) / cpu_khz); \
-    tmp -= (start_time + timer_time);                                         \
-    t##_time += (tmp > 0 ? tmp : 0);                                          \
-} while(0)
-
-#define BM_TIME_PRINTER(str,time) do {                                 \
-    int min,sec,milli,micro;                                           \
-    BM_TIMER_T tmp = (time) * 1000;                                    \
-    micro = (uint)(tmp - ((int)(tmp / 1000)) * 1000);                  \
-    tmp /= 1000;                                                       \
-    milli = (uint)(tmp - ((int)(tmp / 1000)) * 1000);                  \
-    tmp /= 1000;                                                       \
-    sec = (uint)(tmp - ((int)(tmp / 60)) * 60);                        \
-    min = (uint)tmp / 60;                                              \
-    erts_fprintf(file,str": %d:%02d.%03d %03d\n",min,sec,milli,micro);  \
-} while(0)
-
-#else /* !USE_PERFCTR  (Assuming Solaris) */
+/* (Assuming Solaris) */
 
 #define BM_TIMER_T hrtime_t
 #define BM_START_TIMER(t) system_clock = sys_gethrtime()
@@ -196,7 +162,6 @@ extern BM_TIMER_T start_time;
 } while(0)
 
 extern BM_TIMER_T system_clock;
-#endif /* USE_PERFCTR */
 
 extern BM_TIMER_T timer_time;
 extern BM_TIMER_T system_time;
-- 
cgit v1.2.3


From 3a9b8b9e09a56cf23c86794094da321ae0a26145 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 20 Feb 2015 11:50:29 +0100
Subject: erts: Fix erroneous printout in crashdump

An extra '}' were printed in remote links.
---
 erts/emulator/beam/break.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 08265b590d..2cddfe2800 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -181,7 +181,6 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
 	    ASSERT(is_node_name_atom(mon->pid));
 	    erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name,
 		       mon->pid, mon->ref);
-	    erts_print(to, to_arg,"}");
 	} else if (is_atom(mon->name)){ /* local by name */
 	    erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name,
 		       erts_this_dist_entry->sysname, mon->ref);
-- 
cgit v1.2.3


From b493fbf2f6f788e9ca7f65df30c3af2bbcb0c169 Mon Sep 17 00:00:00 2001
From: Wasif Malik <wmalik@gmail.com>
Date: Sat, 21 Feb 2015 21:58:16 +0100
Subject: Fix make_hash description

---
 erts/emulator/beam/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index c505c44905..f810fca9a4 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -707,7 +707,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
 **  If N < 0, Y = FUNNY_NUMBER4 else Y = FUNNY_NUMBER3.
 **  The hash value is Y*h(J) mod 2^32 where h(J) is calculated like
 **  h(0) = <initial hash> 
-**  h(i) = h(i-i)*X + B(i-1)
+**  h(i) = h(i-1)*X + B(i-1)
 ** The above should hold regardless of internal representation.
 ** Pids are hashed like small numbers but with differrent constants, as are
 ** ports.
-- 
cgit v1.2.3


From 0ec91ff571518e199aac7a4da961a80c153483b9 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Fri, 9 May 2014 16:00:54 +0200
Subject: Allow 4-ary BIFs

---
 erts/emulator/beam/beam_emu.c | 2 +-
 erts/emulator/beam/bif.h      | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 3af7c43abf..2d6fad8ae6 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3553,7 +3553,7 @@ get_map_elements_fail:
 	    vbf = (BifFunction) Arg(0);
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = I[-1];
-	    ASSERT(bif_nif_arity <= 3);
+            ASSERT(bif_nif_arity <= 4);
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	    reg[0] = r(0);
 	    {
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 7b69b39511..837cb017ac 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -30,10 +30,12 @@ extern Export* erts_format_cpu_topology_trap;
 #define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS
 #define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS
 #define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS
+#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS
 
 #define BIF_ARG_1  (BIF__ARGS[0])
 #define BIF_ARG_2  (BIF__ARGS[1])
 #define BIF_ARG_3  (BIF__ARGS[2])
+#define BIF_ARG_4  (BIF__ARGS[3])
 
 #define ERTS_IS_PROC_OUT_OF_REDS(p)		\
     ((p)->fcalls > 0				\
-- 
cgit v1.2.3


From c472fc520fc30d6d19adc89bdc42817870274db2 Mon Sep 17 00:00:00 2001
From: Leo Liu <sdl.web@gmail.com>
Date: Fri, 12 Dec 2014 20:56:40 +0800
Subject: Two minor fixes

* Fix documentation on $char for unicode
* Remove duplicate declaration for erts_encode_ext_dist_header_size
---
 erts/emulator/beam/external.h | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index bf00958eb1..10565f67e5 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -156,7 +156,6 @@ void erts_init_atom_cache_map(ErtsAtomCacheMap *);
 void erts_reset_atom_cache_map(ErtsAtomCacheMap *);
 void erts_destroy_atom_cache_map(ErtsAtomCacheMap *);
 void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32);
-Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
 
 Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
 byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *);
-- 
cgit v1.2.3


From e330a493032c35c0ca69cd78142e2ec03f5e370b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 4 Mar 2015 17:08:31 +0100
Subject: erts: Enable command line argument for initial pd size

Use '+hpds size' to set initial process dictionary size for spawned processes.
---
 erts/emulator/beam/erl_init.c         | 13 +++++++++++++
 erts/emulator/beam/erl_process_dict.c |  2 +-
 erts/emulator/beam/erl_vm.h           |  1 +
 3 files changed, 15 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 61f8385efc..743fd235bc 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -194,6 +194,8 @@ int erts_disable_tolerant_timeofday; /* Time correction can be disabled it is
 
 int erts_atom_table_size = ATOM_LIMIT;	/* Maximum number of atoms */
 
+int erts_pd_initial_size = 10;
+
 int erts_modified_timing_level;
 
 int erts_no_crash_dump = 0;	/* Use -d to suppress crash dump. */
@@ -519,6 +521,8 @@ void erts_usage(void)
 	       H_DEFAULT_SIZE);
     erts_fprintf(stderr, "-hmbs size     set minimum binary virtual heap size in words (default %d)\n",
 	       VH_DEFAULT_SIZE);
+    erts_fprintf(stderr, "-hpds size     initial process dictionary size (default %d)\n",
+	       erts_pd_initial_size);
 
     /*    erts_fprintf(stderr, "-i module  set the boot module (default init)\n"); */
 
@@ -1408,6 +1412,7 @@ erl_start(int argc, char **argv)
 	     *
 	     * h|ms  - min_heap_size
 	     * h|mbs - min_bin_vheap_size
+	     * h|pds - erts_pd_initial_size
 	     *
 	     */
 	    if (has_prefix("mbs", sub_param)) {
@@ -1425,6 +1430,14 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 		VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE));
+	    } else if (has_prefix("pds", sub_param)) {
+		arg = get_arg(sub_param+3, argv[i+1], &i);
+		if ((erts_pd_initial_size = atoi(arg)) <= 0) {
+		    erts_fprintf(stderr, "bad initial process dictionary size %s\n", arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM, ("using initial process dictionary size %d\n",
+			    erts_pd_initial_size));
 	    } else {
 	        /* backward compatibility */
 		arg = get_arg(argv[i]+2, argv[i+1], &i);
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 23e5bf737f..af20b26b15 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -47,7 +47,7 @@
 
 /* Hash constant macros */
 #define MAX_HASH            1342177280UL
-#define INITIAL_SIZE        10
+#define INITIAL_SIZE        (erts_pd_initial_size)
 
 /* Hash utility macros  */
 #define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition)
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index b7de8208ad..6687b044ee 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -174,6 +174,7 @@ extern int H_MIN_SIZE;		/* minimum (heap + stack) */
 extern int BIN_VH_MIN_SIZE;	/* minimum virtual (bin) heap */
 
 extern int erts_atom_table_size;/* Atom table size */
+extern int erts_pd_initial_size;/* Initial Process dictionary table size */
 
 #define ORIG_CREATION 0
 #define INTERNAL_CREATION 255
-- 
cgit v1.2.3


From 2b69d295dc73f845305a0e2d7763a53657af0725 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 6 Mar 2015 09:23:42 +0100
Subject: Make access to control_flags safe in non-smp emulator

---
 erts/emulator/beam/io.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 9ae973e108..86fb0a39bf 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4087,6 +4087,9 @@ erts_port_control(Process* c_p,
 				      size,
 				      &resp_bufp,
 				      &resp_size);
+
+	    control_flags = prt->control_flags;
+
 	    finalize_imm_drv_call(&try_call_state);
 	    if (tmp_alloced)
 		erts_free(ERTS_ALC_T_TMP, bufp);
@@ -4094,8 +4097,6 @@ erts_port_control(Process* c_p,
 		return ERTS_PORT_OP_BADARG;
 	    }
 
-	    control_flags = prt->control_flags;
-
 	    hsz = port_control_result_size(control_flags,
 					   resp_bufp,
 					   &resp_size,
-- 
cgit v1.2.3


From d6073d78109f026ef96b29af4ce748242df2389d Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Wed, 11 Mar 2015 14:03:58 +0100
Subject: Fix compilation of match specs with maps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous compilation was just plain wrong with push/pop mismatches.

Reported-by: Björn-Egil Dahlberg
---
 erts/emulator/beam/erl_db_util.c | 63 +++++++++++++++++++++++++---------------
 1 file changed, 39 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 7eb80e3bb1..1d986f2447 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -214,8 +214,8 @@ typedef enum {
     matchPushT,
     matchPushL,
     matchPushM,
-    matchPushK,
     matchPop,
+    matchSwap,
     matchBind,
     matchCmp,
     matchEqBin,
@@ -225,6 +225,7 @@ typedef enum {
     matchEq,
     matchList,
     matchMap,
+    matchKey,
     matchSkip,
     matchPushC,
     matchConsA, /* Car is below Cdr */
@@ -1399,22 +1400,26 @@ restart:
                             }
                             goto error;
                         }
-                        DMC_PUSH(text, matchPushK);
-                        ++(context.stack_used);
+                        DMC_PUSH(text, matchKey);
                         DMC_PUSH(text, dmc_private_copy(&context, key));
-                    }
-                    if (context.stack_used > context.stack_need) {
-                        context.stack_need = context.stack_used;
-                    }
-                    for (i = num_iters; i--; ) {
-                        Eterm value = map_get_values(map_val(t))[i];
-                        DMC_PUSH(text, matchPop);
-                        --(context.stack_used);
-                        res = dmc_one_term(&context, &heap, &stack, &text,
-                                           value);
-                        ASSERT(res != retFail);
-                        if (res == retRestart) {
-                            goto restart;
+                        {
+                            int old_stack = ++(context.stack_used);
+                            Eterm value = map_get_values(map_val(t))[i];
+                            res = dmc_one_term(&context, &heap, &stack, &text,
+                                               value);
+                            ASSERT(res != retFail);
+                            if (res == retRestart) {
+                                goto restart;
+                            }
+                            if (old_stack != context.stack_used) {
+                                ASSERT(old_stack + 1 == context.stack_used);
+                                DMC_PUSH(text, matchSwap);
+                            }
+                            if (context.stack_used > context.stack_need) {
+                                context.stack_need = context.stack_used;
+                            }
+                            DMC_PUSH(text, matchPop);
+                            --(context.stack_used);
                         }
                     }
                     break;
@@ -1960,17 +1965,23 @@ restart:
             }
             *sp++ = map_val_rel(*ep++, base);
             break;
-        case matchPushK:
+        case matchKey:
             t = (Eterm) *pc++;
             tp = erts_maps_get_rel(t, make_map_rel(ep, base), base);
             if (!tp) {
                 FAIL();
             }
-            *sp++ = tp;
+            *sp++ = ep;
+            ep = tp;
             break;
 	case matchPop:
 	    ep = *(--sp);
 	    break;
+        case matchSwap:
+            tp = sp[-1];
+            sp[-1] = sp[-2];
+            sp[-2] = tp;
+            break;
 	case matchBind:
 	    n = *pc++;
 	    variables[n].term = *ep++;
@@ -5302,6 +5313,12 @@ void db_match_dis(Binary *bp)
             ++t;
             erts_printf("Map\t%beu\n", n);
             break;
+        case matchKey:
+            ++t;
+            p = (Eterm) *t;
+            ++t;
+            erts_printf("Key\t%p (%T)\n", t, p);
+            break;
 	case matchPushT:
 	    ++t;
 	    n = *t;
@@ -5318,16 +5335,14 @@ void db_match_dis(Binary *bp)
             ++t;
             erts_printf("PushM\t%beu\n", n);
             break;
-        case matchPushK:
-            ++t;
-            p = (Eterm) *t;
-            ++t;
-            erts_printf("PushK\t%p (%T)\n", t, p);
-            break;
 	case matchPop:
 	    ++t;
 	    erts_printf("Pop\n");
 	    break;
+        case matchSwap:
+            ++t;
+            erts_printf("Swap\n");
+            break;
 	case matchBind:
 	    ++t;
 	    n = *t;
-- 
cgit v1.2.3


From 36576d7de20e4e8cb7b3943cfe1b7e272b5e3971 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Wed, 11 Mar 2015 14:21:37 +0100
Subject: Properly collect variables in match specs with maps

---
 erts/emulator/beam/erl_db_util.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 1d986f2447..748be93fe3 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3286,7 +3286,14 @@ int db_has_variable(Eterm node) {
 		while(arity--) {
 		    ESTACK_PUSH(s,*(++tuple));
 		}
-	    }
+            } else if (is_map(node)) {
+                Eterm *values = map_get_values(map_val(node));
+                int size = map_get_size(map_val(node));
+                ESTACK_PUSH(s, ((map_t *) map_val(node))->keys);
+                while (size--) {
+                    ESTACK_PUSH(s, *(values++));
+                }
+            }
 	    break;
 	case TAG_PRIMARY_IMMED1:
 	    if (node == am_Underscore || db_is_variable(node) >= 0) {
-- 
cgit v1.2.3


From 0e464f7be1ae9b54d0fba748ab2dc7bd435ac118 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Fri, 9 May 2014 16:01:49 +0200
Subject: Create new BIF ets:update_counter/4

Conflicts:
	erts/emulator/beam/bif.tab
	lib/stdlib/src/ets.erl
---
 erts/emulator/beam/bif.tab       |   1 +
 erts/emulator/beam/erl_db.c      |  79 ++++++++++++++--------
 erts/emulator/beam/erl_db_hash.c | 141 +++++++++++++++++++++++++++++----------
 erts/emulator/beam/erl_db_tree.c |  58 +++++++++++++---
 erts/emulator/beam/erl_db_util.c |  10 +--
 erts/emulator/beam/erl_db_util.h |  20 +++---
 6 files changed, 222 insertions(+), 87 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 1d0d214e77..788c866e63 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -613,6 +613,7 @@ bif erlang:fun_info_mfa/1
 #
 
 bif erlang:get_keys/0
+bif ets:update_counter/4
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 4806befd99..fff892ae54 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -805,7 +805,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
 	list = BIF_ARG_3;
     }
 
-    if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) {
+    if (!tb->common.meth->db_lookup_dbterm(BIF_P, tb, BIF_ARG_2, THE_NON_VALUE, &handle)) {
 	cret = DB_ERROR_BADKEY;
 	goto bail_out;
     }
@@ -844,7 +844,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
     }
 
 finalize:
-    tb->common.meth->db_finalize_dbterm(&handle);
+    tb->common.meth->db_finalize_dbterm(cret, &handle);
 
 bail_out:
     UnUseTmpHeap(2,BIF_P);
@@ -863,14 +863,8 @@ bail_out:
     }
 }
 
-/* 
-** update_counter(Tab, Key, Incr) 
-** update_counter(Tab, Key, {Upop}) 
-** update_counter(Tab, Key, [{Upop}]) 
-** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo}
-** Returns new value(s) (integer or [integer])
-*/
-BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
+static BIF_RETTYPE
+do_update_counter(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Eterm arg4)
 {
     DbTable* tb;
     int cret = DB_ERROR_BADITEM;
@@ -880,7 +874,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     Eterm* ret_list_currp = NULL;
     Eterm* ret_list_prevp = NULL;
     Eterm iter;
-    DeclareTmpHeap(cell,5,BIF_P);
+    DeclareTmpHeap(cell, 5, p);
     Eterm *tuple = cell+2;
     DbUpdateHandle handle;
     Uint halloc_size = 0; /* overestimated heap usage */
@@ -888,28 +882,29 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     Eterm* hstart;
     Eterm* hend;
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
+    if ((tb = db_get_table(p, arg1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
+        BIF_ERROR(p, BADARG);
     }
 
-    UseTmpHeap(5,BIF_P);
+    UseTmpHeap(5, p);
 
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
-    if (is_integer(BIF_ARG_3)) {  /* Incr */
-	upop_list = CONS(cell, TUPLE2(tuple, make_small(tb->common.keypos+1),
-				      BIF_ARG_3), NIL);
+    if (is_integer(arg3)) { /* Incr */
+        upop_list = CONS(cell,
+                         TUPLE2(tuple, make_small(tb->common.keypos+1), arg3),
+                         NIL);
     }
-    else if (is_tuple(BIF_ARG_3)) { /* {Upop} */
-	upop_list = CONS(cell, BIF_ARG_3, NIL);
+    else if (is_tuple(arg3)) { /* {Upop} */
+        upop_list = CONS(cell, arg3, NIL);
     }
     else { /* [{Upop}] (probably) */
-	upop_list = BIF_ARG_3;
+        upop_list = arg3;
 	ret_list_prevp = &ret;
     }
 
-    if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) {
+    if (!tb->common.meth->db_lookup_dbterm(p, tb, arg2, arg4, &handle)) {
 	goto bail_out; /* key not found */
     }
 
@@ -982,13 +977,13 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     if (ret_list_prevp) { /* Prepare to return a list */
 	ret = NIL;
 	halloc_size += list_size;
-	hstart = HAlloc(BIF_P, halloc_size);
+	hstart = HAlloc(p, halloc_size);
 	ret_list_currp = hstart;
 	htop = hstart + list_size;
 	hend = hstart + halloc_size;
     }
     else {
-	hstart = htop = HAlloc(BIF_P, halloc_size);
+	hstart = htop = HAlloc(p, halloc_size);
     }
     hend = hstart + halloc_size;
 
@@ -1035,26 +1030,54 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 	   (is_list(ret) && (list_val(ret)+list_size)==ret_list_currp));
     ASSERT(htop <= hend);
 
-    HRelease(BIF_P,hend,htop);
+    HRelease(p, hend, htop);
 
 finalize:
-    tb->common.meth->db_finalize_dbterm(&handle);
+    tb->common.meth->db_finalize_dbterm(cret, &handle);
 
 bail_out:
-    UnUseTmpHeap(5,BIF_P);
+    UnUseTmpHeap(5, p);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
     case DB_ERROR_NONE:
 	BIF_RET(ret);
     case DB_ERROR_SYSRES:
-	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+        BIF_ERROR(p, SYSTEM_LIMIT);
     default:
-	BIF_ERROR(BIF_P, BADARG);
+        BIF_ERROR(p, BADARG);
 	break;
     }
 }
 
+/*
+** update_counter(Tab, Key, Incr)
+** update_counter(Tab, Key, Upop)
+** update_counter(Tab, Key, [{Upop}])
+** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo}
+** Returns new value(s) (integer or [integer])
+*/
+BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
+{
+    return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, THE_NON_VALUE);
+}
+
+/*
+** update_counter(Tab, Key, Incr, Default)
+** update_counter(Tab, Key, Upop, Default)
+** update_counter(Tab, Key, [{Upop}], Default)
+** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo}
+** Returns new value(s) (integer or [integer])
+*/
+BIF_RETTYPE ets_update_counter_4(BIF_ALIST_4)
+{
+    if (is_not_tuple(BIF_ARG_4)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4);
+}
+
+
 /* 
 ** The put BIF 
 */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index c2157457a0..8668a87ba1 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -444,8 +444,11 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl);
 #ifdef HARDDEBUG
 static void db_check_table_hash(DbTableHash *tb);
 #endif
-static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle);
-static void db_finalize_dbterm_hash(DbUpdateHandle* handle);
+static int
+db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
+                      DbUpdateHandle* handle);
+static void
+db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
 
 static ERTS_INLINE void try_shrink(DbTableHash* tb)
 {
@@ -2796,59 +2799,129 @@ static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr,
     return NULL;
 }
 
-static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle)
+static int
+db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
+                      DbUpdateHandle* handle)
 {
     DbTableHash *tb = &tbl->hash;
-    HashDbTerm* b;
-    HashDbTerm** prevp;
-    int ix;
     HashValue hval;
+    HashDbTerm **bp, *b;
     erts_smp_rwmtx_t* lck;
+    int flags = 0;
+
+    ASSERT(tb->common.status & DB_SET);
 
     hval = MAKE_HASH(key);
-    lck = WLOCK_HASH(tb,hval);
-    ix = hash_to_ix(tb, hval);
-    prevp = &BUCKET(tb, ix);
-    b = *prevp;
+    lck = WLOCK_HASH(tb, hval);
+    bp = &BUCKET(tb, hash_to_ix(tb, hval));
+    b = *bp;
 
-    while (b != 0) {
-	if (has_live_key(tb,b,key,hval)) {
-	    handle->tb = tbl;
-	    handle->bp = (void**) prevp;
-	    handle->dbterm = &b->dbterm;
-	    handle->mustResize = 0;
-	    handle->new_size = b->dbterm.size;
-	#if HALFWORD_HEAP
-	    handle->abs_vec = NULL;
-	#endif
-	    handle->lck = lck;
-	    /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */
-	    return 1;
-	}
-	prevp = &b->next;
-	b = *prevp;
+    for (;;) {
+        if (b == NULL) {
+            break;
+        }
+        if (has_key(tb, b, key, hval)) {
+            if (b->hvalue != INVALID_HASH) {
+                goto Ldone;
+            }
+            break;
+        }
+        bp = &b->next;
+        b = *bp;
     }
-    WUNLOCK_HASH(lck);
-    return 0;
+
+    if (obj == THE_NON_VALUE) {
+        WUNLOCK_HASH(lck);
+        return 0;
+    }
+
+    {
+        Eterm *objp = tuple_val(obj);
+        int arity = arityval(*objp);
+        Eterm *htop, *hend;
+
+        ASSERT(arity >= tb->common.keypos);
+        htop = HAlloc(p, arity + 1);
+        hend = htop + arity + 1;
+        sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1));
+        htop[tb->common.keypos] = key;
+        obj = make_tuple(htop);
+
+        if (b == NULL) {
+            HashDbTerm *q = new_dbterm(tb, obj);
+
+            q->hvalue = hval;
+            q->next = NULL;
+            *bp = b = q;
+
+            {
+                int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
+                int nactive = NACTIVE(tb);
+
+                if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) {
+                    grow(tb, nactive);
+                }
+            }
+        } else {
+            HashDbTerm *q, *next = b->next;
+
+            ASSERT(b->hvalue == INVALID_HASH);
+            q = replace_dbterm(tb, b, obj);
+            q->next = next;
+            q->hvalue = hval;
+            *bp = b = q;
+            erts_smp_atomic_inc_nob(&tb->common.nitems);
+        }
+
+        HRelease(p, hend, htop);
+        flags |= DB_NEW_OBJECT;
+    }
+
+Ldone:
+    handle->tb = tbl;
+    handle->bp = (void **)bp;
+    handle->dbterm = &b->dbterm;
+    handle->flags = flags;
+    handle->new_size = b->dbterm.size;
+#if HALFWORD_HEAP
+    handle->abs_vec = NULL;
+#endif
+    handle->lck = lck;
+    return 1;
 }
 
 /* Must be called after call to db_lookup_dbterm
 */
-static void db_finalize_dbterm_hash(DbUpdateHandle* handle)
+static void
+db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
 {
     DbTable* tbl = handle->tb;
-    HashDbTerm* oldp = (HashDbTerm*) *(handle->bp);
+    DbTableHash *tb = &tbl->hash;
+    HashDbTerm **bp = (HashDbTerm **) handle->bp;
+    HashDbTerm *b = *bp;
     erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck;
 
-    ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck));  /* locked by db_lookup_dbterm_hash */
+    ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck));  /* locked by db_lookup_dbterm_hash */
+
+    ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE));
 
-    ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize));
+    if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) {
+        if (IS_FIXED(tb)) {
+            add_fixed_deletion(tb, hash_to_ix(tb, b->hvalue));
+            b->hvalue = INVALID_HASH;
+        } else {
+            *bp = b->next;
+            free_term(tb, b);
+        }
 
-    if (handle->mustResize) {
+        WUNLOCK_HASH(lck);
+        erts_smp_atomic_dec_nob(&tb->common.nitems);
+        try_shrink(tb);
+    } else if (handle->flags & DB_MUST_RESIZE) {
 	db_finalize_resize(handle, offsetof(HashDbTerm,dbterm));
 	WUNLOCK_HASH(lck);
 
-	free_term(&tbl->hash, oldp);
+	free_term(tb, b);
     }
     else {
 	WUNLOCK_HASH(lck);
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 720c0659c3..577da35b75 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -399,8 +399,11 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl);
 #ifdef HARDDEBUG
 static void db_check_table_tree(DbTable *tbl);
 #endif
-static int db_lookup_dbterm_tree(DbTable *, Eterm key, DbUpdateHandle*);
-static void db_finalize_dbterm_tree(DbUpdateHandle*);
+static int
+db_lookup_dbterm_tree(Process *, DbTable *, Eterm key, Eterm obj,
+                      DbUpdateHandle*);
+static void
+db_finalize_dbterm_tree(int cret, DbUpdateHandle *);
 
 /*
 ** Static variables
@@ -2546,16 +2549,43 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key)
     return this;
 }
 
-static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle)
+static int
+db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj,
+                      DbUpdateHandle* handle)
 {
     DbTableTree *tb = &tbl->tree;
     TreeDbTerm **pp = find_node2(tb, key);
-
-    if (pp == NULL) return 0;
+    int flags = 0;
+
+    if (pp == NULL) {
+        if (obj == THE_NON_VALUE) {
+            return 0;
+        } else {
+            Eterm *objp = tuple_val(obj);
+            int arity = arityval(*objp);
+            Eterm *htop, *hend;
+
+            ASSERT(arity >= tb->common.keypos);
+            htop = HAlloc(p, arity + 1);
+            hend = htop + arity + 1;
+            sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1));
+            htop[tb->common.keypos] = key;
+            obj = make_tuple(htop);
+
+            if (db_put_tree(tbl, obj, 1) != DB_ERROR_NONE) {
+                return 0;
+            }
+
+            pp = find_node2(tb, key);
+            ASSERT(pp != NULL);
+            HRelease(p, hend, htop);
+            flags |= DB_NEW_OBJECT;
+        }
+    }
 
     handle->tb = tbl;
     handle->dbterm = &(*pp)->dbterm;
-    handle->mustResize = 0;
+    handle->flags = flags;
     handle->bp = (void**) pp;
     handle->new_size = (*pp)->dbterm.size;
 #if HALFWORD_HEAP
@@ -2564,15 +2594,21 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle
     return 1;
 }
 
-static void db_finalize_dbterm_tree(DbUpdateHandle* handle)
+static void
+db_finalize_dbterm_tree(int cret, DbUpdateHandle *handle)
 {
-    if (handle->mustResize) {
-	TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp;
+    DbTable *tbl = handle->tb;
+    DbTableTree *tb = &tbl->tree;
+    TreeDbTerm *bp = (TreeDbTerm *) *handle->bp;
 
+    if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) {
+        Eterm ret;
+        db_erase_tree(tbl, GETKEY(tb, bp->dbterm.tpl), &ret);
+    } else if (handle->flags & DB_MUST_RESIZE) {
 	db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm));
-	reset_static_stack(&handle->tb->tree);
+        reset_static_stack(tb);
 
-	free_term(&handle->tb->tree, oldp);
+        free_term(tb, bp);
     }
 #ifdef DEBUG
     handle->dbterm = 0;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 7eb80e3bb1..221f091396 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2699,10 +2699,10 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position)
     }
 
     ASSERT(((DbTableCommon*)handle->tb)->compress);
-    ASSERT(!handle->mustResize);
+    ASSERT(!(handle->flags & DB_MUST_RESIZE));
     handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
 					       handle->dbterm);
-    handle->mustResize = 1;
+    handle->flags |= DB_MUST_RESIZE;
     return handle->dbterm->tpl[position];
 }
 
@@ -2735,11 +2735,11 @@ void db_do_update_element(DbUpdateHandle* handle,
     #endif
 	return;
     }
-    if (!handle->mustResize) {
+    if (!(handle->flags & DB_MUST_RESIZE)) {
 	if (handle->tb->common.compress) {
 	    handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
 						       handle->dbterm);
-	    handle->mustResize = 1;
+	    handle->flags |= DB_MUST_RESIZE;
 	    oldval = handle->dbterm->tpl[position];
         #if HALFWORD_HEAP
 	    old_base = NULL;
@@ -2799,7 +2799,7 @@ both_size_set:
 
     /* write new value in old dbterm, finalize will make a flat copy */
     handle->dbterm->tpl[position] = newval;
-    handle->mustResize = 1;
+    handle->flags |= DB_MUST_RESIZE;
 
 #if HALFWORD_HEAP
     if (old_base && newval_sz > 0) {
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 5ace93c8ed..ca206c7f58 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -76,6 +76,9 @@ typedef struct db_term {
 union db_table;
 typedef union db_table DbTable;
 
+#define DB_MUST_RESIZE 1
+#define DB_NEW_OBJECT 2
+
 /* Info about a database entry while it's being updated
  * (by update_counter or update_element)
  */
@@ -84,7 +87,7 @@ typedef struct {
     DbTerm* dbterm;
     void** bp;         /* {Hash|Tree}DbTerm** */
     Uint new_size;
-    int mustResize;
+    int flags;
     void* lck;
 #if HALFWORD_HEAP
     unsigned char* abs_vec;  /* [i] true if dbterm->tpl[i] is absolute Eterm */
@@ -183,15 +186,14 @@ typedef struct db_table_method
 			       void *arg);
     void (*db_check_table)(DbTable* tb);
 
-    /* Lookup a dbterm for updating. Return false if not found.
-    */
-    int (*db_lookup_dbterm)(DbTable*, Eterm key, 
-			    DbUpdateHandle* handle); /* [out] */
+    /* Lookup a dbterm for updating. Return false if not found. */
+    int (*db_lookup_dbterm)(Process *, DbTable *, Eterm key, Eterm obj,
+                            DbUpdateHandle* handle);
 
-    /* Must be called for each db_lookup_dbterm that returned true,
-    ** even if dbterm was not updated.
-    */
-    void (*db_finalize_dbterm)(DbUpdateHandle* handle);
+    /* Must be called for each db_lookup_dbterm that returned true, even if
+    ** dbterm was not updated. If the handle was of a new object and cret is
+    ** not DB_ERROR_NONE, the object is removed from the table. */
+    void (*db_finalize_dbterm)(int cret, DbUpdateHandle* handle);
 
 } DbTableMethod;
 
-- 
cgit v1.2.3


From 10a1f3a2d5d9343bd4d2b8ec77c32ec18da21666 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Mar 2015 13:42:15 +0100
Subject: erts: Add missing binary offheap overhead in binary_to_term

Binary offheap overhead is used to trigger GC when a process is
referring "too much" binary offheap data.

Offheap binaries created from external format (binary_to_term,
distributed messages or compacted ets tables) were not accounted for.

Example: A process receiving a lot of binary data in distributed messages,
while not building much terms on its heap, could cause an extensive
memory consumption for garbage binaries.
---
 erts/emulator/beam/external.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9b9b4b2a62..ae011a34bd 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3396,6 +3396,7 @@ dec_term_atom_common:
 		    pb->size = n;
 		    pb->next = off_heap->first;
 		    off_heap->first = (struct erl_off_heap_header*)pb;
+                    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -3449,6 +3450,7 @@ dec_term_atom_common:
 		    pb->size = n;
 		    pb->next = off_heap->first;
 		    off_heap->first = (struct erl_off_heap_header*)pb;
+                    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -3747,6 +3749,7 @@ dec_term_atom_common:
 		hp += PROC_BIN_SIZE;
 		pb->next = off_heap->first;
 		off_heap->first = (struct erl_off_heap_header*)pb;
+                OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		pb->flags = 0;
 		*objp = make_binary(pb);
 		break;
@@ -3764,6 +3767,7 @@ dec_term_atom_common:
 		hp += PROC_BIN_SIZE;
 		pb->next = off_heap->first;
 		off_heap->first = (struct erl_off_heap_header*)pb;
+                OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		pb->flags = 0;
 
 		sub = (ErlSubBin*)hp;
-- 
cgit v1.2.3


From 8d3dba44bc2ac5ff9e724e90aa832854280b7d1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 27 Jun 2012 17:03:54 +0200
Subject: Initial Persistent HAMT - Map framework

Conflicts:
	erts/emulator/Makefile.in
	erts/emulator/beam/bif.tab
	erts/emulator/beam/erl_gc.c
	erts/emulator/beam/erl_gc.h
	erts/emulator/beam/erl_printf_term.c
	erts/emulator/beam/erl_term.c
	erts/emulator/beam/erl_term.h
---
 erts/emulator/beam/beam_emu.c        |   1 +
 erts/emulator/beam/bif.tab           |  10 +
 erts/emulator/beam/copy.c            |  47 +++-
 erts/emulator/beam/erl_gc.c          |   1 +
 erts/emulator/beam/erl_gc.h          |   3 +-
 erts/emulator/beam/erl_hashmap.c     | 506 +++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_hashmap.h     | 149 +++++++++++
 erts/emulator/beam/erl_printf_term.c |  76 +++++-
 erts/emulator/beam/erl_term.c        |   4 +-
 erts/emulator/beam/erl_term.h        |   9 +-
 erts/emulator/beam/erl_vm.h          |   4 +-
 erts/emulator/beam/global.h          |  10 +-
 12 files changed, 804 insertions(+), 16 deletions(-)
 create mode 100644 erts/emulator/beam/erl_hashmap.c
 create mode 100644 erts/emulator/beam/erl_hashmap.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b89c8b3900..b734d34872 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2801,6 +2801,7 @@ get_map_elements_fail:
 	}
 	PreFetch(1, next);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	reg[0] = r(0);
 	result = (*bf)(c_p, reg, I);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 1d0d214e77..a3fb21ee52 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -614,6 +614,16 @@ bif erlang:fun_info_mfa/1
 
 bif erlang:get_keys/0
 
+# Hash Array Mappped Trie
+bif hashmap:put/3
+bif hashmap:get/2
+#bif hashmap:info/1
+bif hashmap:to_list/1
+bif hashmap:new/0
+# bif hashmap:keys/1
+bif hashmap:size/1
+bif erlang:is_hashmap/1
+
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 0010f6a440..6294ba9412 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -33,6 +33,7 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "dtrace-wrapper.h"
+#include "erl_hashmap.h"
 
 static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*);
 
@@ -127,6 +128,35 @@ Uint size_object(Eterm obj)
 			obj = *bptr;
 			break;
 		    }
+		case HASHMAP_SUBTAG:
+		    switch (MAP_HEADER_TYPE(hdr)) {
+			case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+			case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+			case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+			    {
+				Eterm *head;
+				Uint sz;
+				head  = hashmap_val_rel(obj, base);
+				sz    = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+				sum  += 1 + sz + header_arity(hdr);
+				head += 1 + header_arity(hdr);
+
+				if (sz == 0) {
+				    goto pop_next;
+				}
+				while(sz-- > 1) {
+				    obj = head[sz];
+				    if (!IS_CONST(obj)) {
+					ESTACK_PUSH(s, obj);
+				    }
+				}
+				obj = head[0];
+			    }
+			    break;
+			default:
+			    erl_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+		    }
+		    break;
 		case SUB_BINARY_SUBTAG:
 		    {
 			Eterm real_bin;
@@ -459,7 +489,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		{
 		  ExternalThing *etp = (ExternalThing *) htop;
 
-		  i =  thing_arityval(hdr) + 1;
+		  i  = thing_arityval(hdr) + 1;
 		  tp = htop;
 
 		  while (i--)  {
@@ -473,6 +503,21 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		  *argp = make_external_rel(tp, dst_base);
 		}
 		break;
+	    case HASHMAP_SUBTAG:
+		tp = htop;
+		switch (MAP_HEADER_TYPE(hdr)) {
+		    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+		    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+			*htop++ = *objp++;
+		    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+			i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			while (i--)  { *htop++ = *objp++; }
+			*argp = make_hashmap_rel(tp, dstbase);
+			break;
+		    default:
+			erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+		}
+		break;
 	    case BIN_MATCHSTATE_SUBTAG:
 		erl_exit(ERTS_ABORT_EXIT,
 			 "copy_struct: matchstate term not allowed");
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d1a7ee113b..bdf7aa362e 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -31,6 +31,7 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "erl_map.h"
+#include "erl_hashmap.h"
 #include "error.h"
 #include "big.h"
 #include "erl_gc.h"
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index bf0496c112..3fec553684 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -56,6 +56,8 @@ do {                                                                    \
     switch ((HDR) & _HEADER_SUBTAG_MASK) {                              \
     case SUB_BINARY_SUBTAG: nelts++; break;                             \
     case MAP_SUBTAG: nelts+=map_get_size(PTR) + 1; break;               \
+    case HASHMAP_SUBTAG: nelts=hashmap_bitcount(MAP_HEADER_VAL(HDR));   \
+	 nelts += is_hashmap_header_head(HDR) ? 1 : 0; break;           \
     case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break;   \
     }                                                                   \
     gval    = make_boxed(HTOP);                                         \
@@ -63,7 +65,6 @@ do {                                                                    \
     *HTOP++ = HDR;                                                      \
     *PTR++  = gval;                                                     \
     while (nelts--) *HTOP++ = *PTR++;                                   \
-                                                                        \
 } while(0)
 
 #define in_area(ptr,start,nbytes) \
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
new file mode 100644
index 0000000000..b7d5d2e2eb
--- /dev/null
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -0,0 +1,506 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ *
+ * hashmaps are an adaption of Rich Hickeys Persistent HashMaps
+ *   which were an adaption of Phil Bagwells - Hash Array Mapped Tries
+ *
+ * Author: Björn-Egil Dahlberg
+ */
+/*
+ * Ls = lists:seq(1,188888).
+ * A = lists:foldl(fun(I,O) -> hashmap:put(I,I,O) end, hashmap:new(), Ls).
+ * lists:foreach(fun(I) -> io:format("looking up ~p got ~p~n", [I, hashmap:get(I, A)]), I = hashmap:get(I,A) end, Ls).
+ *
+ * lists:foldl(fun(I,O) -> hashmap:put(I,I,O) end, hashmap:new(), lists:seq(1,7)).
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "erl_process.h"
+#include "error.h"
+#include "bif.h"
+
+#include "erl_hashmap.h"
+
+#ifndef DECL_AM
+#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
+#endif
+
+static char *format_binary(Uint64 x, char *b) {
+    int z;
+    b[64] = '\0';
+    for (z = 0; z < 64; z++) { 
+	b[63-z] = ((x>>z) & 0x1) ? '1' : '0'; 
+    }
+    return b;
+}
+
+static Uint32 hashmap_shift_hash(Uint32 hx, Uint *lvl, Eterm key);
+static Uint32 hashmap_restore_hash(Uint lvl, Eterm key);
+static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node);
+static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
+static Eterm hashmap_to_list(Process *p, Eterm map);
+
+/* hashmap:new/0 */
+
+BIF_RETTYPE hashmap_new_0(BIF_ALIST_0) {
+    Eterm* hp;
+    hashmap_head_t *head;
+
+    hp   = HAlloc(BIF_P, HAMT_HEAD_EMPTY_SZ);
+    head = (hashmap_head_t *) hp;
+
+    head->thing_word = MAP_HEADER_HAMT_HEAD_BITMAP(0);
+    head->size       = 0;
+
+    BIF_RET(make_hashmap(head));
+}
+
+/* hashmap:put/3 */
+
+BIF_RETTYPE hashmap_put_3(BIF_ALIST_3) {
+    if (is_hashmap(BIF_ARG_3)) {
+	Uint32 hx = make_hash2(BIF_ARG_1);
+	Eterm map;
+
+	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+	ASSERT(is_hashmap(map));
+	BIF_RET(map);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* hashmap:to_list/1 */
+
+BIF_RETTYPE hashmap_to_list_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+	return hashmap_to_list(BIF_P, BIF_ARG_1);
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* hashmap:get/2 */
+
+BIF_RETTYPE hashmap_get_2(BIF_ALIST_2) {
+    if (is_hashmap(BIF_ARG_2)) {
+	const Eterm *value;
+	Uint32 hx = make_hash2(BIF_ARG_1);
+	
+	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
+	    BIF_RET(*value);
+	}
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* hashmap:size/1 */
+
+BIF_RETTYPE hashmap_size_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+	Eterm *head, *hp, res;
+	Uint size, hsz=0;
+
+	head = hashmap_val(BIF_ARG_1);
+	size = head[1];
+	(void) erts_bld_uint(NULL, &hsz, size);
+	hp = HAlloc(BIF_P, hsz);
+	res = erts_bld_uint(&hp, NULL, size);
+	BIF_RET(res);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* erlang:is_hashmap/1 */
+
+BIF_RETTYPE is_hashmap_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+	BIF_RET(am_true);
+    }
+    BIF_RET(am_false);
+}
+
+/* impl. */
+
+static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
+    Eterm *ptr, hdr;
+    Uint ix,slot, lvl = 0;
+    Uint32 hval,bp;
+
+    for (;;) {
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
+		ptr = list_val(node);
+		if (EQ(CAR(ptr), key)) {
+		    return &(CDR(ptr));
+		}
+		return NULL;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ix   = hashmap_index(hx);
+			hx   = hashmap_shift_hash(hx,&lvl,key);
+			node = ptr[ix+1];
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ix   = hashmap_index(hx);
+			hx   = hashmap_shift_hash(hx,&lvl,key);
+			node = ptr[ix+2];
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    node  = ptr[slot+1];
+			    break;
+			}
+			/* not occupied */
+			return NULL;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    node  = ptr[slot+2];
+			    break;
+			}
+			/* not occupied */
+			return NULL;
+		    default:
+			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %p\r\n", node);
+		break;
+	}
+    }
+    return NULL;
+}
+
+static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node) {
+    Eterm *hp = NULL, *nhp = NULL;
+    Eterm *ptr;
+    Eterm hdr,res,ckey,fake;
+    Uint32 ix, cix, bp, hval, chx;
+    Uint slot, lvl = 0, clvl;
+    Uint size = 0, n = 0, update_size = 1;
+    DECLARE_ESTACK(stack);
+
+    for (;;) {
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
+		ptr  = list_val(node);
+		ckey = CAR(ptr);
+		if (EQ(ckey, key)) {
+		    update_size = 0;
+		    goto unroll;
+		}
+		goto insert_subnodes;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(hx,&lvl,key);
+			size += HAMT_NODE_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+1];
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(hx,&lvl,key);
+			size += HAMT_HEAD_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+2];
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    node  = ptr[slot+1];
+			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); 
+			    size += HAMT_NODE_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			size += HAMT_NODE_BITMAP_SZ(n+1);
+			goto unroll;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    node  = ptr[slot+2];
+			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); 
+			    size += HAMT_HEAD_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			size += HAMT_HEAD_BITMAP_SZ(n+1);
+			goto unroll;
+		    default:
+			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %p\r\n", node);
+		break;
+	}
+    }
+insert_subnodes:
+    clvl  = lvl;
+    chx   = hashmap_restore_hash(clvl,ckey);
+    size += HAMT_NODE_BITMAP_SZ(2);
+    ix    = hashmap_index(hx);
+    cix   = hashmap_index(chx);
+
+    while (cix == ix) {
+	ESTACK_PUSH(stack, 0);
+	ESTACK_PUSH3(stack, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
+	size += HAMT_NODE_BITMAP_SZ(1);
+	hx    = hashmap_shift_hash(hx,&lvl,key);
+	chx   = hashmap_shift_hash(chx,&clvl,ckey);
+	ix    = hashmap_index(hx);
+	cix   = hashmap_index(chx);
+    }
+    ESTACK_PUSH3(stack, cix, ix, node);
+
+unroll:
+    size += 2;
+    hp  = HAlloc(p, size);
+    res = CONS(hp, key, value); hp += 2;
+
+    do {
+	node = ESTACK_POP(stack);
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		ix   = (Uint32) ESTACK_POP(stack);
+		cix  = (Uint32) ESTACK_POP(stack);
+
+		nhp   = hp;
+		*hp++ = MAP_HEADER_HAMT_NODE_BITMAP((1 << ix) | (1 << cix));
+		if (ix < cix) {
+		    *hp++ = res;
+		    *hp++ = node;
+		} else {
+		    *hp++ = node;
+		    *hp++ = res;
+		}
+		res = make_hashmap(nhp);
+		break;
+	    case TAG_PRIMARY_HEADER:
+		/* subnodes, fake it */
+		fake = node;
+		node = make_boxed(&fake);
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			slot  = (Uint)   ESTACK_POP(stack);
+			nhp   = hp;
+			n     = HAMT_NODE_ARRAY_SZ;
+			while(n--) { *hp++ = *ptr++; }
+			nhp[slot+1] = res;
+			res = make_hashmap(nhp);
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			slot  = (Uint)   ESTACK_POP(stack);
+			nhp   = hp;
+			n     = HAMT_HEAD_ARRAY_SZ - 2;
+			*hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++;
+			*hp++ = (*ptr++) + update_size;
+			while(n--) { *hp++ = *ptr++; }
+			nhp[slot+2] = res;
+			res = make_hashmap(nhp);
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			slot  = (Uint)   ESTACK_POP(stack);
+			bp    = (Uint32) ESTACK_POP(stack);
+			n     = (Uint32) ESTACK_POP(stack);
+			hval  = MAP_HEADER_VAL(hdr);
+			nhp   = hp;
+			*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval | bp); ptr++;
+
+			n -= slot;
+			while(slot--) { *hp++ = *ptr++; }
+			*hp++ = res;
+			if (hval & bp) { ptr++; n--; }
+			while(n--) { *hp++ = *ptr++; }
+
+			if ((hval | bp) == 0xffff) {
+			    *nhp = make_arityval(16);
+			}
+			res = make_hashmap(nhp);
+			break;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			slot  = (Uint)   ESTACK_POP(stack);
+			bp    = (Uint32) ESTACK_POP(stack);
+			n     = (Uint32) ESTACK_POP(stack);
+			hval  = MAP_HEADER_VAL(hdr);
+			nhp   = hp;
+			*hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++;
+			*hp++ = (*ptr++) + update_size;
+
+			n -= slot;
+			while(slot--) { *hp++ = *ptr++; }
+			*hp++ = res;
+			if (hval & bp) { ptr++; n--; }
+			while(n--) { *hp++ = *ptr++; }
+
+			if ((hval | bp) == 0xffff) {
+			    *nhp = MAP_HEADER_HAMT_HEAD_ARRAY;
+			}
+			res = make_hashmap(nhp);
+			break;
+		    default:
+			erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %x\r\n", primary_tag(node));
+		break;
+	}
+
+    } while(!ESTACK_ISEMPTY(stack));
+
+    DESTROY_ESTACK(stack);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+
+static Eterm hashmap_to_list(Process *p, Eterm node) {
+    Eterm *hp;
+    Eterm res = NIL;
+    Eterm *ptr, tup, hdr;
+    Uint sz, n;
+    DECLARE_ESTACK(stack);
+
+    ptr = boxed_val(node);
+    n   = (Uint)ptr[1];
+    hp  = HAlloc(p, n * (2 + 3));
+    ESTACK_PUSH(stack, node);
+    do {
+	node = ESTACK_POP(stack);
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		ptr = list_val(node);
+		tup = TUPLE2(hp, CAR(ptr), CDR(ptr)); hp += 3;
+		res = CONS(hp, tup, res); hp += 2;
+		break;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ptr++;
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ptr++;
+			sz = 16;
+			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
+			break;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			ptr++;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			ASSERT(sz < 17);
+			ptr++;
+			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
+			break;
+		    default:
+			erl_exit(1, "bad header\r\n");
+			break;
+		}
+	}
+    } while(!ESTACK_ISEMPTY(stack));
+
+    DESTROY_ESTACK(stack);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+
+static Uint32 hashmap_restore_hash(Uint lvl, Eterm key) {
+    Eterm heap[2], ret;
+
+    if (lvl < 8) {
+	return make_hash2(key) >> (4*lvl);
+    }
+    ret = CONS(heap, make_small(lvl), key);
+
+    return make_hash2(ret) >> (4*(lvl % 8));
+}
+
+static Uint32 hashmap_shift_hash(Uint32 hx, Uint *lvl, Eterm key) {
+    Eterm heap[2], ret;
+
+    /* rehash with [ lvl | key ] */
+    *lvl = *lvl + 1;
+    if (*lvl % 8) {
+	return hx >> 4;
+    }
+
+    ret = CONS(heap, make_small((*lvl)), key);
+    return make_hash2(ret);
+}
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
new file mode 100644
index 0000000000..03739fa1f9
--- /dev/null
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -0,0 +1,149 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+
+#ifndef __ERL_HASH_H__
+#define __ERL_HASH_H__
+
+#include "sys.h"
+
+Eterm erts_hashmap_get(Eterm key, Eterm map);
+
+/* erl_term.h stuff */
+#define make_hashmap(x)		make_boxed((Eterm*)(x))
+#define make_hashmap_rel 	make_boxed_rel
+#define is_hashmap(x)		(is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
+#define is_hashmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HASHMAP)
+#define hashmap_val(x)		_unchecked_boxed_val((x))
+#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE))
+
+/* HASH */
+
+
+#if defined(__GNUC__)
+#define hashmap_bitcount(x) 	(Uint32) __builtin_popcount((unsigned int) (x))
+#else
+const Uint32 SK5 = 0x55555555, SK3 = 0x33333333;
+const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF;
+
+/* CTPOP emulation */
+Uint32 hashmap_bitcount(Uint32 map) {
+    map -= (( map >> 1  ) & SK5 );
+    map  = ( map & SK3  ) + (( map >> 2 ) & SK3 );
+    map  = ( map & SKF0 ) + (( map >> 4 ) & SKF0);
+    map +=   map >> 8;
+    return ( map + ( map >> 16)) & 0x3F;
+}
+#endif
+
+/* hamt nodes v2.0
+ * 
+ * node :: leaf | array | bitmap
+ * head
+ */
+
+/* the head-node is a bitmap or array with an untagged size
+ */
+typedef struct hashmap_head_s {
+    Eterm thing_word;
+    Uint size;
+    Eterm items[1];
+} hashmap_head_t;
+ 
+/* the bitmap-node
+ * typedef struct hashmap_bitmap_node_s {
+ *     Eterm thing_word;
+ *     Eterm items[1];
+ * } hashmap_bitmap_node_t;
+ *
+ * the array-node is a tuple
+ * typedef struct hashmap_bitmap_node_s {
+ *     Eterm thing_word; 
+ *     Eterm items[1];
+ * } hashmap_bitmap_node_t;
+ *
+ * the leaf-node
+ * cons-cell
+ */
+
+/* thing_word tagscheme
+ * Need two bits for map subtags
+ *
+ * Original HEADER representation:
+ *
+ *     aaaaaaaaaaaaaaaa aaaaaaaaaatttt00       arity:26, tag:4
+ *
+ * For maps we have:
+ *
+ *     vvvvvvvvvvvvvvvv aaaaaaaamm111100       val:16, arity:8, mtype:2
+ *
+ * unsure about trailing zeros
+ *
+ * map-tag:
+ *     00 - flat map tag (non-hamt) -> val:16 = #items
+ *     01 - map-node bitmap tag     -> val:16 = bitmap
+ *     10 - map-head (array-node)   -> val:16 = 0xffff
+ *     11 - map-head (bitmap-node)  -> val:16 = bitmap
+ */
+
+/* erl_map.h stuff */
+
+#define MAP_HEADER_TAG_SZ                 (2)
+#define MAP_HEADER_ARITY_SZ               (8)
+#define MAP_HEADER_VAL_SZ                 (16)
+
+#define MAP_HEADER_TAG_FLAT               (0x0)
+#define MAP_HEADER_TAG_HAMT_NODE_BITMAP   (0x1)
+#define MAP_HEADER_TAG_HAMT_HEAD_ARRAY    (0x2)
+#define MAP_HEADER_TAG_HAMT_HEAD_BITMAP   (0x3)
+
+#define MAP_HEADER_TYPE(Hdr)  (((Hdr) >> (_HEADER_ARITY_OFFS)) & (0x3))
+#define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff))
+#define MAP_HEADER_VAL(Hdr)   (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff))
+
+#define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2)))
+
+#define MAKE_MAP_HEADER(Type,Arity,Val) \
+    (_make_header(((((Uint16)Val) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_HASHMAP))
+
+#define MAP_HEADER_HAMT_HEAD_ARRAY \
+    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff)
+
+#define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \
+    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp)
+
+#define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \
+    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp)
+
+#define HAMT_HEAD_EMPTY_SZ     (2)
+#define HAMT_NODE_ARRAY_SZ     (17)
+#define HAMT_HEAD_ARRAY_SZ     (18)
+#define HAMT_NODE_BITMAP_SZ(n) (1 + n)
+#define HAMT_HEAD_BITMAP_SZ(n) (2 + n)
+
+#define _HEADER_MAP_SUBTAG_MASK    (0xfc) /* 2 bits maps tag + 4 bits subtag + 2 ignore bits */
+/* SUBTAG_NODE_ARRAY is in fact a tuple with 16 elements */
+#define HAMT_SUBTAG_NODE_ARRAY   (((16 << _HEADER_ARITY_OFFS) | ARITYVAL_SUBTAG) & _HEADER_MAP_SUBTAG_MASK)
+#define HAMT_SUBTAG_NODE_BITMAP  ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
+#define HAMT_SUBTAG_HEAD_ARRAY   ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
+#define HAMT_SUBTAG_HEAD_BITMAP  ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
+
+#define hashmap_index(hash)      (((Uint32)hash) & 0xf)
+
+#endif
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index c982dc2080..f07fe30cf6 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -26,12 +26,13 @@
 #include "big.h"
 #include "erl_map.h"
 #include "erl_binary.h"
+#include "erl_hashmap.h"
 
 #define PRINT_CHAR(CNT, FN, ARG, C)					\
 do {									\
     int res__ = erts_printf_char((FN), (ARG), (C));			\
     if (res__ < 0)							\
-	return res__;							\
+	abort();							\
     (CNT) += res__;							\
 } while (0)
 
@@ -39,7 +40,7 @@ do {									\
 do {									\
     int res__ = erts_printf_string((FN), (ARG), (STR));			\
     if (res__ < 0)							\
-	return res__;							\
+	abort();							\
     (CNT) += res__;							\
 } while (0)
 
@@ -47,7 +48,7 @@ do {									\
 do {									\
     int res__ = erts_printf_buf((FN), (ARG), (char*)(BUF), (LEN));	\
     if (res__ < 0)							\
-	return res__;							\
+	abort();							\
     (CNT) += res__;							\
 } while (0)
 
@@ -55,7 +56,7 @@ do {									\
 do {									\
     int res__ = erts_printf_pointer((FN), (ARG), (void *) (PTR));	\
     if (res__ < 0)							\
-	return res__;							\
+	abort();							\
     (CNT) += res__;							\
 } while (0)
 
@@ -63,7 +64,7 @@ do {									\
 do {									\
     int res__ = erts_printf_uword((FN), (ARG), (C), (P), (W), (I));	\
     if (res__ < 0)							\
-	return res__;							\
+	abort();							\
     (CNT) += res__;							\
 } while (0)
 
@@ -71,7 +72,7 @@ do {									\
 do {									\
     int res__ = erts_printf_sword((FN), (ARG), (C), (P), (W), (I));	\
     if (res__ < 0)							\
-	return res__;							\
+	abort();							\
     (CNT) += res__;							\
 } while (0)
 
@@ -79,7 +80,7 @@ do {									\
 do {									\
     int res__ = erts_printf_double((FN), (ARG), (C), (P), (W), (I));	\
     if (res__ < 0)							\
-	return res__;							\
+	abort();							\
     (CNT) += res__;							\
 } while (0)
 
@@ -247,6 +248,17 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount)
 #define PRT_PATCH_FUN_SIZE     ((Eterm) 7)
 #define PRT_LAST_ARRAY_ELEMENT ((Eterm) 8) /* Note! Must be last... */
 
+#if 0
+static char *format_binary(Uint16 x, char *b) {
+    int z;
+    b[16] = '\0';
+    for (z = 0; z < 16; z++) { 
+	b[15-z] = ((x>>z) & 0x1) ? '1' : '0'; 
+    }
+    return b;
+}
+#endif
+
 static int
 print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	   Eterm* obj_base) /* ignored if !HALFWORD_HEAP */
@@ -575,6 +587,54 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		}
 	    }
 	    break;
+	case HASHMAP_DEF:
+	    {
+		Uint n,mapval;
+		Eterm *head;
+		head = hashmap_val(wobj);
+		mapval = MAP_HEADER_VAL(*head);
+		switch (MAP_HEADER_TYPE(*head)) {
+		    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP:
+			PRINT_STRING(res, fn, arg, "#<");
+			PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval);
+			PRINT_STRING(res, fn, arg, ">{");
+			WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
+			n = hashmap_bitcount(mapval);
+			ASSERT(n < 17);
+			head += 2;
+			if (n > 0) {
+			    n--;
+			    WSTACK_PUSH(s, head[n]);
+			    WSTACK_PUSH(s, PRT_TERM);
+			    while (n--) {
+				WSTACK_PUSH(s, PRT_COMMA);
+				WSTACK_PUSH(s, head[n]);
+				WSTACK_PUSH(s, PRT_TERM);
+			    }
+			}
+			break;
+		    case MAP_HEADER_TAG_HAMT_NODE_BITMAP:
+			n = hashmap_bitcount(mapval);
+			head++;
+			PRINT_CHAR(res, fn, arg, '<');
+			PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval);
+			PRINT_STRING(res, fn, arg, ">{");
+			WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
+			ASSERT(n < 17);
+			if (n > 0) {
+			    n--;
+			    WSTACK_PUSH(s, head[n]);
+			    WSTACK_PUSH(s, PRT_TERM);
+			    while (n--) {
+				WSTACK_PUSH(s, PRT_COMMA);
+				WSTACK_PUSH(s, head[n]);
+				WSTACK_PUSH(s, PRT_TERM);
+			    }
+			}
+			break;
+		}
+	    }
+	    break;
 	default:
 	    PRINT_STRING(res, fn, arg, "<unknown:");
 	    PRINT_POINTER(res, fn, arg, wobj);
@@ -584,11 +644,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
     }
 
  L_done:
-    
     DESTROY_WSTACK(s);
     return res;
 }
 
+
 int
 erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision,
 		 ErlPfEterm* term_base)
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 28cbe7004f..d6fb88ea61 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -86,11 +86,13 @@ unsigned tag_val_def(Wterm x)
 	    case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PID_DEF;
 	    case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PORT_DEF;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):	return EXTERNAL_REF_DEF;
+	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):	return MAP_DEF;
 	    case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
 	    case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
 	    case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
-	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):	return MAP_DEF;
+	    case (_TAG_HEADER_HASHMAP >> _TAG_PRIMARY_SIZE):	return HASHMAP_DEF;
 	  }
+ 
 	  break;
       }
       case TAG_PRIMARY_IMMED1: {
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 37014ccf94..fde90997e3 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -21,6 +21,7 @@
 #define __ERL_TERM_H
 
 #include "sys.h" /* defines HALFWORD_HEAP */
+#include "erl_hashmap.h"
 
 typedef UWord Wterm;  /* Full word terms */
 
@@ -141,6 +142,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define HEAP_BINARY_SUBTAG	(0x9 << _TAG_PRIMARY_SIZE) /* BINARY */
 #define SUB_BINARY_SUBTAG	(0xA << _TAG_PRIMARY_SIZE) /* BINARY */
 /*   _BINARY_XXX_MASK depends on 0xB being unused */
+#define HASHMAP_SUBTAG		(0xB << _TAG_PRIMARY_SIZE) /* HASHMAP */
 #define EXTERNAL_PID_SUBTAG	(0xC << _TAG_PRIMARY_SIZE) /* EXTERNAL_PID */
 #define EXTERNAL_PORT_SUBTAG	(0xD << _TAG_PRIMARY_SIZE) /* EXTERNAL_PORT */
 #define EXTERNAL_REF_SUBTAG	(0xE << _TAG_PRIMARY_SIZE) /* EXTERNAL_REF */
@@ -162,6 +164,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define _TAG_HEADER_EXTERNAL_REF  (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG)
 #define _TAG_HEADER_BIN_MATCHSTATE (TAG_PRIMARY_HEADER|BIN_MATCHSTATE_SUBTAG)
 #define _TAG_HEADER_MAP	 	(TAG_PRIMARY_HEADER|MAP_SUBTAG)
+#define _TAG_HEADER_HASHMAP	(TAG_PRIMARY_HEADER|HASHMAP_SUBTAG)
 
 
 #define _TAG_HEADER_MASK	0x3F
@@ -298,7 +301,9 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm)
 /* header (arityval or thing) access methods */
 #define _make_header(sz,tag)  ((Uint)(((sz) << _HEADER_ARITY_OFFS) + (tag)))
 #define is_header(x)	(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER)
-#define _unchecked_header_arity(x)	((x) >> _HEADER_ARITY_OFFS)
+//#define _unchecked_header_arity(x)	((x) >> _HEADER_ARITY_OFFS)
+#define _unchecked_header_arity(x) \
+    (is_hashmap_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS))
 _ET_DECLARE_CHECKED(Uint,header_arity,Eterm)
 #define header_arity(x)	_ET_APPLY(header_arity,(x))
 
@@ -361,6 +366,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
 	((((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_REFC_BIN) || \
 	 (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HEAP_BIN) || \
 	 (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_SUB_BIN))
+
 #define make_binary(x)	make_boxed((Eterm*)(x))
 #define is_binary(x)	(is_boxed((x)) && is_binary_header(*boxed_val((x))))
 #define is_not_binary(x) (!is_binary((x)))
@@ -1095,6 +1101,7 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define FLOAT_DEF		0xe
 #define BIG_DEF			0xf
 #define SMALL_DEF		0x10
+#define HASHMAP_DEF		0x11
 
 #if ET_DEBUG
 extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 6e9216bef3..3a9fb1e07b 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -117,9 +117,9 @@
 
 #if defined(DEBUG) || defined(CHECK_FOR_HOLES)
 #if HALFWORD_HEAP
-# define ERTS_HOLE_MARKER (0xaf5e78ccU)
+# define ERTS_HOLE_MARKER (0xdeadbeef)
 #else
-# define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL)
+# define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef)
 #endif
 #endif
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 5330f389e0..1fb069232a 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -539,6 +539,12 @@ do {							\
     }							\
 } while(0)
 
+#define WSTACK_DEBUG(s) \
+    do { \
+	fprintf(stderr, "wstack size   = %ld\r\n", s.wsp - s.wstart); \
+	fprintf(stderr, "wstack wstart = %p\r\n", s.wstart); \
+	fprintf(stderr, "wstack wsp    = %p\r\n", s.wsp); \
+    } while(0)
 
 /*
  * Do not free the stack after this, it may have pointers into what
@@ -581,7 +587,7 @@ do {						\
     ASSERT(s.wsp <= s.wend);			\
 } while (0)
 
-#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s)))
+#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s))
 
 #define WSTACK_PUSH(s, x)				\
 do {							\
@@ -648,7 +654,7 @@ do {						\
 
 #define WSTACK_COUNT(s) (s.wsp - s.wstart)
 #define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
-#define WSTACK_POP(s) (*(--s.wsp))
+#define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp))
 
 
 /* binary.c */
-- 
cgit v1.2.3


From 666ba589b76857b3592adf96d2cd096de159cb64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Sun, 23 Nov 2014 00:32:05 +0100
Subject: Add hashmap:info/1

---
 erts/emulator/beam/bif.tab       |   2 +-
 erts/emulator/beam/erl_hashmap.c | 131 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 132 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index a3fb21ee52..17e61f4664 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -617,7 +617,7 @@ bif erlang:get_keys/0
 # Hash Array Mappped Trie
 bif hashmap:put/3
 bif hashmap:get/2
-#bif hashmap:info/1
+bif hashmap:info/1
 bif hashmap:to_list/1
 bif hashmap:new/0
 # bif hashmap:keys/1
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index b7d5d2e2eb..5f913edfa6 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -27,6 +27,7 @@
  * lists:foreach(fun(I) -> io:format("looking up ~p got ~p~n", [I, hashmap:get(I, A)]), I = hashmap:get(I,A) end, Ls).
  *
  * lists:foldl(fun(I,O) -> hashmap:put(I,I,O) end, hashmap:new(), lists:seq(1,7)).
+ * lists:foldl(fun(I,O) -> hashmap:info(O), hashmap:put(I,I,O) end, hashmap:new(), lists:seq(1,5)).
  *
  */
 
@@ -61,6 +62,7 @@ static Uint32 hashmap_restore_hash(Uint lvl, Eterm key);
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node);
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
+static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
 /* hashmap:new/0 */
 
@@ -504,3 +506,132 @@ static Uint32 hashmap_shift_hash(Uint32 hx, Uint *lvl, Eterm key) {
     ret = CONS(heap, make_small((*lvl)), key);
     return make_hash2(ret);
 }
+
+/* hashmap:info/0 */
+
+static Eterm hashmap_info(Process *p, Eterm node) {
+    Eterm *hp, **hpp;
+    Eterm res = NIL, info = NIL;
+    Eterm *ptr, tup, hdr;
+    Uint sz;
+    DECL_AM(depth);
+    DECL_AM(leafs);
+    DECL_AM(bitmaps);
+    DECL_AM(arrays);
+    Uint nleaf=0, nbitmap=0, narray=0;
+    Uint bitmap_usage[16], leaf_usage[16];
+    Uint lvl = 0, clvl;
+    DECLARE_ESTACK(stack);
+
+    for (sz = 0; sz < 16; sz++) {
+	bitmap_usage[sz] = 0;
+	leaf_usage[sz] = 0;
+    }
+
+    ptr = boxed_val(node);
+    ESTACK_PUSH(stack, 0);
+    ESTACK_PUSH(stack, node);
+    do {
+	node = ESTACK_POP(stack);
+	clvl = ESTACK_POP(stack);
+	lvl  = MAX(lvl,clvl);
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		nleaf++;
+		leaf_usage[clvl] += 1;
+		break;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			narray++;
+			sz = 16;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+1]);
+			}
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			nbitmap++;
+			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			ASSERT(sz < 17);
+			bitmap_usage[sz-1] += 1;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+1]);
+			}
+			break;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			nbitmap++;
+			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			bitmap_usage[sz-1] += 1;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+2]);
+			}
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			narray++;
+			sz = 16;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+2]);
+			}
+			break;
+		    default:
+			erl_exit(1, "bad header\r\n");
+			break;
+		}
+	}
+    } while(!ESTACK_ISEMPTY(stack));
+
+
+    /* size */
+    sz = 0;
+    hashmap_bld_tuple_uint(NULL,&sz,16,leaf_usage);
+    hashmap_bld_tuple_uint(NULL,&sz,16,bitmap_usage);
+
+    /* alloc */
+    hp   = HAlloc(p, 2+3 + 3*(2+4) + sz);
+
+    info = hashmap_bld_tuple_uint(&hp,NULL,16,leaf_usage);
+    tup  = TUPLE3(hp, AM_leafs, make_small(nleaf),info); hp += 4;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    info = hashmap_bld_tuple_uint(&hp,NULL,16,bitmap_usage);
+    tup  = TUPLE3(hp, AM_bitmaps, make_small(nbitmap), info); hp += 4;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    tup  = TUPLE3(hp, AM_arrays, make_small(narray),NIL); hp += 4;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    tup  = TUPLE2(hp, AM_depth, make_small(lvl)); hp += 3;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    DESTROY_ESTACK(stack);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+
+static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]) {
+    Eterm res = THE_NON_VALUE;
+    Eterm *ts = (Eterm *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
+    Uint i;
+
+
+    for (i = 0; i < n; i++) {
+	ts[i] = erts_bld_uint(hpp, szp, nums[i]);
+    }
+    res = erts_bld_tuplev(hpp, szp, n, ts);
+    erts_free(ERTS_ALC_T_TMP, (void *) ts);
+    return res;
+}
+
+BIF_RETTYPE hashmap_info_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+	BIF_RET(hashmap_info(BIF_P,BIF_ARG_1));
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
-- 
cgit v1.2.3


From c3abb4e6825e7db2e8c4ad647edf55a067c91495 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 17 Nov 2014 20:49:03 +0100
Subject: Add hashmap:remove/2

---
 erts/emulator/beam/bif.tab       |   1 +
 erts/emulator/beam/erl_hashmap.c | 266 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 264 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 17e61f4664..cf606a9deb 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -617,6 +617,7 @@ bif erlang:get_keys/0
 # Hash Array Mappped Trie
 bif hashmap:put/3
 bif hashmap:get/2
+bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:to_list/1
 bif hashmap:new/0
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 5f913edfa6..31d54569e0 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -48,6 +48,7 @@
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 #endif
 
+#if 0
 static char *format_binary(Uint64 x, char *b) {
     int z;
     b[64] = '\0';
@@ -56,11 +57,13 @@ static char *format_binary(Uint64 x, char *b) {
     }
     return b;
 }
+#endif
 
 static Uint32 hashmap_shift_hash(Uint32 hx, Uint *lvl, Eterm key);
 static Uint32 hashmap_restore_hash(Uint lvl, Eterm key);
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node);
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
@@ -109,7 +112,7 @@ BIF_RETTYPE hashmap_get_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_2)) {
 	const Eterm *value;
 	Uint32 hx = make_hash2(BIF_ARG_1);
-	
+
 	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
 	    BIF_RET(*value);
 	}
@@ -117,6 +120,16 @@ BIF_RETTYPE hashmap_get_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
+/* hashmap:remove/2 */
+
+BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
+    if (is_hashmap(BIF_ARG_2)) {
+	Uint32 hx = make_hash2(BIF_ARG_1);
+
+	BIF_RET(hashmap_delete(BIF_P, hx, BIF_ARG_1, BIF_ARG_2));
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
 /* hashmap:size/1 */
 
 BIF_RETTYPE hashmap_size_1(BIF_ALIST_1) {
@@ -432,6 +445,254 @@ unroll:
     return res;
 }
 
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
+    Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
+    Eterm *ptr;
+    Eterm hdr,res = node;
+    Uint32 ix, bp, hval;
+    Uint slot, lvl = 0;
+    Uint size = 0, n = 0;
+    DECLARE_ESTACK(stack);
+
+    for (;;) {
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		if (EQ(CAR(list_val(node)), key)) {
+		    goto unroll;
+		}
+		goto not_found;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(hx,&lvl,key);
+			size += HAMT_NODE_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+1];
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(hx,&lvl,key);
+			size += HAMT_HEAD_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+2];
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    node  = ptr[slot+1];
+			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
+			    size += HAMT_NODE_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			goto not_found;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    node  = ptr[slot+2];
+			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18);
+			    size += HAMT_HEAD_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			goto not_found;
+		    default:
+			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %p\r\n", node);
+		break;
+	}
+    }
+
+unroll:
+    /* the size is bounded and atleast one less than the previous size */
+    size  -= 1;
+    hp     = HAlloc(p, size);
+    hp_end = hp + size;
+    res    = THE_NON_VALUE;
+
+    do {
+	node = ESTACK_POP(stack);
+
+	/* all nodes are things */
+	ptr = boxed_val(node);
+	hdr = *ptr;
+	ASSERT(is_header(hdr));
+
+	switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		ix  = (Uint) ESTACK_POP(stack);
+		nhp = hp;
+		if (res == THE_NON_VALUE) {
+		    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(0xffff ^ (1 << ix)); ptr++;
+		    n     = 16;
+		    n    -= ix;
+		    while(ix--) { *hp++ = *ptr++; }
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		    res = make_hashmap(nhp);
+		} else {
+		    n = HAMT_NODE_ARRAY_SZ;
+		    while(n--) { *hp++ = *ptr++; }
+		    nhp[ix+1] = res;
+		    res = make_hashmap(nhp);
+		}
+		break;
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ix  = (Uint) ESTACK_POP(stack);
+		nhp = hp;
+		if (res == THE_NON_VALUE) {
+		    n     = 16;
+		    n    -= ix;
+		    *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(0xffff ^ (1 << ix)); ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    while(ix--) { *hp++ = *ptr++; }
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		    res = make_hashmap(nhp);
+		} else {
+		    n     = 16;
+		    *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    while(n--) { *hp++ = *ptr++; }
+		    nhp[ix+2] = res;
+		    res = make_hashmap(nhp);
+		}
+		break;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		slot = (Uint)   ESTACK_POP(stack);
+		bp   = (Uint32) ESTACK_POP(stack);
+		n    = (Uint32) ESTACK_POP(stack);
+		nhp  = hp;
+
+		/* bitmap change matrix
+		 * res | none    leaf    bitmap
+		 * ----------------------------
+		 * n=1 | remove  remove  keep
+		 * n=2 | other   keep    keep
+		 * n>2 | shrink  keep    keep
+		 *
+		 * other: (remember, n is 2)
+		 *   shrink if the other bitmap value is a bitmap node
+		 *   remove if the other bitmap value is a leaf
+		 *
+		 * remove:
+		 *   this bitmap node is removed, res is moved up in tree (could be none)
+		 *   this is a special case of shrink
+		 *
+		 * keep:
+		 *   the current path index is still used down in the tree, need to keep it
+		 *   copy as usual with the updated res
+		 *
+		 * shrink:
+		 *   the current path index is no longer used down in the tree, remove it (shrink)
+		 */
+		if (res == THE_NON_VALUE) {
+		    if (n == 1) {
+			break;
+		    } else if (n == 2) {
+			if (slot == 0) {
+			    ix = 2; /* off by one 'cause hdr */
+			} else {
+			    ix = 1; /* off by one 'cause hdr */
+			}
+			if (primary_tag(ptr[ix]) == TAG_PRIMARY_LIST) {
+			    res = ptr[ix];
+			} else {
+			    hval  = MAP_HEADER_VAL(hdr);
+			    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp);
+			    *hp++ = ptr[ix];
+			    res = make_hashmap(nhp);
+			}
+		    } else {
+			/* n > 2 */
+			hval  = MAP_HEADER_VAL(hdr);
+			*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); ptr++;
+			n    -= slot;
+			while(slot--) { *hp++ = *ptr++; }
+			ptr++; n--;
+			while(n--) { *hp++ = *ptr++; }
+			res = make_hashmap(nhp);
+		    }
+		} else if (primary_tag(res) == TAG_PRIMARY_LIST && n == 1) {
+		    break;
+		} else {
+		    /* res is bitmap or leaf && n > 1, keep */
+		    n    -= slot;
+		    *hp++ = *ptr++;
+		    while(slot--) { *hp++ = *ptr++; }
+		    *hp++ = res;
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		    res = make_hashmap(nhp);
+		}
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		slot = (Uint)   ESTACK_POP(stack);
+		bp   = (Uint32) ESTACK_POP(stack);
+		n    = (Uint32) ESTACK_POP(stack);
+		nhp  = hp;
+
+		if (res != THE_NON_VALUE) {
+		    *hp++ = *ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    n    -= slot;
+		    while(slot--) { *hp++ = *ptr++; }
+		    *hp++ = res;
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		} else {
+		    hval  = MAP_HEADER_VAL(hdr);
+		    *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval ^ bp); ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    n    -= slot;
+		    while(slot--) { *hp++ = *ptr++; }
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		}
+		res = make_hashmap(nhp);
+		break;
+	    default:
+		erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+		break;
+	}
+    } while(!ESTACK_ISEMPTY(stack));
+    HRelease(p, hp_end, hp);
+not_found:
+    DESTROY_ESTACK(stack);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+
 static Eterm hashmap_to_list(Process *p, Eterm node) {
     Eterm *hp;
     Eterm res = NIL;
@@ -510,7 +771,7 @@ static Uint32 hashmap_shift_hash(Uint32 hx, Uint *lvl, Eterm key) {
 /* hashmap:info/0 */
 
 static Eterm hashmap_info(Process *p, Eterm node) {
-    Eterm *hp, **hpp;
+    Eterm *hp;
     Eterm res = NIL, info = NIL;
     Eterm *ptr, tup, hdr;
     Uint sz;
@@ -620,7 +881,6 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
     Eterm *ts = (Eterm *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
     Uint i;
 
-
     for (i = 0; i < n; i++) {
 	ts[i] = erts_bld_uint(hpp, szp, nums[i]);
     }
-- 
cgit v1.2.3


From f56956cfac208939bbfa2164c38cfe0c8907aa1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Sun, 23 Nov 2014 07:20:38 +0100
Subject: Refactor hashmap_shift

---
 erts/emulator/beam/erl_hashmap.c | 64 +++++++++++++++-------------------------
 1 file changed, 23 insertions(+), 41 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 31d54569e0..b36f0c6150 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -48,6 +48,11 @@
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 #endif
 
+#define hashmap_restore_hash(Heap,Lvl,Key) \
+    ((Lvl) < 8) ? make_hash2(Key) >> (4*(Lvl)) : make_hash2(CONS(Heap, make_small(Lvl), (Key))) >> (4*((Lvl) % 8))
+#define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
+    ((++(Lvl)) % 8) ? (Hx) >> 4 : make_hash2(CONS(Heap, make_small(Lvl), Key))
+
 #if 0
 static char *format_binary(Uint64 x, char *b) {
     int z;
@@ -59,8 +64,6 @@ static char *format_binary(Uint64 x, char *b) {
 }
 #endif
 
-static Uint32 hashmap_shift_hash(Uint32 hx, Uint *lvl, Eterm key);
-static Uint32 hashmap_restore_hash(Uint lvl, Eterm key);
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node);
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
@@ -160,6 +163,7 @@ BIF_RETTYPE is_hashmap_1(BIF_ALIST_1) {
 
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     Eterm *ptr, hdr;
+    Eterm th[2];
     Uint ix,slot, lvl = 0;
     Uint32 hval,bp;
 
@@ -179,12 +183,12 @@ static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
 		    case HAMT_SUBTAG_NODE_ARRAY:
 			ix   = hashmap_index(hx);
-			hx   = hashmap_shift_hash(hx,&lvl,key);
+			hx   = hashmap_shift_hash(th,hx,lvl,key);
 			node = ptr[ix+1];
 			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			ix   = hashmap_index(hx);
-			hx   = hashmap_shift_hash(hx,&lvl,key);
+			hx   = hashmap_shift_hash(th,hx,lvl,key);
 			node = ptr[ix+2];
 			break;
 		    case HAMT_SUBTAG_NODE_BITMAP:
@@ -195,7 +199,7 @@ static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
 
 			/* occupied */
 			if (bp & hval) {
-			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
 			    node  = ptr[slot+1];
 			    break;
 			}
@@ -209,7 +213,7 @@ static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
 
 			/* occupied */
 			if (bp & hval) {
-			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
 			    node  = ptr[slot+2];
 			    break;
 			}
@@ -232,6 +236,7 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
     Eterm *hp = NULL, *nhp = NULL;
     Eterm *ptr;
     Eterm hdr,res,ckey,fake;
+    Eterm th[2];
     Uint32 ix, cix, bp, hval, chx;
     Uint slot, lvl = 0, clvl;
     Uint size = 0, n = 0, update_size = 1;
@@ -255,14 +260,14 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
 		    case HAMT_SUBTAG_NODE_ARRAY:
 			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(hx,&lvl,key);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
 			size += HAMT_NODE_ARRAY_SZ;
 			ESTACK_PUSH2(stack, ix, node);
 			node  = ptr[ix+1];
 			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(hx,&lvl,key);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
 			size += HAMT_HEAD_ARRAY_SZ;
 			ESTACK_PUSH2(stack, ix, node);
 			node  = ptr[ix+2];
@@ -279,7 +284,7 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 
 			/* occupied */
 			if (bp & hval) {
-			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
 			    node  = ptr[slot+1];
 			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); 
 			    size += HAMT_NODE_BITMAP_SZ(n);
@@ -300,7 +305,7 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 
 			/* occupied */
 			if (bp & hval) {
-			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
 			    node  = ptr[slot+2];
 			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); 
 			    size += HAMT_HEAD_BITMAP_SZ(n);
@@ -321,7 +326,7 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
     }
 insert_subnodes:
     clvl  = lvl;
-    chx   = hashmap_restore_hash(clvl,ckey);
+    chx   = hashmap_restore_hash(th,clvl,ckey);
     size += HAMT_NODE_BITMAP_SZ(2);
     ix    = hashmap_index(hx);
     cix   = hashmap_index(chx);
@@ -330,8 +335,8 @@ insert_subnodes:
 	ESTACK_PUSH(stack, 0);
 	ESTACK_PUSH3(stack, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
 	size += HAMT_NODE_BITMAP_SZ(1);
-	hx    = hashmap_shift_hash(hx,&lvl,key);
-	chx   = hashmap_shift_hash(chx,&clvl,ckey);
+	hx    = hashmap_shift_hash(th,hx,lvl,key);
+	chx   = hashmap_shift_hash(th,chx,clvl,ckey);
 	ix    = hashmap_index(hx);
 	cix   = hashmap_index(chx);
     }
@@ -447,6 +452,7 @@ unroll:
 
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
     Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
+    Eterm th[2];
     Eterm *ptr;
     Eterm hdr,res = node;
     Uint32 ix, bp, hval;
@@ -469,14 +475,14 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
 		    case HAMT_SUBTAG_NODE_ARRAY:
 			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(hx,&lvl,key);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
 			size += HAMT_NODE_ARRAY_SZ;
 			ESTACK_PUSH2(stack, ix, node);
 			node  = ptr[ix+1];
 			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(hx,&lvl,key);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
 			size += HAMT_HEAD_ARRAY_SZ;
 			ESTACK_PUSH2(stack, ix, node);
 			node  = ptr[ix+2];
@@ -493,7 +499,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
 
 			/* occupied */
 			if (bp & hval) {
-			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
 			    node  = ptr[slot+1];
 			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
 			    size += HAMT_NODE_BITMAP_SZ(n);
@@ -513,7 +519,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
 
 			/* occupied */
 			if (bp & hval) {
-			    hx    = hashmap_shift_hash(hx,&lvl,key);
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
 			    node  = ptr[slot+2];
 			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18);
 			    size += HAMT_HEAD_BITMAP_SZ(n);
@@ -744,30 +750,6 @@ static Eterm hashmap_to_list(Process *p, Eterm node) {
     return res;
 }
 
-static Uint32 hashmap_restore_hash(Uint lvl, Eterm key) {
-    Eterm heap[2], ret;
-
-    if (lvl < 8) {
-	return make_hash2(key) >> (4*lvl);
-    }
-    ret = CONS(heap, make_small(lvl), key);
-
-    return make_hash2(ret) >> (4*(lvl % 8));
-}
-
-static Uint32 hashmap_shift_hash(Uint32 hx, Uint *lvl, Eterm key) {
-    Eterm heap[2], ret;
-
-    /* rehash with [ lvl | key ] */
-    *lvl = *lvl + 1;
-    if (*lvl % 8) {
-	return hx >> 4;
-    }
-
-    ret = CONS(heap, make_small((*lvl)), key);
-    return make_hash2(ret);
-}
-
 /* hashmap:info/0 */
 
 static Eterm hashmap_info(Process *p, Eterm node) {
-- 
cgit v1.2.3


From 6c94fede355561f0b4241005e5ffecdba210825d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 25 Nov 2014 16:58:37 +0100
Subject: Don't use modulus for power of 2

---
 erts/emulator/beam/erl_hashmap.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index b36f0c6150..11d2309fe3 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -49,9 +49,9 @@
 #endif
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
-    ((Lvl) < 8) ? make_hash2(Key) >> (4*(Lvl)) : make_hash2(CONS(Heap, make_small(Lvl), (Key))) >> (4*((Lvl) % 8))
+    ((Lvl) < 8) ? make_hash2(Key) >> (4*(Lvl)) : make_hash2(CONS(Heap, make_small(Lvl), (Key))) >> (4*((Lvl) & 7))
 #define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
-    ((++(Lvl)) % 8) ? (Hx) >> 4 : make_hash2(CONS(Heap, make_small(Lvl), Key))
+    ((++(Lvl)) & 7) ? (Hx) >> 4 : make_hash2(CONS(Heap, make_small(Lvl), Key))
 
 #if 0
 static char *format_binary(Uint64 x, char *b) {
-- 
cgit v1.2.3


From f602e2327c330969165430bbba1a0b997200606d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 14 Jan 2015 20:12:52 +0100
Subject: hashmap: is_key/2, keys/1 and values/1

---
 erts/emulator/beam/bif.tab       |   4 +-
 erts/emulator/beam/erl_hashmap.c | 123 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index cf606a9deb..ed85021f8a 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -621,9 +621,11 @@ bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:to_list/1
 bif hashmap:new/0
-# bif hashmap:keys/1
+bif hashmap:is_key/2
+bif hashmap:keys/1
 bif hashmap:size/1
 bif erlang:is_hashmap/1
+bif hashmap:values/1
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 11d2309fe3..5cb768a3fe 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -68,6 +68,8 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
+static Eterm hashmap_keys(Process *p, Eterm map);
+static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
 /* hashmap:new/0 */
@@ -159,6 +161,38 @@ BIF_RETTYPE is_hashmap_1(BIF_ALIST_1) {
     BIF_RET(am_false);
 }
 
+/* hashmap:is_key/2
+ */
+
+BIF_RETTYPE hashmap_is_key_2(BIF_ALIST_2) {
+    if (is_hashmap(BIF_ARG_1)) {
+	Uint32 hx = make_hash2(BIF_ARG_1);
+
+	BIF_RET(hashmap_get(hx, BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* hashmap:keys/1
+ */
+
+BIF_RETTYPE hashmap_keys_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+	BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1));
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/* hashmap:keys/1
+ */
+
+BIF_RETTYPE hashmap_values_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+	BIF_RET(hashmap_values(BIF_P, BIF_ARG_1));
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 /* impl. */
 
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
@@ -750,6 +784,95 @@ static Eterm hashmap_to_list(Process *p, Eterm node) {
     return res;
 }
 
+typedef void hashmap_doer(Eterm*, void*);
+
+static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr, void* farg) {
+    Eterm *ptr, hdr;
+    Uint sz;
+    DECLARE_ESTACK(stack);
+
+    ESTACK_PUSH(stack, node);
+    do {
+	node = ESTACK_POP(stack);
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		ptr = list_val(node);
+		(*fptr)(ptr, farg); /* Do! */
+		break;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ptr++;
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ptr++;
+			sz = 16;
+			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
+			break;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			ptr++;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			ASSERT(sz < 17);
+			ptr++;
+			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
+			break;
+		    default:
+			erl_exit(1, "bad header\r\n");
+			break;
+		}
+	}
+    } while(!ESTACK_ISEMPTY(stack));
+
+    DESTROY_ESTACK(stack);
+}
+
+typedef struct {
+    Eterm* hp;
+    Eterm res;
+}hashmap_keys_state;
+static void hashmap_keys_doer(Eterm* kv, hashmap_keys_state*);
+
+static Eterm hashmap_keys(Process* p, Eterm node) {
+    hashmap_head_t* root;
+    hashmap_keys_state state;
+
+    root = (hashmap_head_t*) boxed_val(node);
+    state.hp  = HAlloc(p, root->size * 2);
+    state.res = NIL;
+    hashmap_do_foreach(node, (hashmap_doer*)hashmap_keys_doer, &state);
+    return state.res;
+}
+
+static void hashmap_keys_doer(Eterm* kv, hashmap_keys_state* statep) {
+    statep->res = CONS(statep->hp, CAR(kv), statep->res);
+    statep->hp += 2;
+}
+
+typedef struct {
+    Eterm* hp;
+    Eterm res;
+}hashmap_values_state;
+static void hashmap_values_doer(Eterm* kv, hashmap_values_state*);
+
+static Eterm hashmap_values(Process* p, Eterm node) {
+    hashmap_head_t* root;
+    hashmap_values_state state;
+
+    root = (hashmap_head_t*) boxed_val(node);
+    state.hp  = HAlloc(p, root->size * 2);
+    state.res = NIL;
+    hashmap_do_foreach(node, (hashmap_doer*)hashmap_values_doer, &state);
+    return state.res;
+}
+
+static void hashmap_values_doer(Eterm* kv, hashmap_values_state* statep) {
+    statep->res = CONS(statep->hp, CDR(kv), statep->res);
+    statep->hp += 2;
+}
+
 /* hashmap:info/0 */
 
 static Eterm hashmap_info(Process *p, Eterm node) {
-- 
cgit v1.2.3


From 96f6e470284858f1b8644ea949283dad24161dfe Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Jan 2015 21:23:44 +0100
Subject: erts: Fix bug in _make_header macro

Called with a signed int 'sz' argument on 64 bit
would cause sign extension 'sz' was larger than 33554431.
---
 erts/emulator/beam/erl_term.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index fde90997e3..72946d9ddf 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -299,7 +299,7 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm)
 #define atom_val(x)	_ET_APPLY(atom_val,(x))
 
 /* header (arityval or thing) access methods */
-#define _make_header(sz,tag)  ((Uint)(((sz) << _HEADER_ARITY_OFFS) + (tag)))
+#define _make_header(sz,tag) ((Uint)(((Uint)(sz) << _HEADER_ARITY_OFFS) + (tag)))
 #define is_header(x)	(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER)
 //#define _unchecked_header_arity(x)	((x) >> _HEADER_ARITY_OFFS)
 #define _unchecked_header_arity(x) \
-- 
cgit v1.2.3


From 7e045b2c7e545d5640e74280b2e7f75761961c39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 12 Mar 2015 18:43:19 +0100
Subject: erts: Fix beam_load assert

---
 erts/emulator/beam/beam_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index fce710f723..02689e5b19 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3727,7 +3727,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     op->a[j+size] = Fail;
 
 #ifdef DEBUG
-    for (i = 0; i < size; i++) {
+    for (i = 0; i < size - 1; i++) {
 	ASSERT(op->a[i+3].val <= op->a[i+4].val);
     }
 #endif
-- 
cgit v1.2.3


From 2dfa8ceaf30d69c9e5753b706dafe741b3cda4ca Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Jan 2015 21:25:50 +0100
Subject: erts: Add matching of hashmaps

---
 erts/emulator/beam/utils.c | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index b341c4d949..cea20a6002 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2325,6 +2325,32 @@ tailrecur_ne:
 		    }
 		    break; /* not equal */
 		}
+	    case HASHMAP_SUBTAG:
+		{
+		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != hdr)
+			goto not_equal;
+
+		    aa = hashmap_val_rel(a, a_base) + 1;
+		    bb = hashmap_val_rel(b, b_base) + 1;
+		    switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			aa++; bb++;
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			sz = 16;
+			break;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			aa++; bb++;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			ASSERT(sz > 0 && sz < 16);
+			break;
+		    default:
+			erl_exit(1, "Unknown hashmap subsubtag\n");
+		    }
+		    goto term_array;
+		}
+	    default:
+		ASSERT(!"Unknown boxed subtab in EQ");
 	    }
 	    break;
 	}
-- 
cgit v1.2.3


From eda3fa566070f658e396fe1a2297d956f4f364c1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 19 Jan 2015 20:40:45 +0100
Subject: erts: First recursive version of hashmap:merge

---
 erts/emulator/beam/bif.tab       |   1 +
 erts/emulator/beam/erl_hashmap.c | 225 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 226 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index ed85021f8a..6408883e96 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -626,6 +626,7 @@ bif hashmap:keys/1
 bif hashmap:size/1
 bif erlang:is_hashmap/1
 bif hashmap:values/1
+bif hashmap:merge/2
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 5cb768a3fe..74f27fb3ac 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -70,6 +70,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, Uint lvl, Uint *sizep, int keepA);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
 /* hashmap:new/0 */
@@ -193,6 +194,16 @@ BIF_RETTYPE hashmap_values_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
+BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
+    if (is_hashmap(BIF_ARG_1) && is_hashmap(BIF_ARG_2)) {
+	hashmap_head_t* a = (hashmap_head_t*) hashmap_val(BIF_ARG_1);
+	hashmap_head_t* b = (hashmap_head_t*) hashmap_val(BIF_ARG_2);
+	Uint size = a->size + b->size;
+	BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0, &size, 0));
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 /* impl. */
 
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
@@ -784,6 +795,220 @@ static Eterm hashmap_to_list(Process *p, Eterm node) {
     return res;
 }
 
+#define HALLOC_EXTRA 200
+
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB,
+			   Uint lvl, Uint *sizep, int keepA) {
+    Eterm *hp, *nhp;
+    Eterm *ptrA, *ptrB;
+    Eterm array[16];
+    Eterm hdrA, hdrB;
+    Eterm th[2];
+    Uint32 ahx, bhx, abm, bbm, rbm, bit;
+    Uint ix;
+
+    if (primary_tag(nodeA) == TAG_PRIMARY_BOXED &&
+	primary_tag(nodeB) == TAG_PRIMARY_LIST) {
+	/* Avoid implementing this combination by switching places */
+	Eterm tmp = nodeA;
+	nodeA = nodeB;
+	nodeB = tmp;
+	keepA = !keepA;
+    }
+
+    switch (primary_tag(nodeA)) {
+    case TAG_PRIMARY_LIST: { /* LEAF NODE [K|V] */
+	ptrA = list_val(nodeA);
+	switch (primary_tag(nodeB)) {
+	case TAG_PRIMARY_LIST: { /* LEAF NODE [K|V] */
+	    ptrB = list_val(nodeB);
+
+	    if (EQ(CAR(ptrA), CAR(ptrB))) {
+		--(*sizep);
+		return keepA ? nodeA : nodeB;
+	    }
+	    ahx = hashmap_restore_hash(th, lvl, CAR(ptrA));
+	    bhx = hashmap_restore_hash(th, lvl, CAR(ptrB));
+	    abm = 1 << hashmap_index(ahx);
+	    bbm = 1 << hashmap_index(bhx);
+
+	    if (abm != bbm) {
+		hp = HAllocX(p, HAMT_NODE_BITMAP_SZ(2), HALLOC_EXTRA);
+		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(abm | bbm);
+		if (abm < bbm) {
+		    hp[1] = CAR(ptrA);
+		    hp[2] = CAR(ptrB);
+		} else {
+		    hp[1] = CAR(ptrB);
+		    hp[2] = CAR(ptrA);
+		}
+	    }
+	    else {
+		hp = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(abm);
+		hp[1] = hashmap_merge(p, nodeA, nodeB, lvl+1, sizep, keepA);
+	    }
+	    return make_hashmap(hp);
+	}
+	case TAG_PRIMARY_BOXED: {
+	    ptrB = boxed_val(nodeB);
+	    ASSERT(is_header(*ptrB));
+	    hdrB = *ptrB++;
+
+	    ahx = hashmap_restore_hash(th, lvl, CAR(ptrA));
+	    abm = 1 << hashmap_index(ahx);
+
+	    switch(hdrB & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ptrB++;
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		bbm = 0xffff;
+		ptrA = &nodeA;
+		break;
+
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		ptrB++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		bbm = MAP_HEADER_VAL(hdrB);
+		ptrA = &nodeA;
+		break;
+
+	    default:
+		erl_exit(1, "bad header tag %ld\r\n", *ptrB & _HEADER_MAP_SUBTAG_MASK);
+		break;
+	    }
+	    break;
+	}
+	default:
+	    erl_exit(1, "bad primary tag %ld\r\n", nodeB);
+	}
+	break;
+    }
+    case TAG_PRIMARY_BOXED: {
+	ptrA = boxed_val(nodeA);
+	hdrA = *ptrA;
+	ASSERT(is_header(hdrA));
+	ptrA++;
+	switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
+	case HAMT_SUBTAG_HEAD_ARRAY:
+	    ptrA++;
+	case HAMT_SUBTAG_NODE_ARRAY: {
+	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
+	    ptrB = boxed_val(nodeB);
+	    hdrB = *ptrB;
+	    ASSERT(is_header(hdrB));
+	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ASSERT(lvl == 0);
+		abm = 0xffff;
+		bbm = 0xffff;
+		ptrB += 2;
+		break;
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		ASSERT(lvl > 0);
+		abm = 0xffff;
+		bbm = 0xffff;
+		ptrB++;
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		ASSERT(lvl == 0);
+		abm = 0xffff;
+		bbm = MAP_HEADER_VAL(hdrB);
+		ptrB += 2;
+		break;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		ASSERT(lvl > 0);
+		abm = 0xffff;
+		bbm = MAP_HEADER_VAL(hdrB);
+		ptrB += 1;
+		break;
+	    default:
+		erl_exit(1, "bad header tag %ld\r\n", *ptrB & _HEADER_MAP_SUBTAG_MASK);
+	    }
+	    break;
+	}
+	case HAMT_SUBTAG_HEAD_BITMAP:
+	    ptrA++;
+	case HAMT_SUBTAG_NODE_BITMAP: {
+	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
+	    abm = MAP_HEADER_VAL(hdrA);
+	    ptrB = boxed_val(nodeB);
+	    hdrB = *ptrB;
+	    ASSERT(is_header(hdrB));
+	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ASSERT(lvl == 0);
+		bbm = 0xffff;
+		ptrB += 2;
+		break;
+
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		ASSERT(lvl > 0);
+		bbm = 0xffff;
+		ptrB++;
+		break;
+
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		ASSERT(lvl == 0);
+		bbm = MAP_HEADER_VAL(hdrB);
+		ptrB += 2;
+		break;
+
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		ASSERT(lvl > 0);
+		bbm = MAP_HEADER_VAL(hdrB);
+		ptrB += 1;
+		break;
+
+	    default:
+		erl_exit(1, "bad header tag %ld\r\n", *ptrB & _HEADER_MAP_SUBTAG_MASK);
+	    }
+	    break;
+	}
+	default:
+	    erl_exit(1, "bad primary tag %ld\r\n", nodeA);
+	}
+	break;
+    }
+    default:
+	erl_exit(1, "bad primary tag %ld\r\n", nodeA);
+    }
+
+    ix = 0;
+    rbm = abm | bbm;
+    ASSERT(rbm != 0);
+    do {
+	Uint32 next = rbm & (rbm-1);
+	bit = rbm ^ next;
+	rbm = next;
+	if (abm & bit) {
+	    if (bbm & bit) {
+		array[ix++] = hashmap_merge(p, *ptrA++, *ptrB++, lvl+1, sizep, keepA);
+	    } else {
+		array[ix++] = *ptrA++;
+	    }
+	} else {
+	    ASSERT(bbm & bit);
+	    array[ix++] = *ptrB++;
+	}
+    }while (rbm);
+
+    if (lvl == 0) {
+	nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(ix), HALLOC_EXTRA);
+	hp = nhp;
+	*hp++ = (ix == 16) ? MAP_HEADER_HAMT_HEAD_ARRAY
+	                   : MAP_HEADER_HAMT_HEAD_BITMAP(abm | bbm);
+	*hp++ = *sizep;
+    } else {
+	nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(ix), HALLOC_EXTRA);
+	hp = nhp;
+	*hp++ = (ix == 16) ? make_arityval(16)
+	                   : MAP_HEADER_HAMT_NODE_BITMAP(abm | bbm);
+    }
+    memcpy(hp, array, ix * sizeof(Eterm));
+    return make_boxed(nhp);
+}
+
 typedef void hashmap_doer(Eterm*, void*);
 
 static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr, void* farg) {
-- 
cgit v1.2.3


From f3c7a82d45db4c8d9a9f3579726719360384da50 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Jan 2015 14:17:41 +0100
Subject: First non-recursive version of hashmap:merge/2

---
 erts/emulator/beam/erl_hashmap.c | 180 +++++++++++++++++++++++++--------------
 1 file changed, 116 insertions(+), 64 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 74f27fb3ac..bdcb479c30 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -70,7 +70,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, Uint lvl, Uint *sizep, int keepA);
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
 /* hashmap:new/0 */
@@ -196,10 +196,7 @@ BIF_RETTYPE hashmap_values_1(BIF_ALIST_1) {
 
 BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_1) && is_hashmap(BIF_ARG_2)) {
-	hashmap_head_t* a = (hashmap_head_t*) hashmap_val(BIF_ARG_1);
-	hashmap_head_t* b = (hashmap_head_t*) hashmap_val(BIF_ARG_2);
-	Uint size = a->size + b->size;
-	BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0, &size, 0));
+	BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -797,15 +794,46 @@ static Eterm hashmap_to_list(Process *p, Eterm node) {
 
 #define HALLOC_EXTRA 200
 
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB,
-			   Uint lvl, Uint *sizep, int keepA) {
+#if defined(VALGRIND) || defined(GO_SUPER_FAST)
+# define UNDEF(V,I)
+#else
+# define UNDEF(V,I) V=I
+#endif
+
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
+    struct {
+	Eterm *ptrA, *ptrB;
+	Uint32 abm, bbm, rbm;
+	Uint ix;
+	int keepA;
+	Eterm array[16];
+    }stack[16];
     Eterm *hp, *nhp;
     Eterm *ptrA, *ptrB;
-    Eterm array[16];
     Eterm hdrA, hdrB;
     Eterm th[2];
     Uint32 ahx, bhx, abm, bbm, rbm, bit;
     Uint ix;
+    Uint size;
+    int keepA = 0;
+    unsigned lvl = 0;
+    Eterm res = THE_NON_VALUE;
+
+    UNDEF(abm,0);
+    UNDEF(bbm,0);
+
+    /*
+     * Strategy: Do depth-first traversal of both trees (at the same time)
+     * and merge each pair of nodes.
+     */
+
+    {
+	hashmap_head_t* a = (hashmap_head_t*) hashmap_val(nodeA);
+	hashmap_head_t* b = (hashmap_head_t*) hashmap_val(nodeB);
+	size = a->size + b->size;
+    }
+
+recurse:
 
     if (primary_tag(nodeA) == TAG_PRIMARY_BOXED &&
 	primary_tag(nodeB) == TAG_PRIMARY_LIST) {
@@ -817,40 +845,27 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB,
     }
 
     switch (primary_tag(nodeA)) {
-    case TAG_PRIMARY_LIST: { /* LEAF NODE [K|V] */
+    case TAG_PRIMARY_LIST: {
 	ptrA = list_val(nodeA);
 	switch (primary_tag(nodeB)) {
-	case TAG_PRIMARY_LIST: { /* LEAF NODE [K|V] */
+	case TAG_PRIMARY_LIST: { /* LEAF + LEAF */
 	    ptrB = list_val(nodeB);
 
 	    if (EQ(CAR(ptrA), CAR(ptrB))) {
-		--(*sizep);
-		return keepA ? nodeA : nodeB;
-	    }
-	    ahx = hashmap_restore_hash(th, lvl, CAR(ptrA));
-	    bhx = hashmap_restore_hash(th, lvl, CAR(ptrB));
-	    abm = 1 << hashmap_index(ahx);
-	    bbm = 1 << hashmap_index(bhx);
-
-	    if (abm != bbm) {
-		hp = HAllocX(p, HAMT_NODE_BITMAP_SZ(2), HALLOC_EXTRA);
-		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(abm | bbm);
-		if (abm < bbm) {
-		    hp[1] = CAR(ptrA);
-		    hp[2] = CAR(ptrB);
-		} else {
-		    hp[1] = CAR(ptrB);
-		    hp[2] = CAR(ptrA);
-		}
-	    }
-	    else {
-		hp = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
-		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(abm);
-		hp[1] = hashmap_merge(p, nodeA, nodeB, lvl+1, sizep, keepA);
+		--size;
+		res = keepA ? nodeA : nodeB;
+	    } else {
+		ahx = hashmap_restore_hash(th, lvl, CAR(ptrA));
+		bhx = hashmap_restore_hash(th, lvl, CAR(ptrB));
+		abm = 1 << hashmap_index(ahx);
+		bbm = 1 << hashmap_index(bhx);
+
+		ptrA = &nodeA;
+		ptrB = &nodeB;
 	    }
-	    return make_hashmap(hp);
+	    break;
 	}
-	case TAG_PRIMARY_BOXED: {
+	case TAG_PRIMARY_BOXED: { /* LEAF + NODE */
 	    ptrB = boxed_val(nodeB);
 	    ASSERT(is_header(*ptrB));
 	    hdrB = *ptrB++;
@@ -884,7 +899,7 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB,
 	}
 	break;
     }
-    case TAG_PRIMARY_BOXED: {
+    case TAG_PRIMARY_BOXED: { /* NODE + NODE */
 	ptrA = boxed_val(nodeA);
 	hdrA = *ptrA;
 	ASSERT(is_header(hdrA));
@@ -974,39 +989,76 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB,
 	erl_exit(1, "bad primary tag %ld\r\n", nodeA);
     }
 
-    ix = 0;
-    rbm = abm | bbm;
-    ASSERT(rbm != 0);
-    do {
-	Uint32 next = rbm & (rbm-1);
-	bit = rbm ^ next;
-	rbm = next;
-	if (abm & bit) {
-	    if (bbm & bit) {
-		array[ix++] = hashmap_merge(p, *ptrA++, *ptrB++, lvl+1, sizep, keepA);
+    for (;;) {
+	if (is_value(res)) { /* We have a complete (sub-)tree or leaf */
+	    if (lvl == 0)
+		return res;
+
+	    /* Pop from stack and continue build parent node */
+	    lvl--;
+	    abm = stack[lvl].abm;
+	    bbm = stack[lvl].bbm;
+	    rbm = stack[lvl].rbm;
+	    ix  = stack[lvl].ix;
+	    stack[lvl].array[ix++] = res;
+	    res = THE_NON_VALUE;
+	    if (rbm) {
+		ptrA = stack[lvl].ptrA;
+		ptrB = stack[lvl].ptrB;
+		keepA = stack[lvl].keepA;
+	    }
+	} else { /* Start build a node */
+	    ix = 0;
+	    rbm = abm | bbm;
+	    ASSERT(!(rbm == 0 && lvl > 0));
+	}
+
+	while (rbm) {
+	    Uint32 next = rbm & (rbm-1);
+	    bit = rbm ^ next;
+	    rbm = next;
+	    if (abm & bit) {
+		if (bbm & bit) {
+		    /* Bit clash. Push and resolve by recursive merge */
+		    if (rbm) {
+			stack[lvl].ptrA = ptrA + 1;
+			stack[lvl].ptrB = ptrB + 1;
+			stack[lvl].keepA = keepA;
+		    }
+		    stack[lvl].abm = abm;
+		    stack[lvl].bbm = bbm;
+		    stack[lvl].rbm = rbm;
+		    stack[lvl].ix  = ix;
+		    nodeA = *ptrA;
+		    nodeB = *ptrB;
+		    lvl++;
+		    goto recurse;
+		} else {
+		    stack[lvl].array[ix++] = *ptrA++;
+		}
 	    } else {
-		array[ix++] = *ptrA++;
+		ASSERT(bbm & bit);
+		stack[lvl].array[ix++] = *ptrB++;
 	    }
+	}
+
+	if (lvl == 0) {
+	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(ix), HALLOC_EXTRA);
+	    hp = nhp;
+	    *hp++ = (ix == 16) ? MAP_HEADER_HAMT_HEAD_ARRAY
+		: MAP_HEADER_HAMT_HEAD_BITMAP(abm | bbm);
+	    *hp++ = size;
 	} else {
-	    ASSERT(bbm & bit);
-	    array[ix++] = *ptrB++;
+	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(ix), HALLOC_EXTRA);
+	    hp = nhp;
+	    *hp++ = (ix == 16) ? make_arityval(16)
+				   : MAP_HEADER_HAMT_NODE_BITMAP(abm | bbm);
 	}
-    }while (rbm);
-
-    if (lvl == 0) {
-	nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(ix), HALLOC_EXTRA);
-	hp = nhp;
-	*hp++ = (ix == 16) ? MAP_HEADER_HAMT_HEAD_ARRAY
-	                   : MAP_HEADER_HAMT_HEAD_BITMAP(abm | bbm);
-	*hp++ = *sizep;
-    } else {
-	nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(ix), HALLOC_EXTRA);
-	hp = nhp;
-	*hp++ = (ix == 16) ? make_arityval(16)
-	                   : MAP_HEADER_HAMT_NODE_BITMAP(abm | bbm);
+	memcpy(hp, stack[lvl].array, ix * sizeof(Eterm));
+	res = make_boxed(nhp);
     }
-    memcpy(hp, array, ix * sizeof(Eterm));
-    return make_boxed(nhp);
+
+    return res;
 }
 
 typedef void hashmap_doer(Eterm*, void*);
-- 
cgit v1.2.3


From 10a4f2777669ef2d7212e234cd272c64bb707e07 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Jan 2015 14:19:24 +0100
Subject: erts: Add missing parenthesis in MAKE_MAP_HEADER

---
 erts/emulator/beam/erl_hashmap.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index 03739fa1f9..4ef795b11e 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -120,7 +120,7 @@ typedef struct hashmap_head_s {
 #define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2)))
 
 #define MAKE_MAP_HEADER(Type,Arity,Val) \
-    (_make_header(((((Uint16)Val) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_HASHMAP))
+    (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_HASHMAP))
 
 #define MAP_HEADER_HAMT_HEAD_ARRAY \
     MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff)
-- 
cgit v1.2.3


From 4007301c3cbd9e86a42cb122692736f493d2d3f9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Jan 2015 18:09:05 +0100
Subject: erts: Refactor hashmap_do_foreach

to use a common struct hashmap_doer_state.
---
 erts/emulator/beam/erl_hashmap.c | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index bdcb479c30..5a18ec3e9b 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -73,6 +73,13 @@ static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
+typedef struct {
+    Eterm* hp;
+    Eterm res;
+}hashmap_doer_state;
+typedef void hashmap_doer(Eterm*, hashmap_doer_state*);
+static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr, hashmap_doer_state*);
+
 /* hashmap:new/0 */
 
 BIF_RETTYPE hashmap_new_0(BIF_ALIST_0) {
@@ -1061,9 +1068,8 @@ recurse:
     return res;
 }
 
-typedef void hashmap_doer(Eterm*, void*);
-
-static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr, void* farg) {
+static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr,
+			       hashmap_doer_state* farg) {
     Eterm *ptr, hdr;
     Uint sz;
     DECLARE_ESTACK(stack);
@@ -1106,46 +1112,38 @@ static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr, void* farg) {
     DESTROY_ESTACK(stack);
 }
 
-typedef struct {
-    Eterm* hp;
-    Eterm res;
-}hashmap_keys_state;
-static void hashmap_keys_doer(Eterm* kv, hashmap_keys_state*);
+static void hashmap_keys_doer(Eterm* kv, hashmap_doer_state*);
 
 static Eterm hashmap_keys(Process* p, Eterm node) {
     hashmap_head_t* root;
-    hashmap_keys_state state;
+    hashmap_doer_state state;
 
     root = (hashmap_head_t*) boxed_val(node);
     state.hp  = HAlloc(p, root->size * 2);
     state.res = NIL;
-    hashmap_do_foreach(node, (hashmap_doer*)hashmap_keys_doer, &state);
+    hashmap_do_foreach(node, hashmap_keys_doer, &state);
     return state.res;
 }
 
-static void hashmap_keys_doer(Eterm* kv, hashmap_keys_state* statep) {
+static void hashmap_keys_doer(Eterm* kv, hashmap_doer_state* statep) {
     statep->res = CONS(statep->hp, CAR(kv), statep->res);
     statep->hp += 2;
 }
 
-typedef struct {
-    Eterm* hp;
-    Eterm res;
-}hashmap_values_state;
-static void hashmap_values_doer(Eterm* kv, hashmap_values_state*);
+static void hashmap_values_doer(Eterm* kv, hashmap_doer_state*);
 
 static Eterm hashmap_values(Process* p, Eterm node) {
     hashmap_head_t* root;
-    hashmap_values_state state;
+    hashmap_doer_state state;
 
     root = (hashmap_head_t*) boxed_val(node);
     state.hp  = HAlloc(p, root->size * 2);
     state.res = NIL;
-    hashmap_do_foreach(node, (hashmap_doer*)hashmap_values_doer, &state);
+    hashmap_do_foreach(node, hashmap_values_doer, &state);
     return state.res;
 }
 
-static void hashmap_values_doer(Eterm* kv, hashmap_values_state* statep) {
+static void hashmap_values_doer(Eterm* kv, hashmap_doer_state* statep) {
     statep->res = CONS(statep->hp, CDR(kv), statep->res);
     statep->hp += 2;
 }
-- 
cgit v1.2.3


From b0b6eff0bebd25ba371a027b824426b87bd2a1d6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Jan 2015 18:09:52 +0100
Subject: erts: Refactor hashmap_to_list to use hashmap_do_foreach

---
 erts/emulator/beam/erl_hashmap.c | 61 ++++++++++------------------------------
 1 file changed, 15 insertions(+), 46 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 5a18ec3e9b..8e651dd776 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -748,55 +748,24 @@ not_found:
     return res;
 }
 
+static void hashmap_to_list_doer(Eterm* kv, hashmap_doer_state* sp);
+
 static Eterm hashmap_to_list(Process *p, Eterm node) {
-    Eterm *hp;
-    Eterm res = NIL;
-    Eterm *ptr, tup, hdr;
-    Uint sz, n;
-    DECLARE_ESTACK(stack);
+    hashmap_head_t* root;
+    hashmap_doer_state state;
 
-    ptr = boxed_val(node);
-    n   = (Uint)ptr[1];
-    hp  = HAlloc(p, n * (2 + 3));
-    ESTACK_PUSH(stack, node);
-    do {
-	node = ESTACK_POP(stack);
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST:
-		ptr = list_val(node);
-		tup = TUPLE2(hp, CAR(ptr), CDR(ptr)); hp += 3;
-		res = CONS(hp, tup, res); hp += 2;
-		break;
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			ptr++;
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ptr++;
-			sz = 16;
-			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
-			break;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			ptr++;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-			ASSERT(sz < 17);
-			ptr++;
-			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
-			break;
-		    default:
-			erl_exit(1, "bad header\r\n");
-			break;
-		}
-	}
-    } while(!ESTACK_ISEMPTY(stack));
+    root = (hashmap_head_t*) boxed_val(node);
+    state.hp  = HAlloc(p, root->size * (2 + 3));
+    state.res = NIL;
+    hashmap_do_foreach(node, hashmap_to_list_doer, &state);
+    return state.res;
+}
 
-    DESTROY_ESTACK(stack);
-    ERTS_HOLE_CHECK(p);
-    return res;
+static void hashmap_to_list_doer(Eterm* kv, hashmap_doer_state* sp) {
+    Eterm tup = TUPLE2(sp->hp, CAR(kv), CDR(kv));
+    sp->hp += 3;
+    sp->res = CONS(sp->hp, tup, sp->res);
+    sp->hp += 2;
 }
 
 #define HALLOC_EXTRA 200
-- 
cgit v1.2.3


From a7a0bb29fd162cdfc6cdf6eb18cb5e2cf22a430e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Jan 2015 18:22:58 +0100
Subject: erts: Add hashmap:find/2

---
 erts/emulator/beam/bif.tab       |  1 +
 erts/emulator/beam/erl_hashmap.c | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 6408883e96..c0fb1f8827 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -617,6 +617,7 @@ bif erlang:get_keys/0
 # Hash Array Mappped Trie
 bif hashmap:put/3
 bif hashmap:get/2
+bif hashmap:find/2
 bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:to_list/1
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 8e651dd776..a5d6daadd7 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -133,6 +133,28 @@ BIF_RETTYPE hashmap_get_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
+/* hashmap:find/2 */
+
+BIF_RETTYPE hashmap_find_2(BIF_ALIST_2) {
+    if (is_hashmap(BIF_ARG_2)) {
+	Eterm *hp, res;
+	const Eterm *value;
+	Uint32 hx = make_hash2(BIF_ARG_1);
+
+	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
+	    hp    = HAlloc(BIF_P, 3);
+	    res   = make_tuple(hp);
+	    *hp++ = make_arityval(2);
+	    *hp++ = am_ok;
+            *hp++ = *value;
+	    BIF_RET(res);
+	}
+	BIF_RET(am_error);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+
 /* hashmap:remove/2 */
 
 BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
-- 
cgit v1.2.3


From a97632afab52befd2e48f963f1c5e178e3871995 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Jan 2015 18:48:57 +0100
Subject: erts: Add hashmap:update/3

---
 erts/emulator/beam/bif.tab       |  1 +
 erts/emulator/beam/erl_hashmap.c | 34 +++++++++++++++++++++++++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index c0fb1f8827..27ae8adcec 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -618,6 +618,7 @@ bif erlang:get_keys/0
 bif hashmap:put/3
 bif hashmap:get/2
 bif hashmap:find/2
+bif hashmap:update/3
 bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:to_list/1
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index a5d6daadd7..9445a7db9a 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -64,7 +64,7 @@ static char *format_binary(Uint64 x, char *b) {
 }
 #endif
 
-static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node);
+static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
@@ -102,13 +102,27 @@ BIF_RETTYPE hashmap_put_3(BIF_ALIST_3) {
 	Uint32 hx = make_hash2(BIF_ARG_1);
 	Eterm map;
 
-	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0);
 	ASSERT(is_hashmap(map));
 	BIF_RET(map);
     }
     BIF_ERROR(BIF_P, BADARG);
 }
 
+/* hashmap:update/3 */
+
+BIF_RETTYPE hashmap_update_3(BIF_ALIST_3) {
+    if (is_hashmap(BIF_ARG_3)) {
+	Uint32 hx = make_hash2(BIF_ARG_1);
+	Eterm map;
+
+	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 1);
+	if (is_value(map))
+	    BIF_RET(map);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 /* hashmap:to_list/1 */
 
 BIF_RETTYPE hashmap_to_list_1(BIF_ALIST_1) {
@@ -303,7 +317,8 @@ static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     return NULL;
 }
 
-static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node) {
+static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
+			    Eterm node, int is_update) {
     Eterm *hp = NULL, *nhp = NULL;
     Eterm *ptr;
     Eterm hdr,res,ckey,fake;
@@ -322,6 +337,10 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 		    update_size = 0;
 		    goto unroll;
 		}
+		if (is_update) {
+		    res = THE_NON_VALUE;
+		    goto bail_out;
+		}
 		goto insert_subnodes;
 	    case TAG_PRIMARY_BOXED:
 		ptr = boxed_val(node);
@@ -362,6 +381,10 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 			    break;
 			}
 			/* not occupied */
+			if (is_update) {
+			    res = THE_NON_VALUE;
+			    goto bail_out;
+			}
 			size += HAMT_NODE_BITMAP_SZ(n+1);
 			goto unroll;
 		    case HAMT_SUBTAG_HEAD_BITMAP:
@@ -383,6 +406,10 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 			    break;
 			}
 			/* not occupied */
+			if (is_update) {
+			    res = THE_NON_VALUE;
+			    goto bail_out;
+			}
 			size += HAMT_HEAD_BITMAP_SZ(n+1);
 			goto unroll;
 		    default:
@@ -515,6 +542,7 @@ unroll:
 
     } while(!ESTACK_ISEMPTY(stack));
 
+bail_out:
     DESTROY_ESTACK(stack);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
     ERTS_HOLE_CHECK(p);
-- 
cgit v1.2.3


From aefea82d97f984c615990729a0f7d4303741f233 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 23 Jan 2015 12:36:01 +0100
Subject: erts: Add RESERVE and FAST_PUSH for ESTACK & WSTACK

---
 erts/emulator/beam/global.h | 48 +++++++++++++++++++++++++++++++++++----------
 erts/emulator/beam/utils.c  | 20 +++++++++++++++----
 2 files changed, 54 insertions(+), 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 1fb069232a..dcba8bbba8 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -372,7 +372,7 @@ extern int stackdump_on_exit;
  * DESTROY_ESTACK(Stack)
  */
 
-typedef struct {
+typedef struct ErtsEStack_ {
     Eterm* start;
     Eterm* sp;
     Eterm* end;
@@ -381,7 +381,7 @@ typedef struct {
 
 #define DEF_ESTACK_SIZE (16)
 
-void erl_grow_estack(ErtsEStack*, Eterm* def_stack);
+void erl_grow_estack(ErtsEStack*, Eterm* def_stack, Uint need);
 #define ESTK_CONCAT(a,b) a##b
 #define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack)
 
@@ -457,7 +457,7 @@ do {						\
 #define ESTACK_PUSH(s, x)				\
 do {							\
     if (s.sp == s.end) {				\
-	erl_grow_estack(&s, ESTK_DEF_STACK(s)); 	\
+	erl_grow_estack(&s, ESTK_DEF_STACK(s), 1); 	\
     }							\
     *s.sp++ = (x);					\
 } while(0)
@@ -465,7 +465,7 @@ do {							\
 #define ESTACK_PUSH2(s, x, y)			\
 do {						\
     if (s.sp > s.end - 2) {			\
-	erl_grow_estack(&s, ESTK_DEF_STACK(s)); \
+	erl_grow_estack(&s, ESTK_DEF_STACK(s), 2); \
     }						\
     *s.sp++ = (x);				\
     *s.sp++ = (y);				\
@@ -474,7 +474,7 @@ do {						\
 #define ESTACK_PUSH3(s, x, y, z)		\
 do {						\
     if (s.sp > s.end - 3) {			\
-	erl_grow_estack(&s, ESTK_DEF_STACK(s)); \
+	erl_grow_estack(&s, ESTK_DEF_STACK(s), 3); \
     }						\
     *s.sp++ = (x);				\
     *s.sp++ = (y);				\
@@ -492,6 +492,20 @@ do {						\
     *s.sp++ = (E4);				\
 } while(0)
 
+#define ESTACK_RESERVE(s, push_cnt)                         \
+do {						            \
+    if (s.sp > s.end - (push_cnt)) {	                    \
+	erl_grow_estack(&s, ESTK_DEF_STACK(s), (push_cnt)); \
+    }						            \
+} while(0)
+
+/* Must be preceded by ESTACK_RESERVE */
+#define ESTACK_FAST_PUSH(s, x)				\
+do {							\
+    ASSERT(s.sp < s.end);                               \
+    *s.sp++ = (x);					\
+} while(0)
+
 #define ESTACK_COUNT(s) (s.sp - s.start)
 #define ESTACK_ISEMPTY(s) (s.sp == s.start)
 #define ESTACK_POP(s) (*(--s.sp))
@@ -501,7 +515,7 @@ do {						\
  * WSTACK: same as ESTACK but with UWord instead of Eterm
  */
 
-typedef struct {
+typedef struct ErtsWStack_ {
     UWord* wstart;
     UWord* wsp;
     UWord* wend;
@@ -510,7 +524,7 @@ typedef struct {
 
 #define DEF_WSTACK_SIZE (16)
 
-void erl_grow_wstack(ErtsWStack*, UWord* def_stack);
+void erl_grow_wstack(ErtsWStack*, UWord* def_stack, Uint need);
 #define WSTK_CONCAT(a,b) a##b
 #define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack)
 
@@ -592,7 +606,7 @@ do {						\
 #define WSTACK_PUSH(s, x)				\
 do {							\
     if (s.wsp == s.wend) {				\
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); 	\
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s), 1); 	\
     }							\
     *s.wsp++ = (x);					\
 } while(0)
@@ -600,7 +614,7 @@ do {							\
 #define WSTACK_PUSH2(s, x, y)			\
 do {						\
     if (s.wsp > s.wend - 2) {			\
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s), 2); \
     }						\
     *s.wsp++ = (x);				\
     *s.wsp++ = (y);				\
@@ -609,7 +623,7 @@ do {						\
 #define WSTACK_PUSH3(s, x, y, z)		\
 do {						\
     if (s.wsp > s.wend - 3) {	\
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s), 3); \
     }						\
     *s.wsp++ = (x);				\
     *s.wsp++ = (y);				\
@@ -652,6 +666,20 @@ do {						\
     *s.wsp++ = (A6);				\
 } while(0)
 
+#define WSTACK_RESERVE(s, push_cnt)             \
+do {						\
+    if (s.wsp > s.wend - (push_cnt)) { 	        \
+	erl_grow_wstack(&s, WSTK_DEF_STACK(s), (push_cnt)); \
+    }                                           \
+} while(0)
+
+/* Must be preceded by WSTACK_RESERVE */
+#define WSTACK_FAST_PUSH(s, x)                  \
+do {						\
+    ASSERT(s.wsp < s.wend);                     \
+    *s.wsp++ = (x);                             \
+} while(0)
+
 #define WSTACK_COUNT(s) (s.wsp - s.wstart)
 #define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
 #define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp))
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index cea20a6002..bd90ec2fba 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -190,11 +190,17 @@ erts_set_hole_marker(Eterm* ptr, Uint sz)
  * Helper function for the ESTACK macros defined in global.h.
  */
 void
-erl_grow_estack(ErtsEStack* s, Eterm* default_estack)
+erl_grow_estack(ErtsEStack* s, Eterm* default_estack, Uint need)
 {
     Uint old_size = (s->end - s->start);
-    Uint new_size = old_size * 2;
+    Uint new_size;
     Uint sp_offs = s->sp - s->start;
+
+    if (need < old_size)
+	new_size = 2*old_size;
+    else
+	new_size = ((need / old_size) + 2) * old_size;
+
     if (s->start != default_estack) {
 	s->start = erts_realloc(s->alloc_type, s->start,
 				new_size*sizeof(Eterm));
@@ -210,11 +216,17 @@ erl_grow_estack(ErtsEStack* s, Eterm* default_estack)
  * Helper function for the WSTACK macros defined in global.h.
  */
 void
-erl_grow_wstack(ErtsWStack* s, UWord* default_wstack)
+erl_grow_wstack(ErtsWStack* s, UWord* default_wstack, Uint need)
 {
     Uint old_size = (s->wend - s->wstart);
-    Uint new_size = old_size * 2;
+    Uint new_size;
     Uint sp_offs = s->wsp - s->wstart;
+
+    if (need < old_size)
+	new_size = 2 * old_size;
+    else
+	new_size = ((need / old_size) + 2) * old_size;
+
     if (s->wstart != default_wstack) {
 	s->wstart = erts_realloc(s->alloc_type, s->wstart,
 				 new_size*sizeof(UWord));
-- 
cgit v1.2.3


From e5d514d5adb230bbc832db5c6cf344a69a8df3bf Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 23 Jan 2015 17:50:57 +0100
Subject: erts: Add term_to_binary for hashmap

---
 erts/emulator/beam/external.c | 80 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 79 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index e5fb2d3ec1..af8db4c265 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2304,7 +2304,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 #define ENC_PATCH_FUN_SIZE ((Eterm) 2)
 #define ENC_BIN_COPY ((Eterm) 3)
 #define ENC_MAP_PAIR ((Eterm) 4)
-#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 5)
+#define ENC_HASHMAP_NODE ((Eterm) 5)
+#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6)
 
 static byte*
 enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
@@ -2413,6 +2414,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    WSTACK_PUSH2(s, ENC_TERM, *vptr);
 	    break;
 	}
+	case ENC_HASHMAP_NODE:
+	    if (is_list(obj)) { /* leaf node [K|V] */
+		ptr = list_val(obj);
+		WSTACK_PUSH2(s, ENC_TERM, CDR(ptr));
+		obj = CAR(ptr);
+	    }
+	    break;
 	case ENC_LAST_ARRAY_ELEMENT:
 	    /* obj is the tuple */
 	    {
@@ -2611,6 +2619,44 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    }
 	    break;
 
+	case HASHMAP_DEF:
+	    {
+		Eterm hdr;
+		Uint node_sz;
+		ptr = boxed_val(obj);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		case HAMT_SUBTAG_HEAD_ARRAY:
+		    *ep++ = MAP_EXT;
+		    ptr++;
+		    put_int32(*ptr, ep); ep += 4;
+		    /*fall through*/
+		case HAMT_SUBTAG_NODE_ARRAY:
+		    node_sz = 16;
+		    break;
+		case HAMT_SUBTAG_HEAD_BITMAP:
+		    *ep++ = MAP_EXT;
+		    ptr++;
+		    put_int32(*ptr, ep); ep += 4;
+		    /*fall through*/
+		case HAMT_SUBTAG_NODE_BITMAP:
+		    node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+		    ASSERT(node_sz < 17);
+		    break;
+		default:
+		    erl_exit(1, "bad header\r\n");
+		}
+
+		ptr++;
+		WSTACK_RESERVE(s, node_sz*2);
+		while(node_sz--) {
+		    WSTACK_FAST_PUSH(s, ENC_HASHMAP_NODE);
+		    WSTACK_FAST_PUSH(s, *ptr++);
+		}
+	    }
+	    break;
+
 	case FLOAT_DEF:
 	    GET_DOUBLE(obj, f);
 	    if (dflags & DFLAG_NEW_FLOATS) {
@@ -4047,6 +4093,38 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		goto outer_loop;
 	    }
 	    break;
+
+	case HASHMAP_DEF:
+	    {
+		Eterm *ptr;
+		Eterm hdr;
+		Uint node_sz;
+		ptr = boxed_val(obj);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		case HAMT_SUBTAG_HEAD_ARRAY: ptr++;
+		case HAMT_SUBTAG_NODE_ARRAY:
+		    node_sz = 16;
+		    break;
+		case HAMT_SUBTAG_HEAD_BITMAP: ptr++;
+		case HAMT_SUBTAG_NODE_BITMAP:
+		    node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+		    ASSERT(node_sz < 17);
+		    break;
+		default:
+		    erl_exit(1, "bad header\r\n");
+		}
+
+		ptr++;
+		ESTACK_RESERVE(s, node_sz);
+		while(node_sz--) {
+		    ESTACK_FAST_PUSH(s, *ptr++);
+		}
+		result += 1 + 4; /* tag + 4 bytes size */
+	    }
+
+	    break;
 	case FLOAT_DEF:
 	    if (dflags & DFLAG_NEW_FLOATS) {
 		result += 9;
-- 
cgit v1.2.3


From b38518ff23477322da221212f29d5ad0d1fb96fa Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 23 Jan 2015 20:02:27 +0100
Subject: erts: Cleanup hashmap_merge_2

---
 erts/emulator/beam/erl_hashmap.c | 212 ++++++++++++++-------------------------
 1 file changed, 78 insertions(+), 134 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 9445a7db9a..1697449234 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -820,34 +820,23 @@ static void hashmap_to_list_doer(Eterm* kv, hashmap_doer_state* sp) {
 
 #define HALLOC_EXTRA 200
 
-#if defined(VALGRIND) || defined(GO_SUPER_FAST)
-# define UNDEF(V,I)
-#else
-# define UNDEF(V,I) V=I
-#endif
-
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     struct {
-	Eterm *ptrA, *ptrB;
-	Uint32 abm, bbm, rbm;
-	Uint ix;
+	Eterm *srcA, *srcB, *dst;
+	Uint32 abm, bbm, rbm;        /* node bitmaps */
 	int keepA;
 	Eterm array[16];
-    }stack[16];
+    }stack[16], *sp = stack;
     Eterm *hp, *nhp;
-    Eterm *ptrA, *ptrB;
     Eterm hdrA, hdrB;
     Eterm th[2];
-    Uint32 ahx, bhx, abm, bbm, rbm, bit;
-    Uint ix;
-    Uint size;
+    Uint32 ahx, bhx;
+    Uint size;  /* total key-value counter */
     int keepA = 0;
     unsigned lvl = 0;
+    unsigned node_sz;
     Eterm res = THE_NON_VALUE;
 
-    UNDEF(abm,0);
-    UNDEF(bbm,0);
-
     /*
      * Strategy: Do depth-first traversal of both trees (at the same time)
      * and merge each pair of nodes.
@@ -872,50 +861,46 @@ recurse:
 
     switch (primary_tag(nodeA)) {
     case TAG_PRIMARY_LIST: {
-	ptrA = list_val(nodeA);
+	sp->srcA = list_val(nodeA);
 	switch (primary_tag(nodeB)) {
 	case TAG_PRIMARY_LIST: { /* LEAF + LEAF */
-	    ptrB = list_val(nodeB);
+	    sp->srcB = list_val(nodeB);
 
-	    if (EQ(CAR(ptrA), CAR(ptrB))) {
+	    if (EQ(CAR(sp->srcA), CAR(sp->srcB))) {
 		--size;
 		res = keepA ? nodeA : nodeB;
 	    } else {
-		ahx = hashmap_restore_hash(th, lvl, CAR(ptrA));
-		bhx = hashmap_restore_hash(th, lvl, CAR(ptrB));
-		abm = 1 << hashmap_index(ahx);
-		bbm = 1 << hashmap_index(bhx);
+		ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
+		bhx = hashmap_restore_hash(th, lvl, CAR(sp->srcB));
+		sp->abm = 1 << hashmap_index(ahx);
+		sp->bbm = 1 << hashmap_index(bhx);
 
-		ptrA = &nodeA;
-		ptrB = &nodeB;
+		sp->srcA = &nodeA;
+		sp->srcB = &nodeB;
 	    }
 	    break;
 	}
 	case TAG_PRIMARY_BOXED: { /* LEAF + NODE */
-	    ptrB = boxed_val(nodeB);
-	    ASSERT(is_header(*ptrB));
-	    hdrB = *ptrB++;
-
-	    ahx = hashmap_restore_hash(th, lvl, CAR(ptrA));
-	    abm = 1 << hashmap_index(ahx);
+	    sp->srcB = boxed_val(nodeB);
+	    ASSERT(is_header(*sp->srcB));
+	    hdrB = *sp->srcB++;
 
+	    ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
+	    sp->abm = 1 << hashmap_index(ahx);
+	    sp->srcA = &nodeA;
 	    switch(hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ptrB++;
+	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
 	    case HAMT_SUBTAG_NODE_ARRAY:
-		bbm = 0xffff;
-		ptrA = &nodeA;
+		sp->bbm = 0xffff;
 		break;
 
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		ptrB++;
+	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
 	    case HAMT_SUBTAG_NODE_BITMAP:
-		bbm = MAP_HEADER_VAL(hdrB);
-		ptrA = &nodeA;
+		sp->bbm = MAP_HEADER_VAL(hdrB);
 		break;
 
 	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *ptrB & _HEADER_MAP_SUBTAG_MASK);
+		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
 		break;
 	    }
 	    break;
@@ -926,83 +911,50 @@ recurse:
 	break;
     }
     case TAG_PRIMARY_BOXED: { /* NODE + NODE */
-	ptrA = boxed_val(nodeA);
-	hdrA = *ptrA;
+	sp->srcA = boxed_val(nodeA);
+	hdrA = *sp->srcA++;
 	ASSERT(is_header(hdrA));
-	ptrA++;
 	switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
-	case HAMT_SUBTAG_HEAD_ARRAY:
-	    ptrA++;
+	case HAMT_SUBTAG_HEAD_ARRAY: sp->srcA++;
 	case HAMT_SUBTAG_NODE_ARRAY: {
 	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
-	    ptrB = boxed_val(nodeB);
-	    hdrB = *ptrB;
+	    sp->abm = 0xffff;
+	    sp->srcB = boxed_val(nodeB);
+	    hdrB = *sp->srcB++;
 	    ASSERT(is_header(hdrB));
 	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ASSERT(lvl == 0);
-		abm = 0xffff;
-		bbm = 0xffff;
-		ptrB += 2;
-		break;
+	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
 	    case HAMT_SUBTAG_NODE_ARRAY:
-		ASSERT(lvl > 0);
-		abm = 0xffff;
-		bbm = 0xffff;
-		ptrB++;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		ASSERT(lvl == 0);
-		abm = 0xffff;
-		bbm = MAP_HEADER_VAL(hdrB);
-		ptrB += 2;
+		sp->bbm = 0xffff;
 		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
 	    case HAMT_SUBTAG_NODE_BITMAP:
-		ASSERT(lvl > 0);
-		abm = 0xffff;
-		bbm = MAP_HEADER_VAL(hdrB);
-		ptrB += 1;
+		sp->bbm = MAP_HEADER_VAL(hdrB);
 		break;
 	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *ptrB & _HEADER_MAP_SUBTAG_MASK);
+		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
 	    }
 	    break;
 	}
-	case HAMT_SUBTAG_HEAD_BITMAP:
-	    ptrA++;
+	case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++;
 	case HAMT_SUBTAG_NODE_BITMAP: {
 	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
-	    abm = MAP_HEADER_VAL(hdrA);
-	    ptrB = boxed_val(nodeB);
-	    hdrB = *ptrB;
+	    sp->abm = MAP_HEADER_VAL(hdrA);
+	    sp->srcB = boxed_val(nodeB);
+	    hdrB = *sp->srcB++;
 	    ASSERT(is_header(hdrB));
 	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ASSERT(lvl == 0);
-		bbm = 0xffff;
-		ptrB += 2;
-		break;
-
+	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
 	    case HAMT_SUBTAG_NODE_ARRAY:
-		ASSERT(lvl > 0);
-		bbm = 0xffff;
-		ptrB++;
+		sp->bbm = 0xffff;
 		break;
-
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		ASSERT(lvl == 0);
-		bbm = MAP_HEADER_VAL(hdrB);
-		ptrB += 2;
-		break;
-
+	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
 	    case HAMT_SUBTAG_NODE_BITMAP:
-		ASSERT(lvl > 0);
-		bbm = MAP_HEADER_VAL(hdrB);
-		ptrB += 1;
+		sp->bbm = MAP_HEADER_VAL(hdrB);
 		break;
 
 	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *ptrB & _HEADER_MAP_SUBTAG_MASK);
+		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
 	    }
 	    break;
 	}
@@ -1022,69 +974,61 @@ recurse:
 
 	    /* Pop from stack and continue build parent node */
 	    lvl--;
-	    abm = stack[lvl].abm;
-	    bbm = stack[lvl].bbm;
-	    rbm = stack[lvl].rbm;
-	    ix  = stack[lvl].ix;
-	    stack[lvl].array[ix++] = res;
+	    sp--;
+	    *sp->dst++ = res;
 	    res = THE_NON_VALUE;
-	    if (rbm) {
-		ptrA = stack[lvl].ptrA;
-		ptrB = stack[lvl].ptrB;
-		keepA = stack[lvl].keepA;
+	    if (sp->rbm) {
+		sp->srcA++;
+		sp->srcB++;
+		keepA = sp->keepA;
 	    }
 	} else { /* Start build a node */
-	    ix = 0;
-	    rbm = abm | bbm;
-	    ASSERT(!(rbm == 0 && lvl > 0));
+	    sp->dst = sp->array;
+	    sp->rbm = sp->abm | sp->bbm;
+	    ASSERT(!(sp->rbm == 0 && lvl > 0));
 	}
 
-	while (rbm) {
-	    Uint32 next = rbm & (rbm-1);
-	    bit = rbm ^ next;
-	    rbm = next;
-	    if (abm & bit) {
-		if (bbm & bit) {
+	while (sp->rbm) {
+	    Uint32 next = sp->rbm & (sp->rbm-1);
+	    Uint32 bit = sp->rbm ^ next;
+	    sp->rbm = next;
+	    if (sp->abm & bit) {
+		if (sp->bbm & bit) {
 		    /* Bit clash. Push and resolve by recursive merge */
-		    if (rbm) {
-			stack[lvl].ptrA = ptrA + 1;
-			stack[lvl].ptrB = ptrB + 1;
-			stack[lvl].keepA = keepA;
+		    if (sp->rbm) {
+			sp->keepA = keepA;
 		    }
-		    stack[lvl].abm = abm;
-		    stack[lvl].bbm = bbm;
-		    stack[lvl].rbm = rbm;
-		    stack[lvl].ix  = ix;
-		    nodeA = *ptrA;
-		    nodeB = *ptrB;
+		    nodeA = *sp->srcA;
+		    nodeB = *sp->srcB;
 		    lvl++;
+		    sp++;
 		    goto recurse;
 		} else {
-		    stack[lvl].array[ix++] = *ptrA++;
+		    *sp->dst++ = *sp->srcA++;
 		}
 	    } else {
-		ASSERT(bbm & bit);
-		stack[lvl].array[ix++] = *ptrB++;
+		ASSERT(sp->bbm & bit);
+		*sp->dst++ = *sp->srcB++;
 	    }
 	}
 
+	node_sz = sp->dst - sp->array;
+	ASSERT(node_sz == hashmap_bitcount(sp->abm | sp->bbm));
 	if (lvl == 0) {
-	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(ix), HALLOC_EXTRA);
+	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(node_sz), HALLOC_EXTRA);
 	    hp = nhp;
-	    *hp++ = (ix == 16) ? MAP_HEADER_HAMT_HEAD_ARRAY
-		: MAP_HEADER_HAMT_HEAD_BITMAP(abm | bbm);
+	    *hp++ = (node_sz == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
+		     : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm));
 	    *hp++ = size;
 	} else {
-	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(ix), HALLOC_EXTRA);
+	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(node_sz), HALLOC_EXTRA);
 	    hp = nhp;
-	    *hp++ = (ix == 16) ? make_arityval(16)
-				   : MAP_HEADER_HAMT_NODE_BITMAP(abm | bbm);
+	    *hp++ = (node_sz == 16 ? make_arityval(16)
+		     : MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm));
 	}
-	memcpy(hp, stack[lvl].array, ix * sizeof(Eterm));
+	memcpy(hp, sp->array, node_sz * sizeof(Eterm));
 	res = make_boxed(nhp);
     }
-
-    return res;
 }
 
 static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr,
-- 
cgit v1.2.3


From b530d8b95055c27c737140c5ec2047919d02aeed Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 26 Jan 2015 17:58:30 +0100
Subject: erts: Add PSTACK

A lightweight stack that can be used to store any type.
PUSH and POP return pointers into stack slots.
---
 erts/emulator/beam/global.h | 61 +++++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/utils.c  | 26 +++++++++++++++++++
 2 files changed, 87 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index dcba8bbba8..09f43348f4 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -685,6 +685,67 @@ do {						\
 #define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp))
 
 
+
+/* PSTACK - Stack of any type.
+ * Usage:
+ * {
+ * #define PSTACK_TYPE MyType
+ *    PSTACK_DECLARE(s,16);
+ *    MyType *sp = PSTACK_PUSH(s);
+ *
+ *    sp->x = ....
+ *    sp->y = ....
+ *    sp = PSTACK_PUSH(s);
+ *    ...
+ *    sp = PSTACK_POP(s);
+ *    if (PSTACK_IS_EMPTY(s)) {
+ *        // sp is invalid when stack is empty after pop
+ *    }
+ *
+ *    PSTACK_DESTROY(s);
+ * }
+ */
+
+
+typedef struct ErtsPStack_ {
+    byte* pstart;
+    byte* psp;
+    byte* pend;
+    ErtsAlcType_t alloc_type;
+}ErtsPStack;
+
+void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes);
+#define PSTK_CONCAT(a,b) a##b
+#define PSTK_DEF_STACK(s) PSTK_CONCAT(s,_default_pstack)
+
+#define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \
+PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE];                            \
+ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */                    \
+                 (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */                 \
+                 (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\
+                 ERTS_ALC_T_ESTACK   /* alloc_type */                      \
+}
+
+#define PSTACK_DESTROY(s)				\
+do {							\
+    if (s.pstart != (byte*)PSTK_DEF_STACK(s)) {		\
+	erts_free(s.alloc_type, s.pstart); 		\
+    }							\
+} while(0)
+
+#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart)
+
+#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp))
+
+#define PSTACK_PUSH(s) 		                                           \
+    (s.psp += sizeof(PSTACK_TYPE),                                         \
+     ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s),           \
+                                          sizeof(PSTACK_TYPE)) : (void)0), \
+     ((PSTACK_TYPE*) s.psp))
+
+#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE)))
+
+
 /* binary.c */
 
 void erts_emasculate_writable_binary(ProcBin* pb);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index bd90ec2fba..a54a93b086 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -239,6 +239,32 @@ erl_grow_wstack(ErtsWStack* s, UWord* default_wstack, Uint need)
     s->wsp = s->wstart + sp_offs;
 }
 
+/*
+ * Helper function for the PSTACK macros defined in global.h.
+ */
+void
+erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes)
+{
+    Uint old_size = s->pend - s->pstart;
+    Uint new_size;
+    Uint sp_offs = s->psp - s->pstart;
+
+    if (need_bytes < old_size)
+	new_size = 2 * old_size;
+    else
+	new_size = ((need_bytes / old_size) + 2) * old_size;
+
+    if (s->pstart != default_pstack) {
+	s->pstart = erts_realloc(s->alloc_type, s->pstart, new_size);
+    } else {
+	byte* new_ptr = erts_alloc(s->alloc_type, new_size);
+	sys_memcpy(new_ptr, s->pstart, old_size);
+	s->pstart = new_ptr;
+    }
+    s->pend = s->pstart + new_size;
+    s->psp = s->pstart + sp_offs;
+}
+
 /* CTYPE macros */
 
 #define LATIN1
-- 
cgit v1.2.3


From fc21440eec0283da271b36181ed24f25dedda0fe Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Feb 2015 23:41:59 +0100
Subject: erts: Make hashmap_merge use dynamic PSTACK

---
 erts/emulator/beam/erl_hashmap.c | 40 ++++++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 1697449234..4ca0877385 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -821,12 +821,16 @@ static void hashmap_to_list_doer(Eterm* kv, hashmap_doer_state* sp) {
 #define HALLOC_EXTRA 200
 
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
-    struct {
-	Eterm *srcA, *srcB, *dst;
+#define PSTACK_TYPE struct HashmapMergePStackType
+    struct HashmapMergePStackType {
+	Eterm *srcA, *srcB;
 	Uint32 abm, bbm, rbm;        /* node bitmaps */
 	int keepA;
+	int ix;
 	Eterm array[16];
-    }stack[16], *sp = stack;
+    };
+    PSTACK_DECLARE(s, 4);
+    struct HashmapMergePStackType* sp = PSTACK_PUSH(s);
     Eterm *hp, *nhp;
     Eterm hdrA, hdrB;
     Eterm th[2];
@@ -834,7 +838,6 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     Uint size;  /* total key-value counter */
     int keepA = 0;
     unsigned lvl = 0;
-    unsigned node_sz;
     Eterm res = THE_NON_VALUE;
 
     /*
@@ -970,12 +973,12 @@ recurse:
     for (;;) {
 	if (is_value(res)) { /* We have a complete (sub-)tree or leaf */
 	    if (lvl == 0)
-		return res;
+		break;
 
 	    /* Pop from stack and continue build parent node */
 	    lvl--;
-	    sp--;
-	    *sp->dst++ = res;
+	    sp = PSTACK_POP(s);
+	    sp->array[sp->ix++] = res;
 	    res = THE_NON_VALUE;
 	    if (sp->rbm) {
 		sp->srcA++;
@@ -983,7 +986,7 @@ recurse:
 		keepA = sp->keepA;
 	    }
 	} else { /* Start build a node */
-	    sp->dst = sp->array;
+	    sp->ix = 0;
 	    sp->rbm = sp->abm | sp->bbm;
 	    ASSERT(!(sp->rbm == 0 && lvl > 0));
 	}
@@ -1001,34 +1004,35 @@ recurse:
 		    nodeA = *sp->srcA;
 		    nodeB = *sp->srcB;
 		    lvl++;
-		    sp++;
+		    sp = PSTACK_PUSH(s);
 		    goto recurse;
 		} else {
-		    *sp->dst++ = *sp->srcA++;
+		    sp->array[sp->ix++] = *sp->srcA++;
 		}
 	    } else {
 		ASSERT(sp->bbm & bit);
-		*sp->dst++ = *sp->srcB++;
+		sp->array[sp->ix++] = *sp->srcB++;
 	    }
 	}
 
-	node_sz = sp->dst - sp->array;
-	ASSERT(node_sz == hashmap_bitcount(sp->abm | sp->bbm));
+	ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm));
 	if (lvl == 0) {
-	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(node_sz), HALLOC_EXTRA);
+	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
 	    hp = nhp;
-	    *hp++ = (node_sz == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
+	    *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
 		     : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm));
 	    *hp++ = size;
 	} else {
-	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(node_sz), HALLOC_EXTRA);
+	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
 	    hp = nhp;
-	    *hp++ = (node_sz == 16 ? make_arityval(16)
+	    *hp++ = (sp->ix == 16 ? make_arityval(16)
 		     : MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm));
 	}
-	memcpy(hp, sp->array, node_sz * sizeof(Eterm));
+	memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
 	res = make_boxed(nhp);
     }
+    PSTACK_DESTROY(s);
+    return res;
 }
 
 static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr,
-- 
cgit v1.2.3


From 85c8e9c956ac7f2fa15abe56a513a2d97839af23 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Feb 2015 21:00:59 +0100
Subject: erts: Make WSTACK usable through pointer

---
 erts/emulator/beam/global.h | 22 ++++++++++++----------
 erts/emulator/beam/utils.c  |  4 ++--
 2 files changed, 14 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 09f43348f4..7585705949 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -519,12 +519,13 @@ typedef struct ErtsWStack_ {
     UWord* wstart;
     UWord* wsp;
     UWord* wend;
+    UWord* wdefault;
     ErtsAlcType_t alloc_type;
 }ErtsWStack;
 
 #define DEF_WSTACK_SIZE (16)
 
-void erl_grow_wstack(ErtsWStack*, UWord* def_stack, Uint need);
+void erl_grow_wstack(ErtsWStack*, Uint need);
 #define WSTK_CONCAT(a,b) a##b
 #define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack)
 
@@ -534,6 +535,7 @@ void erl_grow_wstack(ErtsWStack*, UWord* def_stack, Uint need);
         WSTK_DEF_STACK(s),  /* wstart */ 		\
         WSTK_DEF_STACK(s),  /* wsp */			\
         WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */	\
+        WSTK_DEF_STACK(s),  /* wdflt */ 		\
         ERTS_ALC_T_ESTACK /* alloc_type */		\
     }
 
@@ -606,7 +608,7 @@ do {						\
 #define WSTACK_PUSH(s, x)				\
 do {							\
     if (s.wsp == s.wend) {				\
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s), 1); 	\
+	erl_grow_wstack(&s, 1); 	                \
     }							\
     *s.wsp++ = (x);					\
 } while(0)
@@ -614,7 +616,7 @@ do {							\
 #define WSTACK_PUSH2(s, x, y)			\
 do {						\
     if (s.wsp > s.wend - 2) {			\
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s), 2); \
+	erl_grow_wstack(&s, 2);                 \
     }						\
     *s.wsp++ = (x);				\
     *s.wsp++ = (y);				\
@@ -622,8 +624,8 @@ do {						\
 
 #define WSTACK_PUSH3(s, x, y, z)		\
 do {						\
-    if (s.wsp > s.wend - 3) {	\
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s), 3); \
+    if (s.wsp > s.wend - 3) {	                \
+	erl_grow_wstack(&s, 3);                 \
     }						\
     *s.wsp++ = (x);				\
     *s.wsp++ = (y);				\
@@ -632,8 +634,8 @@ do {						\
 
 #define WSTACK_PUSH4(s, A1, A2, A3, A4)		\
 do {						\
-    if (s.wsp > s.wend - 4) {	\
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+    if (s.wsp > s.wend - 4) {	                \
+	erl_grow_wstack(&s, 4);                 \
     }						\
     *s.wsp++ = (A1);				\
     *s.wsp++ = (A2);				\
@@ -644,7 +646,7 @@ do {						\
 #define WSTACK_PUSH5(s, A1, A2, A3, A4, A5)     \
 do {						\
     if (s.wsp > s.wend - 5) {	                \
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+	erl_grow_wstack(&s, 5);                 \
     }						\
     *s.wsp++ = (A1);				\
     *s.wsp++ = (A2);				\
@@ -656,7 +658,7 @@ do {						\
 #define WSTACK_PUSH6(s, A1, A2, A3, A4, A5, A6) \
 do {						\
     if (s.wsp > s.wend - 6) {	                \
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \
+	erl_grow_wstack(&s, 6);                 \
     }						\
     *s.wsp++ = (A1);				\
     *s.wsp++ = (A2);				\
@@ -669,7 +671,7 @@ do {						\
 #define WSTACK_RESERVE(s, push_cnt)             \
 do {						\
     if (s.wsp > s.wend - (push_cnt)) { 	        \
-	erl_grow_wstack(&s, WSTK_DEF_STACK(s), (push_cnt)); \
+	erl_grow_wstack(&s, (push_cnt));        \
     }                                           \
 } while(0)
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index a54a93b086..471bce8940 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -216,7 +216,7 @@ erl_grow_estack(ErtsEStack* s, Eterm* default_estack, Uint need)
  * Helper function for the WSTACK macros defined in global.h.
  */
 void
-erl_grow_wstack(ErtsWStack* s, UWord* default_wstack, Uint need)
+erl_grow_wstack(ErtsWStack* s, Uint need)
 {
     Uint old_size = (s->wend - s->wstart);
     Uint new_size;
@@ -227,7 +227,7 @@ erl_grow_wstack(ErtsWStack* s, UWord* default_wstack, Uint need)
     else
 	new_size = ((need / old_size) + 2) * old_size;
 
-    if (s->wstart != default_wstack) {
+    if (s->wstart != s->wdefault) {
 	s->wstart = erts_realloc(s->alloc_type, s->wstart,
 				 new_size*sizeof(UWord));
     } else {
-- 
cgit v1.2.3


From 8db065e5f6c07904db0c63175c906334e2eb5c59 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 27 Jan 2015 16:32:37 +0100
Subject: erts: Rewrite hashmap:to_list, keys and values

to use an iterator instead of foreach with callback.

Will be easier to make yielding.
---
 erts/emulator/beam/erl_hashmap.c | 175 +++++++++++++++++++--------------------
 1 file changed, 86 insertions(+), 89 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 4ca0877385..2e3a4b8982 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -73,12 +73,6 @@ static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
-typedef struct {
-    Eterm* hp;
-    Eterm res;
-}hashmap_doer_state;
-typedef void hashmap_doer(Eterm*, hashmap_doer_state*);
-static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr, hashmap_doer_state*);
 
 /* hashmap:new/0 */
 
@@ -798,26 +792,6 @@ not_found:
     return res;
 }
 
-static void hashmap_to_list_doer(Eterm* kv, hashmap_doer_state* sp);
-
-static Eterm hashmap_to_list(Process *p, Eterm node) {
-    hashmap_head_t* root;
-    hashmap_doer_state state;
-
-    root = (hashmap_head_t*) boxed_val(node);
-    state.hp  = HAlloc(p, root->size * (2 + 3));
-    state.res = NIL;
-    hashmap_do_foreach(node, hashmap_to_list_doer, &state);
-    return state.res;
-}
-
-static void hashmap_to_list_doer(Eterm* kv, hashmap_doer_state* sp) {
-    Eterm tup = TUPLE2(sp->hp, CAR(kv), CDR(kv));
-    sp->hp += 3;
-    sp->res = CONS(sp->hp, tup, sp->res);
-    sp->hp += 2;
-}
-
 #define HALLOC_EXTRA 200
 
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
@@ -1035,84 +1009,107 @@ recurse:
     return res;
 }
 
-static void hashmap_do_foreach(Eterm node, hashmap_doer* fptr,
-			       hashmap_doer_state* farg) {
-    Eterm *ptr, hdr;
-    Uint sz;
-    DECLARE_ESTACK(stack);
+static void hashmap_iterator_init(ErtsWStack* s, Eterm node) {
+    WSTACK_PUSH((*s), (UWord)THE_NON_VALUE);  /* end marker */
+    WSTACK_PUSH((*s), (UWord)node);
+}
 
-    ESTACK_PUSH(stack, node);
-    do {
-	node = ESTACK_POP(stack);
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST:
-		ptr = list_val(node);
-		(*fptr)(ptr, farg); /* Do! */
-		break;
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			ptr++;
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ptr++;
-			sz = 16;
-			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
-			break;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			ptr++;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-			ASSERT(sz < 17);
-			ptr++;
-			while(sz--) { ESTACK_PUSH(stack, ptr[sz]); }
-			break;
-		    default:
-			erl_exit(1, "bad header\r\n");
-			break;
-		}
-	}
-    } while(!ESTACK_ISEMPTY(stack));
+static Eterm* hashmap_iterator_next(ErtsWStack* s) {
+    Eterm node, *ptr, hdr;
+    Uint32 sz;
 
-    DESTROY_ESTACK(stack);
+    for (;;) {
+        ASSERT(!WSTACK_ISEMPTY((*s)));
+	node = (Eterm) WSTACK_POP((*s));
+        if (is_non_value(node)) {
+            return NULL;
+        }
+        switch (primary_tag(node)) {
+        case TAG_PRIMARY_LIST:
+            return list_val(node);
+
+        case TAG_PRIMARY_BOXED:
+            ptr = boxed_val(node);
+            hdr = *ptr;
+            ASSERT(is_header(hdr));
+            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_HEAD_ARRAY:
+                ptr++;
+            case HAMT_SUBTAG_NODE_ARRAY:
+                ptr++;
+                sz = 16;
+                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
+                break;
+            case HAMT_SUBTAG_HEAD_BITMAP:
+                ptr++;
+            case HAMT_SUBTAG_NODE_BITMAP:
+                sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                ASSERT(sz < 17);
+                ptr++;
+                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
+                break;
+            default:
+                erl_exit(1, "bad header");
+            }
+            break;
+
+        default:
+            erl_exit(1, "bad hamt node");
+	}
+    }
 }
 
-static void hashmap_keys_doer(Eterm* kv, hashmap_doer_state*);
-
-static Eterm hashmap_keys(Process* p, Eterm node) {
+static Eterm hashmap_to_list(Process *p, Eterm node) {
+    DECLARE_WSTACK(stack);
     hashmap_head_t* root;
-    hashmap_doer_state state;
+    Eterm *hp, *kv;
+    Eterm res = NIL;
 
     root = (hashmap_head_t*) boxed_val(node);
-    state.hp  = HAlloc(p, root->size * 2);
-    state.res = NIL;
-    hashmap_do_foreach(node, hashmap_keys_doer, &state);
-    return state.res;
+    hp  = HAlloc(p, root->size * (2 + 3));
+    hashmap_iterator_init(&stack, node);
+    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
+	Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv));
+	hp += 3;
+	res = CONS(hp, tup, res);
+	hp += 2;
+    }
+    DESTROY_WSTACK(stack);
+    return res;
 }
 
-static void hashmap_keys_doer(Eterm* kv, hashmap_doer_state* statep) {
-    statep->res = CONS(statep->hp, CAR(kv), statep->res);
-    statep->hp += 2;
-}
+static Eterm hashmap_keys(Process* p, Eterm node) {
+    DECLARE_WSTACK(stack);
+    hashmap_head_t* root;
+    Eterm *hp, *kv;
+    Eterm res = NIL;
 
-static void hashmap_values_doer(Eterm* kv, hashmap_doer_state*);
+    root = (hashmap_head_t*) boxed_val(node);
+    hp  = HAlloc(p, root->size * 2);
+    hashmap_iterator_init(&stack, node);
+    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
+	res = CONS(hp, CAR(kv), res);
+	hp += 2;
+    }
+    DESTROY_WSTACK(stack);
+    return res;
+}
 
 static Eterm hashmap_values(Process* p, Eterm node) {
+    DECLARE_WSTACK(stack);
     hashmap_head_t* root;
-    hashmap_doer_state state;
+    Eterm *hp, *kv;
+    Eterm res = NIL;
 
     root = (hashmap_head_t*) boxed_val(node);
-    state.hp  = HAlloc(p, root->size * 2);
-    state.res = NIL;
-    hashmap_do_foreach(node, hashmap_values_doer, &state);
-    return state.res;
-}
-
-static void hashmap_values_doer(Eterm* kv, hashmap_doer_state* statep) {
-    statep->res = CONS(statep->hp, CDR(kv), statep->res);
-    statep->hp += 2;
+    hp  = HAlloc(p, root->size * 2);
+    hashmap_iterator_init(&stack, node);
+    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
+	res = CONS(hp, CDR(kv), res);
+	hp += 2;
+    }
+    DESTROY_WSTACK(stack);
+    return res;
 }
 
 /* hashmap:info/0 */
-- 
cgit v1.2.3


From aad7d6fe4e1e9fe086d275ab3ea34c5285cc8edb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Feb 2015 21:58:45 +0100
Subject: erts: Change to total ordering of keys in small maps

---
 erts/emulator/beam/utils.c | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 471bce8940..908116c64c 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2686,8 +2686,22 @@ tailrecur_ne:
 		}
 		aa += 2;
 		bb += 2;
-		i  += 1; /* increment for tuple-keys */
-		goto term_array;
+		if (exact) {
+		    i  += 1; /* increment for tuple-keys */
+		    goto term_array;
+		}
+		else {
+		    /* Value array */
+		    WSTACK_PUSH3(stack, i, (UWord)(bb+1), (UWord)(aa+1) | TAG_PRIMARY_HEADER);
+		    /* Marker to switch back from 'exact' compare */
+		    WSTACK_PUSH(stack, (UWord)NULL | TAG_PRIMARY_HEADER);
+		    /* Now do 'exact' compare of key tuples */
+		    a = *aa;
+		    b = *bb;
+		    exact = 1;
+		    goto tailrecur;
+		}
+
 	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
 		if (!is_float_rel(b,b_base)) {
 		    a_tag = FLOAT_DEF;
@@ -3061,7 +3075,13 @@ pop_next:
     if (!WSTACK_ISEMPTY(stack)) {
 	UWord something = WSTACK_POP(stack);
 	if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */
-	    aa = (Eterm*) something;
+	    if (something == ((UWord)NULL | TAG_PRIMARY_HEADER)) {
+		/* Done with exact compare of map keys, switch back */
+		ASSERT(exact);
+		exact = 0;
+		goto pop_next;
+	    }
+	    aa = (Eterm *)something;
 	    bb = (Eterm*) WSTACK_POP(stack);
 	    i = WSTACK_POP(stack);
 	    goto term_array;
-- 
cgit v1.2.3


From 442c9b4d11a62c55b46ffb25f27b5ec5fb3adda7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Feb 2015 21:50:25 +0100
Subject: erts: First recursive version of hashmap compare

---
 erts/emulator/beam/erl_hashmap.c | 129 +++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_hashmap.h |   2 +
 erts/emulator/beam/utils.c       |  13 ++++
 3 files changed, 144 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 2e3a4b8982..2fd743a2d8 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -1112,6 +1112,135 @@ static Eterm hashmap_values(Process* p, Eterm node) {
     return res;
 }
 
+static int hash_cmp(Uint32 ha, Uint32 hb)
+{
+    int i;
+    for (i=0; i<8; i++) {
+	int cmp = (int)(ha & 0xF) - (int)(hb & 0xF);
+	if (cmp)
+	    return cmp;
+	ha >>= 4;
+	hb >>= 4;
+    }
+    return 0;
+}
+
+static int key_hash_cmp(Eterm* ap, Eterm* bp)
+{
+    Eterm th[2];
+    unsigned lvl = 0;
+
+    if (ap && bp) {
+	ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0);
+	for (;;) {
+	    Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap));
+	    Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp));
+	    int cmp = hash_cmp(ha, hb);
+	    if (cmp)
+		return cmp;
+	    lvl += 8;
+	}
+    }
+    return ap ? -1 : 1;
+}
+
+int hashmap_cmp(Eterm a, Eterm b)
+{
+    DECLARE_WSTACK(astack);
+    DECLARE_WSTACK(bstack);
+    Eterm *ap;
+    Eterm *bp;
+    Eterm min_key;
+    int cmp_res = 0;
+
+    /* Strategy:
+       Phase 1.
+          While keys are identical
+	  Do synchronous stepping through leafs of both trees in hash order.
+          Maintain min value of min key.
+
+       Phase 2:  (If key diff was found in phase 1)
+          Ignore values from now on.
+          Continue iterate trees by always advancing the one lagging behind hash-wise.
+          Identical keys are skipped
+          A minimal key can only be candidate as tie-breaker if we have passed
+          that hash value in the other tree (which means the key did not exist
+          in the other tree).
+     */
+
+    hashmap_iterator_init(&astack, a);
+    hashmap_iterator_init(&bstack, b);
+
+    for (;;) {   /* Phase 1 */
+	int cmp;
+	ap = hashmap_iterator_next(&astack);
+	bp = hashmap_iterator_next(&bstack);
+	if (!ap) {
+	    ASSERT(!bp);
+	    return cmp_res;
+	}
+	cmp = CMP_TERM(CAR(ap), CAR(bp));
+	if (cmp)
+	    break;
+
+	/* No key diff found so far, compare values */
+	if (!cmp_res || (CMP_TERM(CAR(ap), min_key) < 0)) {
+	    cmp = CMP(CDR(ap), CDR(bp));
+	    if (cmp) {
+		cmp_res = cmp;
+		min_key = CAR(ap);
+	    }
+	}
+    }
+
+    /* Phase 2 */
+
+    if (key_hash_cmp(ap,bp) < 0) {
+	min_key = CAR(ap);
+	cmp_res = -1;
+	ap = hashmap_iterator_next(&astack);
+    }
+    else {
+	min_key = CAR(bp);
+	cmp_res = 1;
+	bp = hashmap_iterator_next(&bstack);
+    }
+
+    for (;;) {
+	int hash_cmp;
+
+	while (ap && bp && CMP_TERM(CAR(ap), CAR(bp)) == 0) {
+	    ap = hashmap_iterator_next(&astack);
+	    bp = hashmap_iterator_next(&bstack);
+	}
+	if (!ap && !bp)
+	    break;
+
+	hash_cmp = key_hash_cmp(ap,bp);
+	if (hash_cmp < 0) {
+	    ASSERT(ap);
+	    if (CMP_TERM(CAR(ap), min_key) < 0) {
+		min_key = CAR(ap);
+                cmp_res = -1;
+	    }
+	    ap = hashmap_iterator_next(&astack);
+	}
+	else if (hash_cmp > 0) {
+	    ASSERT(bp);
+	    if (CMP_TERM(CAR(bp), min_key) < 0) {
+		min_key = CAR(bp);
+		cmp_res = 1;
+	    }
+	    bp = hashmap_iterator_next(&bstack);
+	}
+    }
+
+    DESTROY_WSTACK(astack);
+    DESTROY_WSTACK(bstack);
+
+    return cmp_res;
+}
+
 /* hashmap:info/0 */
 
 static Eterm hashmap_info(Process *p, Eterm node) {
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index 4ef795b11e..4a4163bce9 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -24,11 +24,13 @@
 #include "sys.h"
 
 Eterm erts_hashmap_get(Eterm key, Eterm map);
+int hashmap_cmp(Eterm a, Eterm b);
 
 /* erl_term.h stuff */
 #define make_hashmap(x)		make_boxed((Eterm*)(x))
 #define make_hashmap_rel 	make_boxed_rel
 #define is_hashmap(x)		(is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
+#define is_hashmap_rel(RTERM,BASE)  is_hashmap(rterm2wterm(RTERM,BASE))
 #define is_hashmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HASHMAP)
 #define hashmap_val(x)		_unchecked_boxed_val((x))
 #define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE))
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 908116c64c..9fd53e941d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2702,6 +2702,19 @@ tailrecur_ne:
 		    goto tailrecur;
 		}
 
+	    case (_TAG_HEADER_HASHMAP >> _TAG_PRIMARY_SIZE) :
+		{
+		    if (!is_hashmap_rel(b,b_base)) {
+			a_tag = HASHMAP_DEF;
+			goto mixed_types;
+		    }
+		    i = (((hashmap_head_t*) hashmap_val_rel(a,a_base))->size -
+			 ((hashmap_head_t*) hashmap_val_rel(b,b_base))->size);
+		    if (i) {
+			RETURN_NEQ(i);
+		    }
+		    ON_CMP_GOTO(hashmap_cmp(a, b));
+		}
 	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
 		if (!is_float_rel(b,b_base)) {
 		    a_tag = FLOAT_DEF;
-- 
cgit v1.2.3


From 6cc099bf98f384de1de3c0d8542c83db43fb5cec Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Feb 2015 23:32:12 +0100
Subject: erts: Make hashmap compare non-recursive

---
 erts/emulator/beam/erl_hashmap.c | 103 +---------------
 erts/emulator/beam/erl_hashmap.h |   8 +-
 erts/emulator/beam/global.h      |   9 +-
 erts/emulator/beam/utils.c       | 253 ++++++++++++++++++++++++++++++++++++---
 4 files changed, 250 insertions(+), 123 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 2fd743a2d8..b423111715 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -1009,12 +1009,12 @@ recurse:
     return res;
 }
 
-static void hashmap_iterator_init(ErtsWStack* s, Eterm node) {
+void hashmap_iterator_init(ErtsWStack* s, Eterm node) {
     WSTACK_PUSH((*s), (UWord)THE_NON_VALUE);  /* end marker */
     WSTACK_PUSH((*s), (UWord)node);
 }
 
-static Eterm* hashmap_iterator_next(ErtsWStack* s) {
+Eterm* hashmap_iterator_next(ErtsWStack* s) {
     Eterm node, *ptr, hdr;
     Uint32 sz;
 
@@ -1125,7 +1125,7 @@ static int hash_cmp(Uint32 ha, Uint32 hb)
     return 0;
 }
 
-static int key_hash_cmp(Eterm* ap, Eterm* bp)
+int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
 {
     Eterm th[2];
     unsigned lvl = 0;
@@ -1144,103 +1144,6 @@ static int key_hash_cmp(Eterm* ap, Eterm* bp)
     return ap ? -1 : 1;
 }
 
-int hashmap_cmp(Eterm a, Eterm b)
-{
-    DECLARE_WSTACK(astack);
-    DECLARE_WSTACK(bstack);
-    Eterm *ap;
-    Eterm *bp;
-    Eterm min_key;
-    int cmp_res = 0;
-
-    /* Strategy:
-       Phase 1.
-          While keys are identical
-	  Do synchronous stepping through leafs of both trees in hash order.
-          Maintain min value of min key.
-
-       Phase 2:  (If key diff was found in phase 1)
-          Ignore values from now on.
-          Continue iterate trees by always advancing the one lagging behind hash-wise.
-          Identical keys are skipped
-          A minimal key can only be candidate as tie-breaker if we have passed
-          that hash value in the other tree (which means the key did not exist
-          in the other tree).
-     */
-
-    hashmap_iterator_init(&astack, a);
-    hashmap_iterator_init(&bstack, b);
-
-    for (;;) {   /* Phase 1 */
-	int cmp;
-	ap = hashmap_iterator_next(&astack);
-	bp = hashmap_iterator_next(&bstack);
-	if (!ap) {
-	    ASSERT(!bp);
-	    return cmp_res;
-	}
-	cmp = CMP_TERM(CAR(ap), CAR(bp));
-	if (cmp)
-	    break;
-
-	/* No key diff found so far, compare values */
-	if (!cmp_res || (CMP_TERM(CAR(ap), min_key) < 0)) {
-	    cmp = CMP(CDR(ap), CDR(bp));
-	    if (cmp) {
-		cmp_res = cmp;
-		min_key = CAR(ap);
-	    }
-	}
-    }
-
-    /* Phase 2 */
-
-    if (key_hash_cmp(ap,bp) < 0) {
-	min_key = CAR(ap);
-	cmp_res = -1;
-	ap = hashmap_iterator_next(&astack);
-    }
-    else {
-	min_key = CAR(bp);
-	cmp_res = 1;
-	bp = hashmap_iterator_next(&bstack);
-    }
-
-    for (;;) {
-	int hash_cmp;
-
-	while (ap && bp && CMP_TERM(CAR(ap), CAR(bp)) == 0) {
-	    ap = hashmap_iterator_next(&astack);
-	    bp = hashmap_iterator_next(&bstack);
-	}
-	if (!ap && !bp)
-	    break;
-
-	hash_cmp = key_hash_cmp(ap,bp);
-	if (hash_cmp < 0) {
-	    ASSERT(ap);
-	    if (CMP_TERM(CAR(ap), min_key) < 0) {
-		min_key = CAR(ap);
-                cmp_res = -1;
-	    }
-	    ap = hashmap_iterator_next(&astack);
-	}
-	else if (hash_cmp > 0) {
-	    ASSERT(bp);
-	    if (CMP_TERM(CAR(bp), min_key) < 0) {
-		min_key = CAR(bp);
-		cmp_res = 1;
-	    }
-	    bp = hashmap_iterator_next(&bstack);
-	}
-    }
-
-    DESTROY_WSTACK(astack);
-    DESTROY_WSTACK(bstack);
-
-    return cmp_res;
-}
-
 /* hashmap:info/0 */
 
 static Eterm hashmap_info(Process *p, Eterm node) {
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index 4a4163bce9..8ba249b053 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -24,7 +24,10 @@
 #include "sys.h"
 
 Eterm erts_hashmap_get(Eterm key, Eterm map);
-int hashmap_cmp(Eterm a, Eterm b);
+struct ErtsWStack_;
+void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
+Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
+int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 
 /* erl_term.h stuff */
 #define make_hashmap(x)		make_boxed((Eterm*)(x))
@@ -67,6 +70,9 @@ typedef struct hashmap_head_s {
     Uint size;
     Eterm items[1];
 } hashmap_head_t;
+
+#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
+#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
  
 /* the bitmap-node
  * typedef struct hashmap_bitmap_node_s {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 7585705949..ad7e6fb89d 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -529,7 +529,7 @@ void erl_grow_wstack(ErtsWStack*, Uint need);
 #define WSTK_CONCAT(a,b) a##b
 #define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack)
 
-#define DECLARE_WSTACK(s)				\
+#define WSTACK_DECLARE(s)				\
     UWord WSTK_DEF_STACK(s)[DEF_WSTACK_SIZE];		\
     ErtsWStack s = {					\
         WSTK_DEF_STACK(s),  /* wstart */ 		\
@@ -538,6 +538,7 @@ void erl_grow_wstack(ErtsWStack*, Uint need);
         WSTK_DEF_STACK(s),  /* wdflt */ 		\
         ERTS_ALC_T_ESTACK /* alloc_type */		\
     }
+#define DECLARE_WSTACK WSTACK_DECLARE
 
 #define WSTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
@@ -548,12 +549,13 @@ do {									\
     s.alloc_type = (t);							\
  } while (0)
 
-#define DESTROY_WSTACK(s)				\
+#define WSTACK_DESTROY(s)				\
 do {							\
     if (s.wstart != WSTK_DEF_STACK(s)) {		\
 	erts_free(s.alloc_type, s.wstart); 		\
     }							\
 } while(0)
+#define DESTROY_WSTACK WSTACK_DESTROY
 
 #define WSTACK_DEBUG(s) \
     do { \
@@ -686,7 +688,8 @@ do {						\
 #define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
 #define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp))
 
-
+#define WSTACK_ROLLBACK(s, count) (ASSERT(WSTACK_COUNT(s) >= (count)), \
+                                   s.wsp = s.wstart + (count))
 
 /* PSTACK - Stack of any type.
  * Usage:
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 9fd53e941d..f79cb8db7d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2512,7 +2512,18 @@ Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact)
 Sint erts_cmp(Eterm a, Eterm b, int exact)
 #endif
 {
-    DECLARE_WSTACK(stack);
+#define PSTACK_TYPE struct erts_cmp_hashmap_state
+    struct erts_cmp_hashmap_state {
+        Sint wstack_rollback;
+        int was_exact;
+        Eterm *ap;
+        Eterm *bp;
+        Eterm min_key;
+        Sint cmp_res;   /* result so far -1,0,+1 */
+    };
+    PSTACK_DECLARE(hmap_stack, 1);
+    WSTACK_DECLARE(stack);
+    WSTACK_DECLARE(b_stack); /* only used by hashmaps */
     Eterm* aa;
     Eterm* bb;
     int i;
@@ -2528,6 +2539,26 @@ Sint erts_cmp(Eterm a, Eterm b, int exact)
     Uint32 *anum;
     Uint32 *bnum;
 
+/* The WSTACK contains naked Eterms and Operations marked with header-tags */
+#define OP_BITS 4
+#define OP_MASK 0xF
+#define TERM_ARRAY_OP                 0
+#define SWITCH_EXACT_OFF_OP           1
+#define HASHMAP_PHASE1_ARE_KEYS_EQUAL 2
+#define HASHMAP_PHASE1_IS_MIN_KEY     3
+#define HASHMAP_PHASE1_CMP_VALUES     4
+#define HASHMAP_PHASE2_ARE_KEYS_EQUAL 5
+#define HASHMAP_PHASE2_IS_MIN_KEY_A   6
+#define HASHMAP_PHASE2_IS_MIN_KEY_B   7
+
+
+#define OP_WORD(OP)  (((OP)  << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER)
+#define TERM_ARRAY_OP_WORD(SZ) OP_WORD(((SZ) << OP_BITS) | TERM_ARRAY_OP)
+
+#define GET_OP(WORD) (ASSERT(is_header(WORD)), ((WORD) >> _TAG_PRIMARY_SIZE) & OP_MASK)
+#define GET_OP_ARG(WORD) (ASSERT(is_header(WORD)), ((WORD) >> (OP_BITS + _TAG_PRIMARY_SIZE)))
+
+
 #define RETURN_NEQ(cmp) { j=(cmp); ASSERT(j != 0); goto not_equal; }
 #define ON_CMP_GOTO(cmp) if ((j=(cmp)) == 0) goto pop_next; else goto not_equal
 
@@ -2543,6 +2574,8 @@ Sint erts_cmp(Eterm a, Eterm b, int exact)
     } while (0)
 
 
+bodyrecur:
+    j = 0;
 tailrecur:
     if (is_same(a,a_base,b,b_base)) {	/* Equal values or pointers. */
 	goto pop_next;
@@ -2692,28 +2725,60 @@ tailrecur_ne:
 		}
 		else {
 		    /* Value array */
-		    WSTACK_PUSH3(stack, i, (UWord)(bb+1), (UWord)(aa+1) | TAG_PRIMARY_HEADER);
-		    /* Marker to switch back from 'exact' compare */
-		    WSTACK_PUSH(stack, (UWord)NULL | TAG_PRIMARY_HEADER);
+		    WSTACK_PUSH3(stack, (UWord)(bb+1), (UWord)(aa+1), TERM_ARRAY_OP_WORD(i));
+		    /* Switch back from 'exact' key compare */
+		    WSTACK_PUSH(stack, OP_WORD(SWITCH_EXACT_OFF_OP));
 		    /* Now do 'exact' compare of key tuples */
 		    a = *aa;
 		    b = *bb;
 		    exact = 1;
-		    goto tailrecur;
+		    goto bodyrecur;
 		}
 
 	    case (_TAG_HEADER_HASHMAP >> _TAG_PRIMARY_SIZE) :
 		{
+                    struct erts_cmp_hashmap_state* sp;
 		    if (!is_hashmap_rel(b,b_base)) {
 			a_tag = HASHMAP_DEF;
 			goto mixed_types;
 		    }
-		    i = (((hashmap_head_t*) hashmap_val_rel(a,a_base))->size -
-			 ((hashmap_head_t*) hashmap_val_rel(b,b_base))->size);
+		    i = hashmap_size_rel(a,a_base) - hashmap_size_rel(b,b_base);
 		    if (i) {
 			RETURN_NEQ(i);
 		    }
-		    ON_CMP_GOTO(hashmap_cmp(a, b));
+                    if (hashmap_size_rel(a,a_base) == 0) {
+                        goto pop_next;
+                    }
+
+                /* Hashmap compare strategy:
+                   Phase 1. While keys are identical
+                     Do synchronous stepping through leafs of both trees in hash
+                     order. Maintain value compare result of minimal key.
+
+                   Phase 2. If key diff was found in phase 1
+                     Ignore values from now on.
+                     Continue iterate trees by always advancing the one
+                     lagging behind hash-wise. Identical keys are skipped.
+                     A minimal key can only be candidate as tie-breaker if we
+                     have passed that hash value in the other tree (which means
+                     the key did not exist in the other tree).
+                */
+
+                    sp = PSTACK_PUSH(hmap_stack);
+                    hashmap_iterator_init(&stack, a);
+                    hashmap_iterator_init(&b_stack, b);
+                    sp->ap = hashmap_iterator_next(&stack);
+                    sp->bp = hashmap_iterator_next(&b_stack);
+                    sp->cmp_res = 0;
+                    ASSERT(sp->ap && sp->bp);
+
+                    a = CAR(sp->ap);
+                    b = CAR(sp->bp);
+                    sp->was_exact = exact;
+                    exact = 1;
+                    WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL));
+                    sp->wstack_rollback = WSTACK_COUNT(stack);
+                    goto bodyrecur;
 		}
 	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
 		if (!is_float_rel(b,b_base)) {
@@ -3074,8 +3139,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
 		    goto not_equal;
 		}
 	    } else {
-		/* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */
-		WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER);
+		WSTACK_PUSH3(stack, (UWord)bb, (UWord)aa, TERM_ARRAY_OP_WORD(i));
 		goto tailrecur_ne;
 	    }
 	}
@@ -3087,28 +3151,179 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
 pop_next:
     if (!WSTACK_ISEMPTY(stack)) {
 	UWord something = WSTACK_POP(stack);
-	if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */
-	    if (something == ((UWord)NULL | TAG_PRIMARY_HEADER)) {
+        struct erts_cmp_hashmap_state* sp;
+	if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* an operation */
+	    switch (GET_OP(something)) {
+	    case TERM_ARRAY_OP:
+		i = GET_OP_ARG(something);
+		aa = (Eterm*)WSTACK_POP(stack);
+		bb = (Eterm*) WSTACK_POP(stack);
+		goto term_array;
+
+	    case SWITCH_EXACT_OFF_OP:
 		/* Done with exact compare of map keys, switch back */
 		ASSERT(exact);
 		exact = 0;
 		goto pop_next;
-	    }
-	    aa = (Eterm *)something;
-	    bb = (Eterm*) WSTACK_POP(stack);
-	    i = WSTACK_POP(stack);
-	    goto term_array;
+
+            case HASHMAP_PHASE1_ARE_KEYS_EQUAL: {
+                sp = PSTACK_TOP(hmap_stack);
+                if (j) {
+                    /* Key diff found, enter phase 2 */
+                    if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) {
+                        sp->min_key = CAR(sp->ap);
+                        sp->cmp_res = -1;
+                        sp->ap = hashmap_iterator_next(&stack);
+                    }
+                    else {
+                        sp->min_key = CAR(sp->bp);
+                        sp->cmp_res = 1;
+                        sp->bp = hashmap_iterator_next(&b_stack);
+                    }
+                    exact = 1; /* only exact key compares in phase 2 */
+                    goto case_HASHMAP_PHASE2_LOOP;
+                }
+
+                /* No key diff found so far, compare values if min key */
+
+                if (sp->cmp_res) {
+                    a = CAR(sp->ap);
+                    b = sp->min_key;
+                    exact = 1;
+                    WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_IS_MIN_KEY));
+                    sp->wstack_rollback = WSTACK_COUNT(stack);
+                    goto bodyrecur;
+                }
+                /* no min key-value found yet */
+                a = CDR(sp->ap);
+                b = CDR(sp->bp);
+                exact = sp->was_exact;
+                WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES));
+                sp->wstack_rollback = WSTACK_COUNT(stack);
+                goto bodyrecur;
+            }
+            case HASHMAP_PHASE1_IS_MIN_KEY:
+                sp = PSTACK_TOP(hmap_stack);
+                if (j < 0) {
+                    a = CDR(sp->ap);
+                    b = CDR(sp->bp);
+                    exact = sp->was_exact;
+                    WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES));
+                    sp->wstack_rollback = WSTACK_COUNT(stack);
+                    goto bodyrecur;
+                }
+                goto case_HASHMAP_PHASE1_LOOP;
+
+            case HASHMAP_PHASE1_CMP_VALUES:
+                sp = PSTACK_TOP(hmap_stack);
+                if (j) {
+                    sp->cmp_res = j;
+                    sp->min_key = CAR(sp->ap);
+                }
+            case_HASHMAP_PHASE1_LOOP:
+                sp->ap = hashmap_iterator_next(&stack);
+                sp->bp = hashmap_iterator_next(&b_stack);
+                if (!sp->ap) {
+                    /* end of maps with identical keys */
+                    ASSERT(!sp->bp);
+                    j = sp->cmp_res;
+                    exact = sp->was_exact;
+                    (void) PSTACK_POP(hmap_stack);
+                    ON_CMP_GOTO(j);
+                }
+                a = CAR(sp->ap);
+                b = CAR(sp->bp);
+                exact = 1;
+                WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL));
+                sp->wstack_rollback = WSTACK_COUNT(stack);
+                goto bodyrecur;
+
+            case_HASHMAP_PHASE2_LOOP:
+                if (sp->ap && sp->bp) {
+                    a = CAR(sp->ap);
+                    b = CAR(sp->bp);
+                    ASSERT(exact);
+                    WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_ARE_KEYS_EQUAL));
+                    sp->wstack_rollback = WSTACK_COUNT(stack);
+                    goto bodyrecur;
+                }
+                goto case_HASHMAP_PHASE2_NEXT_STEP;
+
+            case HASHMAP_PHASE2_ARE_KEYS_EQUAL:
+                sp = PSTACK_TOP(hmap_stack);
+                if (j == 0) {
+                    /* keys are equal, skip them */
+                    sp->ap = hashmap_iterator_next(&stack);
+                    sp->bp = hashmap_iterator_next(&b_stack);
+                    goto case_HASHMAP_PHASE2_LOOP;
+                }
+                /* fall through */
+            case_HASHMAP_PHASE2_NEXT_STEP:
+                if (sp->ap || sp->bp) {
+                    if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) {
+                        ASSERT(sp->ap);
+                        a = CAR(sp->ap);
+                        b = sp->min_key;
+                        ASSERT(exact);
+                        WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_A));
+                    }
+                    else { /* hash_cmp > 0 */
+                        ASSERT(sp->bp);
+                        a = CAR(sp->bp);
+                        b = sp->min_key;
+                        ASSERT(exact);
+                        WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_B));
+                    }
+                    sp->wstack_rollback = WSTACK_COUNT(stack);
+                    goto bodyrecur;
+                }
+                /* End of both maps */
+                j = sp->cmp_res;
+                exact = sp->was_exact;
+                (void) PSTACK_POP(hmap_stack);
+                ON_CMP_GOTO(j);
+
+            case HASHMAP_PHASE2_IS_MIN_KEY_A:
+                sp = PSTACK_TOP(hmap_stack);
+                if (j < 0) {
+                    sp->min_key = CAR(sp->ap);
+                    sp->cmp_res = -1;
+                }
+                sp->ap = hashmap_iterator_next(&stack);
+                goto case_HASHMAP_PHASE2_LOOP;
+
+            case HASHMAP_PHASE2_IS_MIN_KEY_B:
+                sp = PSTACK_TOP(hmap_stack);
+                if (j < 0) {
+                    sp->min_key = CAR(sp->bp);
+                    sp->cmp_res = 1;
+                }
+                sp->bp = hashmap_iterator_next(&b_stack);
+                goto case_HASHMAP_PHASE2_LOOP;
+
+            default:
+                ASSERT(!"Invalid cmp op");
+            } /* switch */
 	}
 	a = (Eterm) something;
 	b = (Eterm) WSTACK_POP(stack);
 	goto tailrecur;
     }
 
-    DESTROY_WSTACK(stack);
+    ASSERT(PSTACK_IS_EMPTY(hmap_stack));
+    PSTACK_DESTROY(hmap_stack);
+    WSTACK_DESTROY(stack);
+    WSTACK_DESTROY(b_stack);
     return 0;
 
 not_equal:
-    DESTROY_WSTACK(stack);
+    if (!PSTACK_IS_EMPTY(hmap_stack)) {
+        WSTACK_ROLLBACK(stack, PSTACK_TOP(hmap_stack)->wstack_rollback);
+        goto pop_next;
+    }
+    PSTACK_DESTROY(hmap_stack);
+    WSTACK_DESTROY(stack);
+    WSTACK_DESTROY(b_stack);
     return j;
 
 #undef CMP_NODES
-- 
cgit v1.2.3


From 04237c0948fdaba9af1a50c7fd9c5512afd10bb2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Feb 2015 19:45:45 +0100
Subject: erts: Make hashmap use the new hash function

---
 erts/emulator/beam/erl_hashmap.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index b423111715..583ff52c26 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -48,10 +48,12 @@
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 #endif
 
+#define hashmap_make_hash(Key) make_hash_vsn(Key, 3)
+
 #define hashmap_restore_hash(Heap,Lvl,Key) \
-    ((Lvl) < 8) ? make_hash2(Key) >> (4*(Lvl)) : make_hash2(CONS(Heap, make_small(Lvl), (Key))) >> (4*((Lvl) & 7))
+    ((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small(Lvl), (Key))) >> (4*((Lvl) & 7))
 #define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
-    ((++(Lvl)) & 7) ? (Hx) >> 4 : make_hash2(CONS(Heap, make_small(Lvl), Key))
+    ((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small(Lvl), Key))
 
 #if 0
 static char *format_binary(Uint64 x, char *b) {
@@ -93,7 +95,7 @@ BIF_RETTYPE hashmap_new_0(BIF_ALIST_0) {
 
 BIF_RETTYPE hashmap_put_3(BIF_ALIST_3) {
     if (is_hashmap(BIF_ARG_3)) {
-	Uint32 hx = make_hash2(BIF_ARG_1);
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
 	Eterm map;
 
 	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0);
@@ -107,7 +109,7 @@ BIF_RETTYPE hashmap_put_3(BIF_ALIST_3) {
 
 BIF_RETTYPE hashmap_update_3(BIF_ALIST_3) {
     if (is_hashmap(BIF_ARG_3)) {
-	Uint32 hx = make_hash2(BIF_ARG_1);
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
 	Eterm map;
 
 	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 1);
@@ -132,7 +134,7 @@ BIF_RETTYPE hashmap_to_list_1(BIF_ALIST_1) {
 BIF_RETTYPE hashmap_get_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_2)) {
 	const Eterm *value;
-	Uint32 hx = make_hash2(BIF_ARG_1);
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
 
 	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
 	    BIF_RET(*value);
@@ -147,7 +149,7 @@ BIF_RETTYPE hashmap_find_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_2)) {
 	Eterm *hp, res;
 	const Eterm *value;
-	Uint32 hx = make_hash2(BIF_ARG_1);
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
 
 	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
 	    hp    = HAlloc(BIF_P, 3);
@@ -167,7 +169,7 @@ BIF_RETTYPE hashmap_find_2(BIF_ALIST_2) {
 
 BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_2)) {
-	Uint32 hx = make_hash2(BIF_ARG_1);
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
 
 	BIF_RET(hashmap_delete(BIF_P, hx, BIF_ARG_1, BIF_ARG_2));
     }
@@ -204,7 +206,7 @@ BIF_RETTYPE is_hashmap_1(BIF_ALIST_1) {
 
 BIF_RETTYPE hashmap_is_key_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_1)) {
-	Uint32 hx = make_hash2(BIF_ARG_1);
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
 
 	BIF_RET(hashmap_get(hx, BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
     }
-- 
cgit v1.2.3


From 354ec7fe8ee31cde73c811bd5dea4f3e6787d10a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 17 Feb 2015 20:10:46 +0100
Subject: erts: Add hashing of hashmaps

---
 erts/emulator/beam/utils.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f79cb8db7d..0f2d89bbfe 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1288,6 +1288,55 @@ make_hash2(Eterm term)
 		goto hash2_common;
 	    }
 	    break;
+            case HASHMAP_SUBTAG:
+            {
+                Eterm* ptr = boxed_val(term) + 1;
+                Uint size;
+                int i;
+                switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+                case HAMT_SUBTAG_HEAD_ARRAY:
+                case HAMT_SUBTAG_HEAD_BITMAP:
+                    size = *ptr++;
+                    UINT32_HASH(size, HCONST_16);
+                    if (size == 0)
+                        goto hash2_common;
+                    ESTACK_PUSH(s, hash_xor_values);
+                    ESTACK_PUSH(s, hash_xor_keys);
+                    ESTACK_PUSH(s, hash);
+                    ESTACK_PUSH(s, HASH_MAP_TAIL);
+                    hash = 0;
+                    hash_xor_keys = 0;
+                    hash_xor_values = 0;
+                }
+                switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+                case HAMT_SUBTAG_HEAD_ARRAY:
+                case HAMT_SUBTAG_NODE_ARRAY:
+                    i = 16;
+                    break;
+                case HAMT_SUBTAG_HEAD_BITMAP:
+                case HAMT_SUBTAG_NODE_BITMAP:
+                    i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                    break;
+                default:
+                    erl_exit(1, "bad header");
+                }
+                while (i) {
+                    if (is_list(*ptr)) {
+                        Eterm* cons = list_val(*ptr);
+                        ESTACK_PUSH(s, HASH_MAP_KEY);
+                        ESTACK_PUSH(s, CAR(cons));
+                        ESTACK_PUSH(s, HASH_MAP_VAL);
+                        ESTACK_PUSH(s, CDR(cons));
+                    }
+                    else {
+                        ASSERT(is_boxed(*ptr));
+                        ESTACK_PUSH(s, *ptr);
+                    }
+                    i--; ptr++;
+                }
+                goto hash2_common;
+            }
+            break;
 	    case EXPORT_SUBTAG:
 	    {
 		Export* ep = *((Export **) (export_val(term) + 1));
-- 
cgit v1.2.3


From 7e3c3c7106095cc5afdbd5cd7e96b907784ada01 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 17 Feb 2015 19:11:24 +0100
Subject: erts: Fix bug in hashmap_restore/shift_hash

Deep hashing should ignore the three lowest bits of the level.
---
 erts/emulator/beam/erl_hashmap.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 583ff52c26..6bca2f1665 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -51,9 +51,9 @@
 #define hashmap_make_hash(Key) make_hash_vsn(Key, 3)
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
-    ((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small(Lvl), (Key))) >> (4*((Lvl) & 7))
+    (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7)))
 #define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
-    ((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small(Lvl), Key))
+    (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key)))
 
 #if 0
 static char *format_binary(Uint64 x, char *b) {
-- 
cgit v1.2.3


From af34cc859ae9979e9f4f8abf29646efd6b1c9c04 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 17 Feb 2015 19:21:06 +0100
Subject: erts: Add ERTS_UNDEF macro

---
 erts/emulator/beam/sys.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 828f5b427a..d2a8b9e7f4 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -188,6 +188,16 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 #  define ASSERT(e) ((void) 1)
 #endif
 
+/* ERTS_UNDEF can be used to silence false warnings about
+ * "variable may be used uninitialized" while keeping the variable
+ * marked as undefined by valgrind.
+ */
+#ifdef VALGRIND
+#  define ERTS_UNDEF(V,I)
+#else
+#  define ERTS_UNDEF(V,I) V = I
+#endif
+
 /*
  * Compile time assert
  * (the actual compiler error msg can be a bit confusing)
-- 
cgit v1.2.3


From 7e3b9d5ab2e0c53b606a653896f3a2857ea5cbce Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 17 Feb 2015 19:27:45 +0100
Subject: erts: Add micro optimization to phash2 of tuples

---
 erts/emulator/beam/utils.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 0f2d89bbfe..3f9cb5dbea 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1252,11 +1252,12 @@ make_hash2(Eterm term)
 		UINT32_HASH(arity, HCONST_9);
 		if (arity == 0) /* Empty tuple */ 
 		    goto hash2_common;
-		for (i = arity; i >= 1; i--) {
-		    tmp = elem[i];
-		    ESTACK_PUSH(s, tmp);
+		for (i = arity; ; i--) {
+		    term = elem[i];
+                    if (i == 1)
+                        break;
+                    ESTACK_PUSH(s, term);
 		}
-		goto hash2_common;
 	    }
 	    break;
 	    case MAP_SUBTAG:
-- 
cgit v1.2.3


From 70bb13c626ffbffc9c7d6fbe1d69e91dd0a853be Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 17 Feb 2015 19:35:59 +0100
Subject: erts: Change phash2 of maps to be sensitive to key-value combos.

The old hashing did not care which value belonged to which key,
for example:

would hash the same.
---
 erts/emulator/beam/utils.c | 67 +++++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 3f9cb5dbea..d234e8fc68 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1133,10 +1133,11 @@ Uint32
 make_hash2(Eterm term)
 {
     Uint32 hash;
-    Uint32 hash_xor_keys   = 0;
-    Uint32 hash_xor_values = 0;
+    Uint32 hash_xor_pairs;
     DeclareTmpHeapNoproc(tmp_big,2);
 
+    ERTS_UNDEF(hash_xor_pairs, 0);
+
 /* (HCONST * {2, ..., 16}) mod 2^32 */
 #define HCONST_2 0x3c6ef372UL
 #define HCONST_3 0xdaa66d2bUL
@@ -1155,8 +1156,8 @@ make_hash2(Eterm term)
 #define HCONST_16 0xe3779b90UL
 
 #define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF))
-#define HASH_MAP_KEY  (_make_header(2,_TAG_HEADER_REF))
-#define HASH_MAP_VAL  (_make_header(3,_TAG_HEADER_REF))
+#define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF))
+#define HASH_CDR      (_make_header(3,_TAG_HEADER_REF))
 
 #define UINT32_HASH_2(Expr1, Expr2, AConst)       \
          do {                                     \
@@ -1233,9 +1234,9 @@ make_hash2(Eterm term)
 	    if (c > 0)
 		UINT32_HASH(sh, HCONST_4);
 	    if (is_list(term)) {
-		term = *ptr;
-		tmp = *++ptr;
-		ESTACK_PUSH(s, tmp);	    
+		tmp = CDR(ptr);
+                ESTACK_PUSH(s, tmp);
+		term = CAR(ptr);
 	    }
 	}
 	break;
@@ -1271,20 +1272,20 @@ make_hash2(Eterm term)
 		if (size == 0) {
 		    goto hash2_common;
 		}
-		ESTACK_PUSH4(s, hash_xor_values, hash_xor_keys, hash, HASH_MAP_TAIL);
-		hash = 0;
-		hash_xor_keys = 0;
-		hash_xor_values = 0;
-		for (i = size - 1; i >= 0; i--) {
-		    tmp = vs[i];
-		    ESTACK_PUSH2(s, HASH_MAP_VAL, tmp);
-		}
-		/* We do not want to expose the tuple representation.
-		 * Do not push the keys as a tuple.
+		/* We want a portable hash function that is *independent* of
+		 * the order in which keys and values are encountered.
+		 * We therefore calculate context independent hashes for all    				      .
+		 * key-value pairs and then xor them together.
 		 */
+		ESTACK_PUSH(s, hash_xor_pairs);
+		ESTACK_PUSH(s, hash);
+		ESTACK_PUSH(s, HASH_MAP_TAIL);
+		hash = 0;
+		hash_xor_pairs = 0;
 		for (i = size - 1; i >= 0; i--) {
-		    tmp = ks[i];
-		    ESTACK_PUSH2(s, HASH_MAP_KEY, tmp);
+		    ESTACK_PUSH(s, HASH_MAP_PAIR);
+		    ESTACK_PUSH(s, vs[i]);
+                    ESTACK_PUSH(s, ks[i]);
 		}
 		goto hash2_common;
 	    }
@@ -1301,13 +1302,11 @@ make_hash2(Eterm term)
                     UINT32_HASH(size, HCONST_16);
                     if (size == 0)
                         goto hash2_common;
-                    ESTACK_PUSH(s, hash_xor_values);
-                    ESTACK_PUSH(s, hash_xor_keys);
+                    ESTACK_PUSH(s, hash_xor_pairs);
                     ESTACK_PUSH(s, hash);
                     ESTACK_PUSH(s, HASH_MAP_TAIL);
                     hash = 0;
-                    hash_xor_keys = 0;
-                    hash_xor_values = 0;
+                    hash_xor_pairs = 0;
                 }
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
                 case HAMT_SUBTAG_HEAD_ARRAY:
@@ -1324,10 +1323,9 @@ make_hash2(Eterm term)
                 while (i) {
                     if (is_list(*ptr)) {
                         Eterm* cons = list_val(*ptr);
-                        ESTACK_PUSH(s, HASH_MAP_KEY);
-                        ESTACK_PUSH(s, CAR(cons));
-                        ESTACK_PUSH(s, HASH_MAP_VAL);
+                        ESTACK_PUSH(s, HASH_MAP_PAIR);
                         ESTACK_PUSH(s, CDR(cons));
+                        ESTACK_PUSH(s, CAR(cons));
                     }
                     else {
                         ASSERT(is_boxed(*ptr));
@@ -1437,7 +1435,8 @@ make_hash2(Eterm term)
 		do {
 		    Uint t;
 		    Uint32 x, y;
-		    t = i < n ? BIG_DIGIT(ptr, i++) : 0;
+                    ASSERT(i < n);
+		    t = BIG_DIGIT(ptr, i++);
 		    x = t & 0xffffffff;
 		    y = t >> 32;
 		    UINT32_HASH_2(x, y, con);
@@ -1545,18 +1544,12 @@ make_hash2(Eterm term)
 	    switch (term) {
 		case HASH_MAP_TAIL: {
 		    hash = (Uint32) ESTACK_POP(s);
-		    UINT32_HASH(hash_xor_keys, HCONST_16);
-		    UINT32_HASH(hash_xor_values, HCONST_16);
-		    hash_xor_keys = (Uint32) ESTACK_POP(s);
-		    hash_xor_values = (Uint32) ESTACK_POP(s);
+                    UINT32_HASH(hash_xor_pairs, HCONST_19);
+		    hash_xor_pairs = (Uint32) ESTACK_POP(s);
 		    goto hash2_common;
 		}
-		case HASH_MAP_KEY:
-		    hash_xor_keys ^= hash;
-		    hash = 0;
-		    goto hash2_common;
-		case HASH_MAP_VAL:
-		    hash_xor_values ^= hash;
+		case HASH_MAP_PAIR:
+		    hash_xor_pairs ^= hash;
 		    hash = 0;
 		    goto hash2_common;
 		default:
-- 
cgit v1.2.3


From 0d10df770cfd1407bb79c31da34108dda1b868a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 21 Jan 2015 16:08:17 +0100
Subject: erts: Add fallback for builtin clz

* __builtin_clz may not exist
* fix bitcount fallback
---
 erts/emulator/beam/erl_hashmap.c | 27 +++++++++++++++++++++++++++
 erts/emulator/beam/erl_hashmap.h | 17 ++++-------------
 2 files changed, 31 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 6bca2f1665..3da2a53333 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -1273,3 +1273,30 @@ BIF_RETTYPE hashmap_info_1(BIF_ALIST_1) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
+
+/* implementation of builtin emulations */
+
+#if !defined(__GNUC__)
+/* Count leading zeros emulation */
+Uint32 hashmap_clz(Uint32 x) {
+    Uint32 y;
+    int n = 32;
+    y = x >>16;  if (y != 0) {n = n -16;  x = y;}
+    y = x >> 8;  if (y != 0) {n = n - 8;  x = y;}
+    y = x >> 4;  if (y != 0) {n = n - 4;  x = y;}
+    y = x >> 2;  if (y != 0) {n = n - 2;  x = y;}
+    y = x >> 1;  if (y != 0) return n - 2;
+    return n - x;
+}
+const Uint32 SK5 = 0x55555555, SK3 = 0x33333333;
+const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF;
+
+/* CTPOP emulation */
+Uint32 hashmap_bitcount(Uint32 x) {
+    x -= ((x >> 1  ) & SK5);
+    x  =  (x & SK3 ) + ((x >> 2 ) & SK3 );
+    x  =  (x & SKF0) + ((x >> 4 ) & SKF0);
+    x +=   x >> 8;
+    return (x + (x >> 16)) & 0x3F;
+}
+#endif
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index 8ba249b053..1964787218 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -40,21 +40,12 @@ int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 
 /* HASH */
 
-
 #if defined(__GNUC__)
-#define hashmap_bitcount(x) 	(Uint32) __builtin_popcount((unsigned int) (x))
+#define hashmap_clz(x)       ((Uint32) __builtin_clz((unsigned int)(x)))
+#define hashmap_bitcount(x)  ((Uint32) __builtin_popcount((unsigned int) (x)))
 #else
-const Uint32 SK5 = 0x55555555, SK3 = 0x33333333;
-const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF;
-
-/* CTPOP emulation */
-Uint32 hashmap_bitcount(Uint32 map) {
-    map -= (( map >> 1  ) & SK5 );
-    map  = ( map & SK3  ) + (( map >> 2 ) & SK3 );
-    map  = ( map & SKF0 ) + (( map >> 4 ) & SKF0);
-    map +=   map >> 8;
-    return ( map + ( map >> 16)) & 0x3F;
-}
+Uint32 hashmap_clz(Uint32 x);
+Uint32 hashmap_bitcount(Uint32 x);
 #endif
 
 /* hamt nodes v2.0
-- 
cgit v1.2.3


From e179bd5be24b6b07cc4e8d0e58211fd62599ee9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 3 Feb 2015 15:00:44 +0100
Subject: erts: Fix hashmap head array printf term

---
 erts/emulator/beam/erl_printf_term.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index f07fe30cf6..b07b7785dd 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -594,6 +594,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		head = hashmap_val(wobj);
 		mapval = MAP_HEADER_VAL(*head);
 		switch (MAP_HEADER_TYPE(*head)) {
+		    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY:
 		    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP:
 			PRINT_STRING(res, fn, arg, "#<");
 			PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval);
-- 
cgit v1.2.3


From 831ac12e04004c2e93aafc9f52264a57757fa2eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 10 Dec 2014 17:52:28 +0100
Subject: erts: Add hashmap:from_list/1

---
 erts/emulator/beam/bif.tab       |   1 +
 erts/emulator/beam/erl_hashmap.c | 415 +++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_hashmap.h |   3 +
 3 files changed, 419 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 27ae8adcec..5ffd5b37b5 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -621,6 +621,7 @@ bif hashmap:find/2
 bif hashmap:update/3
 bif hashmap:remove/2
 bif hashmap:info/1
+bif hashmap:from_list/1
 bif hashmap:to_list/1
 bif hashmap:new/0
 bif hashmap:is_key/2
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 3da2a53333..ddb55c53ce 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -67,14 +67,28 @@ static char *format_binary(Uint64 x, char *b) {
 #endif
 
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
+
+/* for hashmap_from_list/1 */
+typedef struct {
+    Uint32 hx;
+    Uint32 skip;
+    Uint i;
+    Eterm  val;
+} hxnode_t;
+
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
+static Eterm hashmap_from_list(Process *p, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
+static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
+static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
+static int hxnodecmp(hxnode_t* a, hxnode_t* b);
+static int hxnodecmpkey(hxnode_t* a, hxnode_t* b);
 
 /* hashmap:new/0 */
 
@@ -129,6 +143,15 @@ BIF_RETTYPE hashmap_to_list_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
+/* hashmap:from_list/1 */
+
+BIF_RETTYPE hashmap_from_list_1(BIF_ALIST_1) {
+    if (is_list(BIF_ARG_1) || is_nil(BIF_ARG_1)) {
+	return hashmap_from_list(BIF_P, BIF_ARG_1);
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
 /* hashmap:get/2 */
 
 BIF_RETTYPE hashmap_get_2(BIF_ALIST_2) {
@@ -794,6 +817,398 @@ not_found:
     return res;
 }
 
+#define swizzle32(D,S) \
+    do { \
+	(D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20  \
+	    | ((S) & 0x00000f00) << 12 | ((S) & 0x0000f000) << 4   \
+	    | ((S) & 0x000f0000) >> 4  | ((S) & 0x00f00000) >> 12  \
+	    | ((S) & 0x0f000000) >> 20 | ((S) & 0xf0000000) >> 28; \
+    } while(0)
+
+
+static Eterm hashmap_from_list(Process *p, Eterm list) {
+    Eterm *kv, res, item = list;
+    Eterm *hp;
+    Eterm tmp[2];
+    Uint32 sw, hx;
+    Uint jx = 0, ix = 0, lx, cx, n = 0;
+    hxnode_t *hxns;
+
+    /* Calculate size and check validity */
+
+    if (is_nil(list)) {
+	hp    = HAlloc(p, HAMT_HEAD_EMPTY_SZ);
+	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(0);
+	hp[1] = 0;
+	return make_hashmap(hp);
+    }
+
+    while(is_list(item)) {
+	res = CAR(list_val(item));
+	if (is_not_tuple(res))
+	    goto error;
+
+	kv = tuple_val(res);
+	if (*kv != make_arityval(2))
+	    goto error;
+
+	n++;
+	item = CDR(list_val(item));
+    }
+
+    if (is_not_nil(item))
+	goto error;
+
+    hp = HAlloc(p, (2 * n));
+
+    /* create tmp hx values and leaf ptrs */
+    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
+    item = list;
+
+    while(is_list(item)) {
+	res = CAR(list_val(item));
+	kv  = tuple_val(res);
+	hx  = hashmap_restore_hash(tmp,0,kv[1]);
+	swizzle32(sw,hx);
+	hxns[ix].hx   = sw;
+	hxns[ix].val  = CONS(hp, kv[1], kv[2]); hp += 2;
+	hxns[ix].skip = 1; /* will be reassigned in from_array */
+	hxns[ix].i    = ix;
+	ix++;
+	item = CDR(list_val(item));
+    }
+
+    ASSERT(n > 0);
+
+    /* sort and compact array (remove non-unique entries) */
+    qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
+
+    ix = 0, cx = 0;
+    while(ix < n - 1) {
+	if (hxns[ix].hx == hxns[ix+1].hx) {
+
+	    /* find region of equal hash values */
+	    jx = ix + 1;
+	    while(jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; }
+	    /* find all correct keys from region
+	     * (last in list but now hash sorted so we check highest id instead) */
+
+	    /* resort with keys instead of hash value within region */
+	    
+	    qsort(&hxns[ix], jx - ix, sizeof(hxnode_t),
+		    (int (*)(const void *, const void *)) hxnodecmpkey);
+
+	    while(ix < jx) {
+		lx = ix;
+		while(ix < jx && EQ(CAR(list_val(hxns[ix].val)), CAR(list_val(hxns[lx].val)))) {
+		    if (hxns[ix].i > hxns[lx].i) {
+			lx = ix;
+		    }
+		    ix++;
+		}
+		hxns[cx].hx  = hxns[lx].hx;
+		hxns[cx].val = hxns[lx].val;
+		cx++;
+	    }
+	    ix = jx;
+	    continue;
+	}
+	if (ix > cx) {
+	    hxns[cx].hx  = hxns[ix].hx;
+	    hxns[cx].val = hxns[ix].val;
+	}
+	cx++;
+	ix++;
+    }
+
+    if (ix < n) {
+	hxns[cx].hx  = hxns[ix].hx;
+	hxns[cx].val = hxns[ix].val;
+	cx++;
+    }
+
+    if (cx > 1) {
+	/* recursive decompose array */
+	res = hashmap_from_sorted_unique_array(p, hxns, cx, 0);
+    } else {
+	 /* hash value has been swizzled, need to drag it down to get the
+	 * correct slot. */
+	hp    = HAlloc(p, HAMT_HEAD_BITMAP_SZ(1));
+	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf));
+	hp[1] = 1;
+	hp[2] = hxns[0].val;
+	res   = make_hashmap(hp);
+    }
+
+    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+    BIF_RET(res);
+error:
+    BIF_ERROR(p, BADARG);
+}
+
+
+static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int lvl) {
+    Eterm res = NIL;
+    Uint i,ix,jx,elems;
+    Uint32 sw, hx;
+    Eterm val;
+    Eterm th[2];
+    hxnode_t *tmp;
+
+    ASSERT(lvl < 32);
+    ix = 0;
+    elems = 1;
+    while (ix < n - 1) {
+	if (hxns[ix].hx == hxns[ix+1].hx) {
+	    jx = ix + 1;
+	    while (jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; }
+	    tmp = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, ((jx - ix)) * sizeof(hxnode_t));
+
+	    for(i = 0; i < jx - ix; i++) {
+		val = hxns[i + ix].val;
+		hx  = hashmap_restore_hash(th, lvl + 8, CAR(list_val(val)));
+		swizzle32(sw,hx);
+		tmp[i].hx   = sw;
+		tmp[i].val  = val;
+		tmp[i].i    = i;
+		tmp[i].skip = 1;
+	    }
+
+	    qsort(tmp, jx - ix, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
+
+	    hxns[ix].skip = jx - ix;
+	    hxns[ix].val  = hashmap_from_sorted_unique_array(p, tmp, jx - ix, lvl + 8);
+	    erts_free(ERTS_ALC_T_TMP, (void *) tmp);
+	    ix = jx;
+	    if (ix < n) { elems++; }
+	    continue;
+	}
+	hxns[ix].skip = 1;
+	elems++;
+	ix++;
+    }
+
+    res = hashmap_from_chunked_array(p, hxns, elems, !lvl);
+
+    ERTS_HOLE_CHECK(p);
+
+    return res;
+}
+
+#define maskval(V,L)      (((V) >> ((7 - (L))*4)) & 0xf)
+#define cdepth(V1,V2)     (hashmap_clz((V1) ^ (V2)) >> 2)
+
+/* n must be > 1
+ * hash values in hxns has to be unique
+ */
+
+#define HALLOC_EXTRA 200
+static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root) {
+    Uint ix, d, dn, dc, slot, elems;
+    Uint32 v, vp, vn, hdr;
+    Uint bp, sz;
+    DECLARE_ESTACK(stack);
+    Eterm res = NIL, *hp = NULL, *nhp;
+
+    ASSERT(n > 1);
+
+    /* push initial nodes on the stack,
+     * this is the starting depth */
+
+    ix = 0;
+    d  = 0;
+    vp = hxns[ix].hx;
+    v  = hxns[ix + hxns[ix].skip].hx;
+
+    ASSERT(vp > v);
+    slot = maskval(vp,d);
+
+    while(slot == maskval(v,d)) {
+	ESTACK_PUSH(stack, 1 << slot);
+	d++;
+	slot = maskval(vp,d);
+    }
+
+    res = hxns[ix].val;
+
+    if (hxns[ix].skip > 1) {
+	dc = 7;
+	/* build collision nodes */
+	while (dc > d) {
+	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(vp,dc));
+	    hp[1] = res;
+	    res   = make_hashmap(hp);
+	    dc--;
+	}
+    }
+
+    ESTACK_PUSH(stack, res);
+    ESTACK_PUSH(stack, 1 << slot);
+
+    /* all of the other nodes .. */
+    elems = n - 2; /* remove first and last elements */
+    while(elems--) {
+	hdr = ESTACK_POP(stack);
+	ix  = ix + hxns[ix].skip;
+
+	/* determine if node or subtree should be built by looking
+	 * at the next value. */
+
+	vn = hxns[ix + hxns[ix].skip].hx;
+	dn = cdepth(v,vn);
+	ASSERT(v > vn);
+
+	res = hxns[ix].val;
+
+	if (hxns[ix].skip > 1) {
+	    int wat = (d > dn) ? d : dn;
+	    dc = 7;
+	    /* build collision nodes */
+	    while (dc > wat) {
+		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
+		hp[1] = res;
+		res   = make_hashmap(hp);
+		dc--;
+	    }
+	}
+
+	/* next depth is higher (implies collision) */
+	if (d < dn) {
+	    /* hdr is the popped one initially */
+	    while(d < dn) {
+		slot = maskval(v, d);
+		bp   = 1 << slot;
+		ESTACK_PUSH(stack, hdr | bp);
+		d++;
+		hdr = 0; /* clear hdr for all other collisions */
+	    }
+
+	    slot = maskval(v, d);
+	    bp   = 1 << slot;
+	    /* no more collisions */
+	    ESTACK_PUSH(stack,res);
+	    ESTACK_PUSH(stack,bp);
+	} else if (d == dn) {
+	    /* no collisions at all */
+	    slot = maskval(v, d);
+	    bp   = 1 << slot;
+	    ESTACK_PUSH(stack,res);
+	    ESTACK_PUSH(stack,hdr | bp);
+	} else {
+	    /* dn < n, we have a drop and we are done
+	     * build nodes and subtree */
+	    while (dn != d) {
+		slot  = maskval(v, d);
+		bp    = 1 << slot;
+		/* OR bitposition before sz calculation to handle
+		 * redundant collisions */
+		hdr  |= bp;
+		sz    = hashmap_bitcount(hdr);
+		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
+		nhp   = hp;
+		*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+		*hp++ = res; sz--;
+		while (sz--) { *hp++ = ESTACK_POP(stack); }
+		ASSERT((hp - nhp) < 18);
+		res = make_hashmap(nhp);
+
+		/* we need to pop the next hdr and push if we don't need it */
+
+		hdr = ESTACK_POP(stack);
+		d--;
+	    }
+	    ESTACK_PUSH(stack, res);
+	    ESTACK_PUSH(stack, hdr);
+	}
+
+	vp = v;
+	v  = vn;
+	d  = dn;
+	ERTS_HOLE_CHECK(p);
+    }
+
+    /* v and vp are reused from above */
+    dn  = cdepth(vp,v);
+    ix  = ix + hxns[ix].skip;
+    res = hxns[ix].val;
+
+    if (hxns[ix].skip > 1) {
+	dc = 7;
+	/* build collision nodes */
+	while (dc > dn) {
+	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
+	    hp[1] = res;
+	    res   = make_hashmap(hp);
+	    dc--;
+	}
+    }
+
+    hdr = ESTACK_POP(stack);
+    /* pop remaining subtree if any */
+    while (dn) {
+	slot  = maskval(v, dn);
+	bp    = 1 << slot;
+	/* OR bitposition before sz calculation to handle
+	 * redundant collisions */
+	hdr  |= bp;
+	sz    = hashmap_bitcount(hdr);
+	hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
+	nhp   = hp;
+	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+	*hp++ = res; sz--;
+
+	while (sz--) { *hp++ = ESTACK_POP(stack); }
+	res = make_hashmap(nhp);
+	hdr = ESTACK_POP(stack);
+	dn--;
+    }
+
+    /* and finally the root .. */
+
+    slot  = maskval(v, dn);
+    bp    = 1 << slot;
+    hdr  |= bp;
+    sz    = hashmap_bitcount(hdr);
+    hp    = HAlloc(p, sz + /* hdr + item */ (is_root ? 2 : 1));
+    nhp   = hp;
+
+    if (is_root) {
+	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr);
+	*hp++ = n;
+    } else {
+	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+    }
+
+    *hp++ = res; sz--;
+    while (sz--) { *hp++ = ESTACK_POP(stack); }
+
+    res = make_hashmap(nhp);
+
+    ASSERT(ESTACK_COUNT(stack) == 0);
+    DESTROY_ESTACK(stack);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+#undef HALLOC_EXTRA
+
+static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) {
+    return CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val)));
+}
+
+static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
+    if (a->hx < b->hx)
+	return 1;
+    else if (a->hx == b->hx)
+	return 0;
+    else
+	return -1;
+}
+
 #define HALLOC_EXTRA 200
 
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index 1964787218..f05505bae0 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -127,6 +127,9 @@ typedef struct hashmap_head_s {
 #define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \
     MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp)
 
+#define MAP_HEADER_HAMT_NODE_ARRAY \
+    make_arityval(16)
+
 #define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \
     MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp)
 
-- 
cgit v1.2.3


From 7a12c43da25e3dcad54212f538ebae3dc13f5c2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 17 Feb 2015 14:35:14 +0100
Subject: erts: Refactor erl_hashmap header includes

---
 erts/emulator/beam/erl_hashmap.c | 54 +++++++++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_hashmap.h | 43 +++-----------------------------
 erts/emulator/beam/erl_term.h    | 24 +++++++++++++++++-
 erts/emulator/beam/external.c    |  1 +
 erts/emulator/beam/utils.c       |  1 +
 5 files changed, 74 insertions(+), 49 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index ddb55c53ce..5646820ae1 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -84,7 +84,7 @@ static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
-
+static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n);
 static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
 static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
 static int hxnodecmp(hxnode_t* a, hxnode_t* b);
@@ -831,7 +831,7 @@ static Eterm hashmap_from_list(Process *p, Eterm list) {
     Eterm *hp;
     Eterm tmp[2];
     Uint32 sw, hx;
-    Uint jx = 0, ix = 0, lx, cx, n = 0;
+    Uint ix = 0, n = 0;
     hxnode_t *hxns;
 
     /* Calculate size and check validity */
@@ -880,6 +880,47 @@ static Eterm hashmap_from_list(Process *p, Eterm list) {
 
     ASSERT(n > 0);
 
+    res = hashmap_from_unsorted_array(p, hxns, n);
+
+    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+    BIF_RET(res);
+error:
+    BIF_ERROR(p, BADARG);
+}
+
+Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
+    Eterm tmp[2];
+    Uint32 sw, hx;
+    Uint ix;
+    hxnode_t *hxns;
+    Eterm res;
+
+    /* create tmp hx values and leaf ptrs */
+    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
+
+    for (ix = 0; ix < n; ix++) {
+	hx  = hashmap_restore_hash(tmp,0,CAR(list_val(leafs[ix])));
+	swizzle32(sw,hx);
+	hxns[ix].hx   = sw;
+	hxns[ix].val  = leafs[ix];
+	hxns[ix].skip = 1;
+	hxns[ix].i    = ix;
+    }
+
+    res = hashmap_from_unsorted_array(p, hxns, n);
+
+    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+    return res;
+}
+
+static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
+    Uint jx = 0, ix = 0, lx, cx;
+    Eterm res;
+
     /* sort and compact array (remove non-unique entries) */
     qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
 
@@ -931,6 +972,7 @@ static Eterm hashmap_from_list(Process *p, Eterm list) {
 	/* recursive decompose array */
 	res = hashmap_from_sorted_unique_array(p, hxns, cx, 0);
     } else {
+	Eterm *hp;
 	 /* hash value has been swizzled, need to drag it down to get the
 	 * correct slot. */
 	hp    = HAlloc(p, HAMT_HEAD_BITMAP_SZ(1));
@@ -940,15 +982,9 @@ static Eterm hashmap_from_list(Process *p, Eterm list) {
 	res   = make_hashmap(hp);
     }
 
-    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
-
-    BIF_RET(res);
-error:
-    BIF_ERROR(p, BADARG);
+    return res;
 }
 
-
 static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int lvl) {
     Eterm res = NIL;
     Uint i,ix,jx,elems;
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index f05505bae0..b5fbc636e6 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -22,21 +22,14 @@
 #define __ERL_HASH_H__
 
 #include "sys.h"
+#include "erl_term.h"
 
 Eterm erts_hashmap_get(Eterm key, Eterm map);
 struct ErtsWStack_;
 void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
-
-/* erl_term.h stuff */
-#define make_hashmap(x)		make_boxed((Eterm*)(x))
-#define make_hashmap_rel 	make_boxed_rel
-#define is_hashmap(x)		(is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
-#define is_hashmap_rel(RTERM,BASE)  is_hashmap(rterm2wterm(RTERM,BASE))
-#define is_hashmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HASHMAP)
-#define hashmap_val(x)		_unchecked_boxed_val((x))
-#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE))
+Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
 
 /* HASH */
 
@@ -54,8 +47,8 @@ Uint32 hashmap_bitcount(Uint32 x);
  * head
  */
 
-/* the head-node is a bitmap or array with an untagged size
- */
+/* the head-node is a bitmap or array with an untagged size */
+
 typedef struct hashmap_head_s {
     Eterm thing_word;
     Uint size;
@@ -65,21 +58,6 @@ typedef struct hashmap_head_s {
 #define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
 #define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
  
-/* the bitmap-node
- * typedef struct hashmap_bitmap_node_s {
- *     Eterm thing_word;
- *     Eterm items[1];
- * } hashmap_bitmap_node_t;
- *
- * the array-node is a tuple
- * typedef struct hashmap_bitmap_node_s {
- *     Eterm thing_word; 
- *     Eterm items[1];
- * } hashmap_bitmap_node_t;
- *
- * the leaf-node
- * cons-cell
- */
 
 /* thing_word tagscheme
  * Need two bits for map subtags
@@ -103,19 +81,6 @@ typedef struct hashmap_head_s {
 
 /* erl_map.h stuff */
 
-#define MAP_HEADER_TAG_SZ                 (2)
-#define MAP_HEADER_ARITY_SZ               (8)
-#define MAP_HEADER_VAL_SZ                 (16)
-
-#define MAP_HEADER_TAG_FLAT               (0x0)
-#define MAP_HEADER_TAG_HAMT_NODE_BITMAP   (0x1)
-#define MAP_HEADER_TAG_HAMT_HEAD_ARRAY    (0x2)
-#define MAP_HEADER_TAG_HAMT_HEAD_BITMAP   (0x3)
-
-#define MAP_HEADER_TYPE(Hdr)  (((Hdr) >> (_HEADER_ARITY_OFFS)) & (0x3))
-#define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff))
-#define MAP_HEADER_VAL(Hdr)   (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff))
-
 #define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2)))
 
 #define MAKE_MAP_HEADER(Type,Arity,Val) \
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 72946d9ddf..7605a41cd8 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -21,7 +21,6 @@
 #define __ERL_TERM_H
 
 #include "sys.h" /* defines HALFWORD_HEAP */
-#include "erl_hashmap.h"
 
 typedef UWord Wterm;  /* Full word terms */
 
@@ -996,6 +995,29 @@ _ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm)
 _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #define external_ref_node(x) _ET_APPLY(external_ref_node,(x))
 
+/* maps */
+
+#define MAP_HEADER_TAG_SZ                 (2)
+#define MAP_HEADER_ARITY_SZ               (8)
+#define MAP_HEADER_VAL_SZ                 (16)
+
+#define MAP_HEADER_TAG_FLAT               (0x0)
+#define MAP_HEADER_TAG_HAMT_NODE_BITMAP   (0x1)
+#define MAP_HEADER_TAG_HAMT_HEAD_ARRAY    (0x2)
+#define MAP_HEADER_TAG_HAMT_HEAD_BITMAP   (0x3)
+
+#define MAP_HEADER_TYPE(Hdr)  (((Hdr) >> (_HEADER_ARITY_OFFS)) & (0x3))
+#define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff))
+#define MAP_HEADER_VAL(Hdr)   (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff))
+
+#define make_hashmap(x)		make_boxed((Eterm*)(x))
+#define make_hashmap_rel 	make_boxed_rel
+#define is_hashmap(x)		(is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
+#define is_hashmap_rel(RTERM,BASE)  is_hashmap(rterm2wterm(RTERM,BASE))
+#define is_hashmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HASHMAP)
+#define hashmap_val(x)		_unchecked_boxed_val((x))
+#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE))
+
 /* number tests */
 
 #define is_integer(x)		(is_small(x) || is_big(x))
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index af8db4c265..9030b528a4 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -45,6 +45,7 @@
 #include "erl_bits.h"
 #include "erl_zlib.h"
 #include "erl_map.h"
+#include "erl_hashmap.h"
 
 #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d234e8fc68..f595cfbacd 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -32,6 +32,7 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "erl_map.h"
+#include "erl_hashmap.h"
 #include "packet_parser.h"
 #include "erl_gc.h"
 #define ERTS_WANT_DB_INTERNAL__
-- 
cgit v1.2.3


From 9cfaa729d7319ede30f62ffaaf82eb10fbaf8a60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 19 Feb 2015 16:25:03 +0100
Subject: erts: Move hashmap:size/1 to maps

---
 erts/emulator/beam/bif.tab         |  1 -
 erts/emulator/beam/erl_bif_guard.c | 27 +++++++++++++++------------
 erts/emulator/beam/erl_hashmap.c   | 14 --------------
 erts/emulator/beam/erl_map.c       | 10 ++++++++++
 4 files changed, 25 insertions(+), 27 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 5ffd5b37b5..8606a41c7b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -626,7 +626,6 @@ bif hashmap:to_list/1
 bif hashmap:new/0
 bif hashmap:is_key/2
 bif hashmap:keys/1
-bif hashmap:size/1
 bif erlang:is_hashmap/1
 bif hashmap:values/1
 bif hashmap:merge/2
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index bbd8aa31d9..a5d1d3a5cb 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -34,6 +34,7 @@
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_map.h"
+#include "erl_hashmap.h"
 
 static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live);
 
@@ -459,23 +460,25 @@ Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live)
 Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg = reg[live];
+    Eterm* hp;
+    Uint size;
     if (is_map(arg)) {
 	map_t *mp = (map_t*)map_val(arg);
-	Uint size = map_get_size(mp);
-	if (IS_USMALL(0, size)) {
-	    return make_small(size);
-	} else {
-	    Eterm* hp;
-	    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
-		erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
-	    }
-	    hp = p->htop;
-	    p->htop += BIG_UINT_HEAP_SIZE;
-	    return uint_to_big(size, hp);
-	}
+	size = map_get_size(mp);
+    } else if (is_hashmap(arg)) {
+	size = hashmap_size(arg);
     } else {
 	BIF_ERROR(p, BADARG);
     }
+    if (IS_USMALL(0, size)) {
+	return make_small(size);
+    }
+    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
+	erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
+    }
+    hp = p->htop;
+    p->htop += BIG_UINT_HEAP_SIZE;
+    return uint_to_big(size, hp);
 }
 
 Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live)
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 5646820ae1..3f6e12cdd8 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -200,20 +200,6 @@ BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
 }
 /* hashmap:size/1 */
 
-BIF_RETTYPE hashmap_size_1(BIF_ALIST_1) {
-    if (is_hashmap(BIF_ARG_1)) {
-	Eterm *head, *hp, res;
-	Uint size, hsz=0;
-
-	head = hashmap_val(BIF_ARG_1);
-	size = head[1];
-	(void) erts_bld_uint(NULL, &hsz, size);
-	hp = HAlloc(BIF_P, hsz);
-	res = erts_bld_uint(&hp, NULL, size);
-	BIF_RET(res);
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
 
 /* erlang:is_hashmap/1 */
 
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index b2a16eb5ed..ecbb91bc33 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -77,6 +77,16 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) {
 	erts_bld_uint(NULL, &hsz, n);
 	hp = HAlloc(BIF_P, hsz);
 	BIF_RET(erts_bld_uint(&hp, NULL, n));
+    } else if (is_hashmap(BIF_ARG_1)) {
+	Eterm *head, *hp, res;
+	Uint size, hsz=0;
+
+	head = hashmap_val(BIF_ARG_1);
+	size = head[1];
+	(void) erts_bld_uint(NULL, &hsz, size);
+	hp = HAlloc(BIF_P, hsz);
+	res = erts_bld_uint(&hp, NULL, size);
+	BIF_RET(res);
     }
 
     BIF_ERROR(BIF_P, BADARG);
-- 
cgit v1.2.3


From 903740ac57b00d404f430876b82cb21e0bb684a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 19 Feb 2015 16:53:32 +0100
Subject: erts: Move hashmap:to_list/1, keys/1 and values/1 to maps

---
 erts/emulator/beam/bif.tab       |   3 -
 erts/emulator/beam/erl_hashmap.c | 133 +--------------------------------------
 erts/emulator/beam/erl_hashmap.h |  10 +--
 erts/emulator/beam/erl_map.c     | 115 ++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_map.h     |   7 +++
 5 files changed, 123 insertions(+), 145 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 8606a41c7b..6bd9291d34 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -622,12 +622,9 @@ bif hashmap:update/3
 bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:from_list/1
-bif hashmap:to_list/1
 bif hashmap:new/0
 bif hashmap:is_key/2
-bif hashmap:keys/1
 bif erlang:is_hashmap/1
-bif hashmap:values/1
 bif hashmap:merge/2
 
 #
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 3f6e12cdd8..064ae73acd 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -79,9 +79,6 @@ typedef struct {
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_from_list(Process *p, Eterm node);
-static Eterm hashmap_to_list(Process *p, Eterm map);
-static Eterm hashmap_keys(Process *p, Eterm map);
-static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n);
@@ -135,14 +132,6 @@ BIF_RETTYPE hashmap_update_3(BIF_ALIST_3) {
 
 /* hashmap:to_list/1 */
 
-BIF_RETTYPE hashmap_to_list_1(BIF_ALIST_1) {
-    if (is_hashmap(BIF_ARG_1)) {
-	return hashmap_to_list(BIF_P, BIF_ARG_1);
-    }
-
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 /* hashmap:from_list/1 */
 
 BIF_RETTYPE hashmap_from_list_1(BIF_ALIST_1) {
@@ -222,25 +211,10 @@ BIF_RETTYPE hashmap_is_key_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* hashmap:keys/1
- */
-
-BIF_RETTYPE hashmap_keys_1(BIF_ALIST_1) {
-    if (is_hashmap(BIF_ARG_1)) {
-	BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1));
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
+/* hashmap:keys/1 */
 
-/* hashmap:keys/1
- */
 
-BIF_RETTYPE hashmap_values_1(BIF_ALIST_1) {
-    if (is_hashmap(BIF_ARG_1)) {
-	BIF_RET(hashmap_values(BIF_P, BIF_ARG_1));
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
+/* hashmap:values/1 */
 
 BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_1) && is_hashmap(BIF_ARG_2)) {
@@ -1448,109 +1422,6 @@ recurse:
     return res;
 }
 
-void hashmap_iterator_init(ErtsWStack* s, Eterm node) {
-    WSTACK_PUSH((*s), (UWord)THE_NON_VALUE);  /* end marker */
-    WSTACK_PUSH((*s), (UWord)node);
-}
-
-Eterm* hashmap_iterator_next(ErtsWStack* s) {
-    Eterm node, *ptr, hdr;
-    Uint32 sz;
-
-    for (;;) {
-        ASSERT(!WSTACK_ISEMPTY((*s)));
-	node = (Eterm) WSTACK_POP((*s));
-        if (is_non_value(node)) {
-            return NULL;
-        }
-        switch (primary_tag(node)) {
-        case TAG_PRIMARY_LIST:
-            return list_val(node);
-
-        case TAG_PRIMARY_BOXED:
-            ptr = boxed_val(node);
-            hdr = *ptr;
-            ASSERT(is_header(hdr));
-            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-            case HAMT_SUBTAG_HEAD_ARRAY:
-                ptr++;
-            case HAMT_SUBTAG_NODE_ARRAY:
-                ptr++;
-                sz = 16;
-                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
-                break;
-            case HAMT_SUBTAG_HEAD_BITMAP:
-                ptr++;
-            case HAMT_SUBTAG_NODE_BITMAP:
-                sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-                ASSERT(sz < 17);
-                ptr++;
-                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
-                break;
-            default:
-                erl_exit(1, "bad header");
-            }
-            break;
-
-        default:
-            erl_exit(1, "bad hamt node");
-	}
-    }
-}
-
-static Eterm hashmap_to_list(Process *p, Eterm node) {
-    DECLARE_WSTACK(stack);
-    hashmap_head_t* root;
-    Eterm *hp, *kv;
-    Eterm res = NIL;
-
-    root = (hashmap_head_t*) boxed_val(node);
-    hp  = HAlloc(p, root->size * (2 + 3));
-    hashmap_iterator_init(&stack, node);
-    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
-	Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv));
-	hp += 3;
-	res = CONS(hp, tup, res);
-	hp += 2;
-    }
-    DESTROY_WSTACK(stack);
-    return res;
-}
-
-static Eterm hashmap_keys(Process* p, Eterm node) {
-    DECLARE_WSTACK(stack);
-    hashmap_head_t* root;
-    Eterm *hp, *kv;
-    Eterm res = NIL;
-
-    root = (hashmap_head_t*) boxed_val(node);
-    hp  = HAlloc(p, root->size * 2);
-    hashmap_iterator_init(&stack, node);
-    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
-	res = CONS(hp, CAR(kv), res);
-	hp += 2;
-    }
-    DESTROY_WSTACK(stack);
-    return res;
-}
-
-static Eterm hashmap_values(Process* p, Eterm node) {
-    DECLARE_WSTACK(stack);
-    hashmap_head_t* root;
-    Eterm *hp, *kv;
-    Eterm res = NIL;
-
-    root = (hashmap_head_t*) boxed_val(node);
-    hp  = HAlloc(p, root->size * 2);
-    hashmap_iterator_init(&stack, node);
-    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
-	res = CONS(hp, CDR(kv), res);
-	hp += 2;
-    }
-    DESTROY_WSTACK(stack);
-    return res;
-}
-
 static int hash_cmp(Uint32 ha, Uint32 hb)
 {
     int i;
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index b5fbc636e6..5a9aa05f61 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -25,9 +25,6 @@
 #include "erl_term.h"
 
 Eterm erts_hashmap_get(Eterm key, Eterm map);
-struct ErtsWStack_;
-void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
-Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
 
@@ -46,18 +43,13 @@ Uint32 hashmap_bitcount(Uint32 x);
  * node :: leaf | array | bitmap
  * head
  */
-
-/* the head-node is a bitmap or array with an untagged size */
-
 typedef struct hashmap_head_s {
     Eterm thing_word;
     Uint size;
     Eterm items[1];
 } hashmap_head_t;
 
-#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
-#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
- 
+
 
 /* thing_word tagscheme
  * Need two bits for map subtags
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ecbb91bc33..289cd4fd13 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -31,6 +31,7 @@
 #include "bif.h"
 
 #include "erl_map.h"
+#include "erl_hashmap.h"
 
 /* BIFs
  *
@@ -62,6 +63,10 @@
  * - erts_internal:map_to_tuple_keys/1
  */
 
+static Eterm hashmap_to_list(Process *p, Eterm map);
+static Eterm hashmap_keys(Process *p, Eterm map);
+static Eterm hashmap_values(Process *p, Eterm map);
+
 /* erlang:map_size/1
  * the corresponding instruction is implemented in:
  *     beam/erl_bif_guard.c
@@ -92,8 +97,7 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* maps:to_list/1
- */
+/* maps:to_list/1 */
 
 BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
@@ -114,6 +118,8 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
 	}
 
 	BIF_RET(res);
+    } else if (is_hashmap(BIF_ARG_1)) {
+	return hashmap_to_list(BIF_P, BIF_ARG_1);
     }
 
     BIF_ERROR(BIF_P, BADARG);
@@ -386,6 +392,8 @@ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
 	}
 
 	BIF_RET(res);
+    } else if (is_hashmap(BIF_ARG_1)) {
+	BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1));
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -786,10 +794,113 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
 	}
 
 	BIF_RET(res);
+    } else if (is_hashmap(BIF_ARG_1)) {
+	BIF_RET(hashmap_values(BIF_P, BIF_ARG_1));
     }
     BIF_ERROR(BIF_P, BADARG);
 }
 
+static Eterm hashmap_to_list(Process *p, Eterm node) {
+    DECLARE_WSTACK(stack);
+    Eterm *hp, *kv;
+    Eterm res = NIL;
+
+    hp  = HAlloc(p, hashmap_size(node) * (2 + 3));
+    hashmap_iterator_init(&stack, node);
+    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
+	Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv));
+	hp += 3;
+	res = CONS(hp, tup, res);
+	hp += 2;
+    }
+    DESTROY_WSTACK(stack);
+    return res;
+}
+
+void hashmap_iterator_init(ErtsWStack* s, Eterm node) {
+    WSTACK_PUSH((*s), (UWord)THE_NON_VALUE);  /* end marker */
+    WSTACK_PUSH((*s), (UWord)node);
+}
+
+Eterm* hashmap_iterator_next(ErtsWStack* s) {
+    Eterm node, *ptr, hdr;
+    Uint32 sz;
+
+    for (;;) {
+        ASSERT(!WSTACK_ISEMPTY((*s)));
+	node = (Eterm) WSTACK_POP((*s));
+        if (is_non_value(node)) {
+            return NULL;
+        }
+        switch (primary_tag(node)) {
+        case TAG_PRIMARY_LIST:
+            return list_val(node);
+
+        case TAG_PRIMARY_BOXED:
+            ptr = boxed_val(node);
+            hdr = *ptr;
+            ASSERT(is_header(hdr));
+            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_HEAD_ARRAY:
+                ptr++;
+            case HAMT_SUBTAG_NODE_ARRAY:
+                ptr++;
+                sz = 16;
+                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
+                break;
+            case HAMT_SUBTAG_HEAD_BITMAP:
+                ptr++;
+            case HAMT_SUBTAG_NODE_BITMAP:
+                sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                ASSERT(sz < 17);
+                ptr++;
+                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
+                break;
+            default:
+                erl_exit(1, "bad header");
+            }
+            break;
+
+        default:
+            erl_exit(1, "bad hamt node");
+	}
+    }
+}
+
+static Eterm hashmap_keys(Process* p, Eterm node) {
+    DECLARE_WSTACK(stack);
+    hashmap_head_t* root;
+    Eterm *hp, *kv;
+    Eterm res = NIL;
+
+    root = (hashmap_head_t*) boxed_val(node);
+    hp  = HAlloc(p, root->size * 2);
+    hashmap_iterator_init(&stack, node);
+    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
+	res = CONS(hp, CAR(kv), res);
+	hp += 2;
+    }
+    DESTROY_WSTACK(stack);
+    return res;
+}
+
+static Eterm hashmap_values(Process* p, Eterm node) {
+    DECLARE_WSTACK(stack);
+    hashmap_head_t* root;
+    Eterm *hp, *kv;
+    Eterm res = NIL;
+
+    root = (hashmap_head_t*) boxed_val(node);
+    hp  = HAlloc(p, root->size * 2);
+    hashmap_iterator_init(&stack, node);
+    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
+	res = CONS(hp, CDR(kv), res);
+	hp += 2;
+    }
+    DESTROY_WSTACK(stack);
+    return res;
+}
+
 int erts_validate_and_sort_map(map_t* mp)
 {
     Eterm *ks  = map_get_keys(mp);
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 2e02ca4677..c104e08e27 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -42,8 +42,12 @@ typedef struct map_s {
  * -----------
  */
 
+/* the head-node is a bitmap or array with an untagged size */
 
 
+#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
+#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
+
 /* erl_term.h stuff */
 #define make_map(x)		make_boxed((Eterm*)(x))
 #define make_map_rel(x, BASE)   make_boxed_rel((Eterm*)(x),(BASE))
@@ -62,10 +66,13 @@ typedef struct map_s {
 #define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
 #define MAP_HEADER_SIZE        (sizeof(map_t) / sizeof(Eterm))
 
+struct ErtsWStack_;
 Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
 int   erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
 int   erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
 int   erts_validate_and_sort_map(map_t* map);
+void  hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
+Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 
 #if HALFWORD_HEAP
 const Eterm *
-- 
cgit v1.2.3


From 7da662fb9eb519625b3833fec34419c32620f041 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 19 Feb 2015 17:19:31 +0100
Subject: erts: Move erlang:is_hashmap/1 to maps

---
 erts/emulator/beam/beam_emu.c    |  2 +-
 erts/emulator/beam/bif.tab       |  1 -
 erts/emulator/beam/erl_bif_op.c  |  2 +-
 erts/emulator/beam/erl_hashmap.c | 11 +----------
 erts/emulator/beam/erl_term.h    |  1 +
 5 files changed, 4 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b734d34872..034436e975 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -699,7 +699,7 @@ void** beam_ops;
         Fail; 								  \
     }
 
-#define IsMap(Src, Fail) if (is_not_map(Src)) { Fail; }
+#define IsMap(Src, Fail) if (is_not_map(Src) && is_not_hashmap(Src)) { Fail; }
 
 #define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; }
 
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 6bd9291d34..f7f6eb9213 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -624,7 +624,6 @@ bif hashmap:info/1
 bif hashmap:from_list/1
 bif hashmap:new/0
 bif hashmap:is_key/2
-bif erlang:is_hashmap/1
 bif hashmap:merge/2
 
 #
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index 37dd6457db..11c6c9e556 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -324,7 +324,7 @@ BIF_RETTYPE is_record_3(BIF_ALIST_3)
 	
 BIF_RETTYPE is_map_1(BIF_ALIST_1)
 {
-    if (is_map(BIF_ARG_1)) {
+    if (is_map(BIF_ARG_1) || is_hashmap(BIF_ARG_1)) {
 	BIF_RET(am_true);
     }
     BIF_RET(am_false);
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 064ae73acd..e51767146e 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -189,18 +189,9 @@ BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
 }
 /* hashmap:size/1 */
 
-
 /* erlang:is_hashmap/1 */
 
-BIF_RETTYPE is_hashmap_1(BIF_ALIST_1) {
-    if (is_hashmap(BIF_ARG_1)) {
-	BIF_RET(am_true);
-    }
-    BIF_RET(am_false);
-}
-
-/* hashmap:is_key/2
- */
+/* hashmap:is_key/2 */
 
 BIF_RETTYPE hashmap_is_key_2(BIF_ALIST_2) {
     if (is_hashmap(BIF_ARG_1)) {
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 7605a41cd8..264bf8bd74 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1013,6 +1013,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #define make_hashmap(x)		make_boxed((Eterm*)(x))
 #define make_hashmap_rel 	make_boxed_rel
 #define is_hashmap(x)		(is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
+#define is_not_hashmap(x)       (!is_hashmap(x))
 #define is_hashmap_rel(RTERM,BASE)  is_hashmap(rterm2wterm(RTERM,BASE))
 #define is_hashmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HASHMAP)
 #define hashmap_val(x)		_unchecked_boxed_val((x))
-- 
cgit v1.2.3


From f659c631f33ad86e7532c7198bbce6d7e07fe1dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 11:48:46 +0100
Subject: erts: Move hashmap:get/2, find/2 and is_key/2 to maps

---
 erts/emulator/beam/bif.tab       |   3 -
 erts/emulator/beam/erl_hashmap.c | 121 +--------------------------------------
 erts/emulator/beam/erl_map.c     |  97 +++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_map.h     |   7 +++
 4 files changed, 105 insertions(+), 123 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f7f6eb9213..5b0f90d418 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -616,14 +616,11 @@ bif erlang:get_keys/0
 
 # Hash Array Mappped Trie
 bif hashmap:put/3
-bif hashmap:get/2
-bif hashmap:find/2
 bif hashmap:update/3
 bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:from_list/1
 bif hashmap:new/0
-bif hashmap:is_key/2
 bif hashmap:merge/2
 
 #
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index e51767146e..7e0e7fde04 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -42,19 +42,13 @@
 #include "error.h"
 #include "bif.h"
 
+#include "erl_map.h"
 #include "erl_hashmap.h"
 
 #ifndef DECL_AM
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 #endif
 
-#define hashmap_make_hash(Key) make_hash_vsn(Key, 3)
-
-#define hashmap_restore_hash(Heap,Lvl,Key) \
-    (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7)))
-#define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
-    (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key)))
-
 #if 0
 static char *format_binary(Uint64 x, char *b) {
     int z;
@@ -76,7 +70,6 @@ typedef struct {
     Eterm  val;
 } hxnode_t;
 
-static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_from_list(Process *p, Eterm node);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
@@ -143,40 +136,8 @@ BIF_RETTYPE hashmap_from_list_1(BIF_ALIST_1) {
 }
 /* hashmap:get/2 */
 
-BIF_RETTYPE hashmap_get_2(BIF_ALIST_2) {
-    if (is_hashmap(BIF_ARG_2)) {
-	const Eterm *value;
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-
-	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
-	    BIF_RET(*value);
-	}
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 /* hashmap:find/2 */
 
-BIF_RETTYPE hashmap_find_2(BIF_ALIST_2) {
-    if (is_hashmap(BIF_ARG_2)) {
-	Eterm *hp, res;
-	const Eterm *value;
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-
-	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
-	    hp    = HAlloc(BIF_P, 3);
-	    res   = make_tuple(hp);
-	    *hp++ = make_arityval(2);
-	    *hp++ = am_ok;
-            *hp++ = *value;
-	    BIF_RET(res);
-	}
-	BIF_RET(am_error);
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
-
-
 /* hashmap:remove/2 */
 
 BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
@@ -193,18 +154,8 @@ BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
 
 /* hashmap:is_key/2 */
 
-BIF_RETTYPE hashmap_is_key_2(BIF_ALIST_2) {
-    if (is_hashmap(BIF_ARG_1)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-
-	BIF_RET(hashmap_get(hx, BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 /* hashmap:keys/1 */
 
-
 /* hashmap:values/1 */
 
 BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
@@ -216,76 +167,6 @@ BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
 
 /* impl. */
 
-static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
-    Eterm *ptr, hdr;
-    Eterm th[2];
-    Uint ix,slot, lvl = 0;
-    Uint32 hval,bp;
-
-    for (;;) {
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
-		ptr = list_val(node);
-		if (EQ(CAR(ptr), key)) {
-		    return &(CDR(ptr));
-		}
-		return NULL;
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ix   = hashmap_index(hx);
-			hx   = hashmap_shift_hash(th,hx,lvl,key);
-			node = ptr[ix+1];
-			break;
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			ix   = hashmap_index(hx);
-			hx   = hashmap_shift_hash(th,hx,lvl,key);
-			node = ptr[ix+2];
-			break;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+1];
-			    break;
-			}
-			/* not occupied */
-			return NULL;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+2];
-			    break;
-			}
-			/* not occupied */
-			return NULL;
-		    default:
-			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
-			break;
-		}
-		break;
-	    default:
-		erl_exit(1, "bad primary tag %p\r\n", node);
-		break;
-	}
-    }
-    return NULL;
-}
 
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 			    Eterm node, int is_update) {
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 289cd4fd13..0ac5885e6b 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -63,6 +63,7 @@
  * - erts_internal:map_to_tuple_keys/1
  */
 
+static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -181,6 +182,20 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
 	    BIF_RET(res);
 	}
 
+	BIF_RET(am_error);
+    } else if (is_hashmap(BIF_ARG_2)) {
+	Eterm *hp, res;
+	const Eterm *value;
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
+
+	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
+	    hp    = HAlloc(BIF_P, 3);
+	    res   = make_tuple(hp);
+	    *hp++ = make_arityval(2);
+	    *hp++ = am_ok;
+            *hp++ = *value;
+	    BIF_RET(res);
+	}
 	BIF_RET(am_error);
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -209,7 +224,15 @@ BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
 	hp = HAlloc(BIF_P, 3);
 	BIF_P->fvalue = TUPLE2(hp, error, BIF_ARG_1);
 	BIF_ERROR(BIF_P, EXC_ERROR_2);
+    } else if (is_hashmap(BIF_ARG_2)) {
+	const Eterm *value;
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
+
+	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
+	    BIF_RET(*value);
+	}
     }
+
     BIF_ERROR(BIF_P, BADARG);
 }
 
@@ -365,6 +388,9 @@ BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
 	    }
 	}
 	BIF_RET(am_false);
+    } else if (is_hashmap(BIF_ARG_2)) {
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
+	BIF_RET(hashmap_get(hx, BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -867,6 +893,77 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
     }
 }
 
+static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
+    Eterm *ptr, hdr;
+    Eterm th[2];
+    Uint ix,slot, lvl = 0;
+    Uint32 hval,bp;
+
+    for (;;) {
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
+		ptr = list_val(node);
+		if (EQ(CAR(ptr), key)) {
+		    return &(CDR(ptr));
+		}
+		return NULL;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ix   = hashmap_index(hx);
+			hx   = hashmap_shift_hash(th,hx,lvl,key);
+			node = ptr[ix+1];
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ix   = hashmap_index(hx);
+			hx   = hashmap_shift_hash(th,hx,lvl,key);
+			node = ptr[ix+2];
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
+			    node  = ptr[slot+1];
+			    break;
+			}
+			/* not occupied */
+			return NULL;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
+			    node  = ptr[slot+2];
+			    break;
+			}
+			/* not occupied */
+			return NULL;
+		    default:
+			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %p\r\n", node);
+		break;
+	}
+    }
+    return NULL;
+}
+
 static Eterm hashmap_keys(Process* p, Eterm node) {
     DECLARE_WSTACK(stack);
     hashmap_head_t* root;
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index c104e08e27..48bc316e3b 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -47,6 +47,13 @@ typedef struct map_s {
 
 #define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
 #define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
+#define hashmap_make_hash(Key) make_hash2(Key)
+
+#define hashmap_restore_hash(Heap,Lvl,Key) \
+    (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7)))
+#define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
+    (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key)))
+
 
 /* erl_term.h stuff */
 #define make_map(x)		make_boxed((Eterm*)(x))
-- 
cgit v1.2.3


From d8c6b91a9dfb06dbe1143e6b06ef26dab153ca4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 24 Feb 2015 10:50:51 +0100
Subject: erts: Refactor maps:get/2, find/2 and is_key/2

---
 erts/emulator/beam/erl_map.c | 110 +++++++++++++------------------------------
 1 file changed, 33 insertions(+), 77 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 0ac5885e6b..304b5470f5 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -137,38 +137,45 @@ erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base)
 erts_maps_get(Eterm key, Eterm map)
 #endif
 {
-    Eterm *ks, *vs;
-    map_t *mp;
-    Uint n, i;
+    Uint32 hx;
+    if (is_map(map)) {
+	Eterm *ks, *vs;
+	map_t *mp;
+	Uint n, i;
 
-    mp  = (map_t *)map_val_rel(map, map_base);
-    n   = map_get_size(mp);
+	mp  = (map_t *)map_val_rel(map, map_base);
+	n   = map_get_size(mp);
 
-    if (n == 0) {
-        return NULL;
-    }
+	if (n == 0) {
+	    return NULL;
+	}
 
-    ks  = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1;
-    vs  = map_get_values(mp);
+	ks  = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1;
+	vs  = map_get_values(mp);
 
-    if (is_immed(key)) {
-        for (i = 0; i < n; i++) {
-            if (ks[i] == key) {
-                return &vs[i];
-            }
-        }
-    }
+	if (is_immed(key)) {
+	    for (i = 0; i < n; i++) {
+		if (ks[i] == key) {
+		    return &vs[i];
+		}
+	    }
+	}
 
-    for (i = 0; i < n; i++) {
-        if (eq_rel(ks[i], NULL, key, map_base)) {
-            return &vs[i];
-        }
+	for (i = 0; i < n; i++) {
+	    if (eq_rel(ks[i], NULL, key, map_base)) {
+		return &vs[i];
+	    }
+	}
+	return NULL;
     }
-    return NULL;
+    ASSERT(is_hashmap(map));
+    hx = hashmap_make_hash(key);
+
+    return hashmap_get(hx, key, map);
 }
 
 BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
+    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
         Eterm *hp, res;
         const Eterm *value;
 
@@ -181,21 +188,6 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
             *hp++ = *value;
 	    BIF_RET(res);
 	}
-
-	BIF_RET(am_error);
-    } else if (is_hashmap(BIF_ARG_2)) {
-	Eterm *hp, res;
-	const Eterm *value;
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-
-	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
-	    hp    = HAlloc(BIF_P, 3);
-	    res   = make_tuple(hp);
-	    *hp++ = make_arityval(2);
-	    *hp++ = am_ok;
-            *hp++ = *value;
-	    BIF_RET(res);
-	}
 	BIF_RET(am_error);
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -207,7 +199,7 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
  */
 
 BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
+    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
 	Eterm *hp;
         Eterm error;
         const Eterm *value;
@@ -224,15 +216,7 @@ BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
 	hp = HAlloc(BIF_P, 3);
 	BIF_P->fvalue = TUPLE2(hp, error, BIF_ARG_1);
 	BIF_ERROR(BIF_P, EXC_ERROR_2);
-    } else if (is_hashmap(BIF_ARG_2)) {
-	const Eterm *value;
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-
-	if ((value = hashmap_get(hx, BIF_ARG_1, BIF_ARG_2)) != NULL) {
-	    BIF_RET(*value);
-	}
     }
-
     BIF_ERROR(BIF_P, BADARG);
 }
 
@@ -361,36 +345,8 @@ error:
  */
 
 BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
-	Eterm *ks, key;
-	map_t *mp;
-	Uint n,i;
-
-	mp  = (map_t*)map_val(BIF_ARG_2);
-	key = BIF_ARG_1;
-	n   = map_get_size(mp);
-	ks  = map_get_keys(mp);
-
-	if (n == 0)
-	    BIF_RET(am_false);
-
-	if (is_immed(key)) {
-	    for( i = 0; i < n; i++) {
-		if (ks[i] == key) {
-		    BIF_RET(am_true);
-		}
-	    }
-	}
-
-	for( i = 0; i < n; i++) {
-	    if (EQ(ks[i], key)) {
-		BIF_RET(am_true);
-	    }
-	}
-	BIF_RET(am_false);
-    } else if (is_hashmap(BIF_ARG_2)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-	BIF_RET(hashmap_get(hx, BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+	BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
     }
     BIF_ERROR(BIF_P, BADARG);
 }
-- 
cgit v1.2.3


From ad19eacbdb3b32bb74897ad2425808b9d669ccb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 15:37:55 +0100
Subject: erts: Move hashmap:put/3, update/3 to maps

---
 erts/emulator/beam/bif.tab       |   2 -
 erts/emulator/beam/erl_hashmap.c | 257 ---------------------------------------
 erts/emulator/beam/erl_map.c     | 247 +++++++++++++++++++++++++++++++++++++
 3 files changed, 247 insertions(+), 259 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 5b0f90d418..55a7b62e7d 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -615,8 +615,6 @@ bif erlang:fun_info_mfa/1
 bif erlang:get_keys/0
 
 # Hash Array Mappped Trie
-bif hashmap:put/3
-bif hashmap:update/3
 bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:from_list/1
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 7e0e7fde04..594a404e9f 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -60,7 +60,6 @@ static char *format_binary(Uint64 x, char *b) {
 }
 #endif
 
-static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
 
 /* for hashmap_from_list/1 */
 typedef struct {
@@ -97,32 +96,8 @@ BIF_RETTYPE hashmap_new_0(BIF_ALIST_0) {
 
 /* hashmap:put/3 */
 
-BIF_RETTYPE hashmap_put_3(BIF_ALIST_3) {
-    if (is_hashmap(BIF_ARG_3)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-	Eterm map;
-
-	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0);
-	ASSERT(is_hashmap(map));
-	BIF_RET(map);
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 /* hashmap:update/3 */
 
-BIF_RETTYPE hashmap_update_3(BIF_ALIST_3) {
-    if (is_hashmap(BIF_ARG_3)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-	Eterm map;
-
-	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 1);
-	if (is_value(map))
-	    BIF_RET(map);
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 /* hashmap:to_list/1 */
 
 /* hashmap:from_list/1 */
@@ -168,238 +143,6 @@ BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
 /* impl. */
 
 
-static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
-			    Eterm node, int is_update) {
-    Eterm *hp = NULL, *nhp = NULL;
-    Eterm *ptr;
-    Eterm hdr,res,ckey,fake;
-    Eterm th[2];
-    Uint32 ix, cix, bp, hval, chx;
-    Uint slot, lvl = 0, clvl;
-    Uint size = 0, n = 0, update_size = 1;
-    DECLARE_ESTACK(stack);
-
-    for (;;) {
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
-		ptr  = list_val(node);
-		ckey = CAR(ptr);
-		if (EQ(ckey, key)) {
-		    update_size = 0;
-		    goto unroll;
-		}
-		if (is_update) {
-		    res = THE_NON_VALUE;
-		    goto bail_out;
-		}
-		goto insert_subnodes;
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(th,hx,lvl,key);
-			size += HAMT_NODE_ARRAY_SZ;
-			ESTACK_PUSH2(stack, ix, node);
-			node  = ptr[ix+1];
-			break;
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(th,hx,lvl,key);
-			size += HAMT_HEAD_ARRAY_SZ;
-			ESTACK_PUSH2(stack, ix, node);
-			node  = ptr[ix+2];
-			break;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-			n    = hashmap_bitcount(hval);
-
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+1];
-			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); 
-			    size += HAMT_NODE_BITMAP_SZ(n);
-			    break;
-			}
-			/* not occupied */
-			if (is_update) {
-			    res = THE_NON_VALUE;
-			    goto bail_out;
-			}
-			size += HAMT_NODE_BITMAP_SZ(n+1);
-			goto unroll;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-			n    = hashmap_bitcount(hval);
-
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+2];
-			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); 
-			    size += HAMT_HEAD_BITMAP_SZ(n);
-			    break;
-			}
-			/* not occupied */
-			if (is_update) {
-			    res = THE_NON_VALUE;
-			    goto bail_out;
-			}
-			size += HAMT_HEAD_BITMAP_SZ(n+1);
-			goto unroll;
-		    default:
-			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
-			break;
-		}
-		break;
-	    default:
-		erl_exit(1, "bad primary tag %p\r\n", node);
-		break;
-	}
-    }
-insert_subnodes:
-    clvl  = lvl;
-    chx   = hashmap_restore_hash(th,clvl,ckey);
-    size += HAMT_NODE_BITMAP_SZ(2);
-    ix    = hashmap_index(hx);
-    cix   = hashmap_index(chx);
-
-    while (cix == ix) {
-	ESTACK_PUSH(stack, 0);
-	ESTACK_PUSH3(stack, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
-	size += HAMT_NODE_BITMAP_SZ(1);
-	hx    = hashmap_shift_hash(th,hx,lvl,key);
-	chx   = hashmap_shift_hash(th,chx,clvl,ckey);
-	ix    = hashmap_index(hx);
-	cix   = hashmap_index(chx);
-    }
-    ESTACK_PUSH3(stack, cix, ix, node);
-
-unroll:
-    size += 2;
-    hp  = HAlloc(p, size);
-    res = CONS(hp, key, value); hp += 2;
-
-    do {
-	node = ESTACK_POP(stack);
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST:
-		ix   = (Uint32) ESTACK_POP(stack);
-		cix  = (Uint32) ESTACK_POP(stack);
-
-		nhp   = hp;
-		*hp++ = MAP_HEADER_HAMT_NODE_BITMAP((1 << ix) | (1 << cix));
-		if (ix < cix) {
-		    *hp++ = res;
-		    *hp++ = node;
-		} else {
-		    *hp++ = node;
-		    *hp++ = res;
-		}
-		res = make_hashmap(nhp);
-		break;
-	    case TAG_PRIMARY_HEADER:
-		/* subnodes, fake it */
-		fake = node;
-		node = make_boxed(&fake);
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			slot  = (Uint)   ESTACK_POP(stack);
-			nhp   = hp;
-			n     = HAMT_NODE_ARRAY_SZ;
-			while(n--) { *hp++ = *ptr++; }
-			nhp[slot+1] = res;
-			res = make_hashmap(nhp);
-			break;
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			slot  = (Uint)   ESTACK_POP(stack);
-			nhp   = hp;
-			n     = HAMT_HEAD_ARRAY_SZ - 2;
-			*hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++;
-			*hp++ = (*ptr++) + update_size;
-			while(n--) { *hp++ = *ptr++; }
-			nhp[slot+2] = res;
-			res = make_hashmap(nhp);
-			break;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			slot  = (Uint)   ESTACK_POP(stack);
-			bp    = (Uint32) ESTACK_POP(stack);
-			n     = (Uint32) ESTACK_POP(stack);
-			hval  = MAP_HEADER_VAL(hdr);
-			nhp   = hp;
-			*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval | bp); ptr++;
-
-			n -= slot;
-			while(slot--) { *hp++ = *ptr++; }
-			*hp++ = res;
-			if (hval & bp) { ptr++; n--; }
-			while(n--) { *hp++ = *ptr++; }
-
-			if ((hval | bp) == 0xffff) {
-			    *nhp = make_arityval(16);
-			}
-			res = make_hashmap(nhp);
-			break;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			slot  = (Uint)   ESTACK_POP(stack);
-			bp    = (Uint32) ESTACK_POP(stack);
-			n     = (Uint32) ESTACK_POP(stack);
-			hval  = MAP_HEADER_VAL(hdr);
-			nhp   = hp;
-			*hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++;
-			*hp++ = (*ptr++) + update_size;
-
-			n -= slot;
-			while(slot--) { *hp++ = *ptr++; }
-			*hp++ = res;
-			if (hval & bp) { ptr++; n--; }
-			while(n--) { *hp++ = *ptr++; }
-
-			if ((hval | bp) == 0xffff) {
-			    *nhp = MAP_HEADER_HAMT_HEAD_ARRAY;
-			}
-			res = make_hashmap(nhp);
-			break;
-		    default:
-			erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
-			break;
-		}
-		break;
-	    default:
-		erl_exit(1, "bad primary tag %x\r\n", primary_tag(node));
-		break;
-	}
-
-    } while(!ESTACK_ISEMPTY(stack));
-
-bail_out:
-    DESTROY_ESTACK(stack);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
-    ERTS_HOLE_CHECK(p);
-    return res;
-}
-
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
     Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
     Eterm th[2];
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 304b5470f5..f99a1b431b 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -64,6 +64,7 @@
  */
 
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
+static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -594,6 +595,13 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
     if (is_map(BIF_ARG_3)) {
 	BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3));
+    } else if (is_hashmap(BIF_ARG_3)) {
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
+	Eterm map;
+
+	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0);
+	ASSERT(is_hashmap(map));
+	BIF_RET(map);
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -748,6 +756,13 @@ BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
 	if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) {
 	    BIF_RET(res);
 	}
+    } else if (is_hashmap(BIF_ARG_3)) {
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
+	Eterm map;
+
+	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 1);
+	if (is_value(map))
+	    BIF_RET(map);
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -920,6 +935,238 @@ static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     return NULL;
 }
 
+static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
+			    Eterm node, int is_update) {
+    Eterm *hp = NULL, *nhp = NULL;
+    Eterm *ptr;
+    Eterm hdr,res,ckey,fake;
+    Eterm th[2];
+    Uint32 ix, cix, bp, hval, chx;
+    Uint slot, lvl = 0, clvl;
+    Uint size = 0, n = 0, update_size = 1;
+    DECLARE_ESTACK(stack);
+
+    for (;;) {
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
+		ptr  = list_val(node);
+		ckey = CAR(ptr);
+		if (EQ(ckey, key)) {
+		    update_size = 0;
+		    goto unroll;
+		}
+		if (is_update) {
+		    res = THE_NON_VALUE;
+		    goto bail_out;
+		}
+		goto insert_subnodes;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
+			size += HAMT_NODE_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+1];
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
+			size += HAMT_HEAD_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+2];
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
+			    node  = ptr[slot+1];
+			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
+			    size += HAMT_NODE_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			if (is_update) {
+			    res = THE_NON_VALUE;
+			    goto bail_out;
+			}
+			size += HAMT_NODE_BITMAP_SZ(n+1);
+			goto unroll;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
+			    node  = ptr[slot+2];
+			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18);
+			    size += HAMT_HEAD_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			if (is_update) {
+			    res = THE_NON_VALUE;
+			    goto bail_out;
+			}
+			size += HAMT_HEAD_BITMAP_SZ(n+1);
+			goto unroll;
+		    default:
+			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %p\r\n", node);
+		break;
+	}
+    }
+insert_subnodes:
+    clvl  = lvl;
+    chx   = hashmap_restore_hash(th,clvl,ckey);
+    size += HAMT_NODE_BITMAP_SZ(2);
+    ix    = hashmap_index(hx);
+    cix   = hashmap_index(chx);
+
+    while (cix == ix) {
+	ESTACK_PUSH(stack, 0);
+	ESTACK_PUSH3(stack, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
+	size += HAMT_NODE_BITMAP_SZ(1);
+	hx    = hashmap_shift_hash(th,hx,lvl,key);
+	chx   = hashmap_shift_hash(th,chx,clvl,ckey);
+	ix    = hashmap_index(hx);
+	cix   = hashmap_index(chx);
+    }
+    ESTACK_PUSH3(stack, cix, ix, node);
+
+unroll:
+    size += 2;
+    hp  = HAlloc(p, size);
+    res = CONS(hp, key, value); hp += 2;
+
+    do {
+	node = ESTACK_POP(stack);
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		ix   = (Uint32) ESTACK_POP(stack);
+		cix  = (Uint32) ESTACK_POP(stack);
+
+		nhp   = hp;
+		*hp++ = MAP_HEADER_HAMT_NODE_BITMAP((1 << ix) | (1 << cix));
+		if (ix < cix) {
+		    *hp++ = res;
+		    *hp++ = node;
+		} else {
+		    *hp++ = node;
+		    *hp++ = res;
+		}
+		res = make_hashmap(nhp);
+		break;
+	    case TAG_PRIMARY_HEADER:
+		/* subnodes, fake it */
+		fake = node;
+		node = make_boxed(&fake);
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			slot  = (Uint)   ESTACK_POP(stack);
+			nhp   = hp;
+			n     = HAMT_NODE_ARRAY_SZ;
+			while(n--) { *hp++ = *ptr++; }
+			nhp[slot+1] = res;
+			res = make_hashmap(nhp);
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			slot  = (Uint)   ESTACK_POP(stack);
+			nhp   = hp;
+			n     = HAMT_HEAD_ARRAY_SZ - 2;
+			*hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++;
+			*hp++ = (*ptr++) + update_size;
+			while(n--) { *hp++ = *ptr++; }
+			nhp[slot+2] = res;
+			res = make_hashmap(nhp);
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			slot  = (Uint)   ESTACK_POP(stack);
+			bp    = (Uint32) ESTACK_POP(stack);
+			n     = (Uint32) ESTACK_POP(stack);
+			hval  = MAP_HEADER_VAL(hdr);
+			nhp   = hp;
+			*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval | bp); ptr++;
+
+			n -= slot;
+			while(slot--) { *hp++ = *ptr++; }
+			*hp++ = res;
+			if (hval & bp) { ptr++; n--; }
+			while(n--) { *hp++ = *ptr++; }
+
+			if ((hval | bp) == 0xffff) {
+			    *nhp = make_arityval(16);
+			}
+			res = make_hashmap(nhp);
+			break;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			slot  = (Uint)   ESTACK_POP(stack);
+			bp    = (Uint32) ESTACK_POP(stack);
+			n     = (Uint32) ESTACK_POP(stack);
+			hval  = MAP_HEADER_VAL(hdr);
+			nhp   = hp;
+			*hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++;
+			*hp++ = (*ptr++) + update_size;
+
+			n -= slot;
+			while(slot--) { *hp++ = *ptr++; }
+			*hp++ = res;
+			if (hval & bp) { ptr++; n--; }
+			while(n--) { *hp++ = *ptr++; }
+
+			if ((hval | bp) == 0xffff) {
+			    *nhp = MAP_HEADER_HAMT_HEAD_ARRAY;
+			}
+			res = make_hashmap(nhp);
+			break;
+		    default:
+			erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %x\r\n", primary_tag(node));
+		break;
+	}
+
+    } while(!ESTACK_ISEMPTY(stack));
+
+bail_out:
+    DESTROY_ESTACK(stack);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+
 static Eterm hashmap_keys(Process* p, Eterm node) {
     DECLARE_WSTACK(stack);
     hashmap_head_t* root;
-- 
cgit v1.2.3


From 8c9341288af6b66f0fd5caf229c87af29c59e711 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 17:58:52 +0100
Subject: erts: Reindent erts_maps_update

---
 erts/emulator/beam/erl_map.c | 78 ++++++++++++++++++++++----------------------
 1 file changed, 39 insertions(+), 39 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index f99a1b431b..3e1092bfbe 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -698,56 +698,56 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
  */
 
 int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
-	Sint n,i;
-	Eterm* hp,*shp;
-	Eterm *ks,*vs;
-	map_t *mp = (map_t*)map_val(map);
+    Sint n,i;
+    Eterm* hp,*shp;
+    Eterm *ks,*vs;
+    map_t *mp = (map_t*)map_val(map);
 
-	if ((n = map_get_size(mp)) == 0) {
-	    return 0;
-	}
+    if ((n = map_get_size(mp)) == 0) {
+	return 0;
+    }
 
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
+    ks  = map_get_keys(mp);
+    vs  = map_get_values(mp);
 
-	/* only allocate for values,
-	 * assume key-tuple will be intact
-	 */
+    /* only allocate for values,
+     * assume key-tuple will be intact
+     */
 
-	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
-	shp = hp;
-	*hp++ = MAP_HEADER;
-	*hp++ = n;
-	*hp++ = mp->keys;
+    hp  = HAlloc(p, MAP_HEADER_SIZE + n);
+    shp = hp;
+    *hp++ = MAP_HEADER;
+    *hp++ = n;
+    *hp++ = mp->keys;
 
-	if (is_immed(key)) {
-	    for( i = 0; i < n; i ++) {
-		if (ks[i] == key) {
-		    goto found_key;
-		} else {
-		    *hp++ = *vs++;
-		}
+    if (is_immed(key)) {
+	for( i = 0; i < n; i ++) {
+	    if (ks[i] == key) {
+		goto found_key;
+	    } else {
+		*hp++ = *vs++;
 	    }
-	} else {
-	    for( i = 0; i < n; i ++) {
-		if (EQ(ks[i], key)) {
-		    goto found_key;
-		} else {
-		    *hp++ = *vs++;
-		}
+	}
+    } else {
+	for( i = 0; i < n; i ++) {
+	    if (EQ(ks[i], key)) {
+		goto found_key;
+	    } else {
+		*hp++ = *vs++;
 	    }
 	}
+    }
 
-	HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
-	return 0;
+    HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
+    return 0;
 
 found_key:
-	*hp++ = value;
-	vs++;
-	if (++i < n)
-	    sys_memcpy(hp, vs, (n - i)*sizeof(Eterm));
-	*res = make_map(shp);
-	return 1;
+    *hp++ = value;
+    vs++;
+    if (++i < n)
+	sys_memcpy(hp, vs, (n - i)*sizeof(Eterm));
+    *res = make_map(shp);
+    return 1;
 }
 
 BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
-- 
cgit v1.2.3


From 1d2c8a4280e0eb5d7182986862f20045bc3ed6f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 24 Feb 2015 14:17:51 +0100
Subject: erts: Refactor maps:update/3 and maps:put/3

---
 erts/emulator/beam/erl_map.c | 368 ++++++++++++++++++++++++-------------------
 1 file changed, 206 insertions(+), 162 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 3e1092bfbe..8c44f45125 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -486,122 +486,11 @@ BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
     BIF_RET(make_map(mp));
 }
 
-/* maps:put/3
- */
-
-Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
-    Sint n,i;
-    Sint c = 0;
-    Eterm* hp, *shp;
-    Eterm *ks,*vs, res, tup;
-    map_t *mp = (map_t*)map_val(map);
-
-    n = map_get_size(mp);
-
-    if (n == 0) {
-	hp    = HAlloc(p, MAP_HEADER_SIZE + 1 + 2);
-	tup   = make_tuple(hp);
-	*hp++ = make_arityval(1);
-	*hp++ = key;
-	res   = make_map(hp);
-	*hp++ = MAP_HEADER;
-	*hp++ = 1;
-	*hp++ = tup;
-	*hp++ = value;
-
-	return res;
-    }
-
-    ks  = map_get_keys(mp);
-    vs  = map_get_values(mp);
-
-    /* only allocate for values,
-     * assume key-tuple will be intact
-     */
-
-    hp  = HAlloc(p, MAP_HEADER_SIZE + n);
-    shp = hp; /* save hp, used if optimistic update fails */
-    res = make_map(hp);
-    *hp++ = MAP_HEADER;
-    *hp++ = n;
-    *hp++ = mp->keys;
-
-    if (is_immed(key)) {
-	for( i = 0; i < n; i ++) {
-	    if (ks[i] == key) {
-		*hp++ = value;
-		vs++;
-		c = 1;
-	    } else {
-		*hp++ = *vs++;
-	    }
-	}
-    } else {
-	for( i = 0; i < n; i ++) {
-	    if (EQ(ks[i], key)) {
-		*hp++ = value;
-		vs++;
-		c = 1;
-	    } else {
-		*hp++ = *vs++;
-	    }
-	}
-    }
-
-    if (c)
-	return res;
-
-    /* need to make a new tuple,
-     * use old hp since it needs to be recreated anyway.
-     */
-    tup    = make_tuple(shp);
-    *shp++ = make_arityval(n+1);
-
-    hp    = HAlloc(p, 3 + n + 1);
-    res   = make_map(hp);
-    *hp++ = MAP_HEADER;
-    *hp++ = n + 1;
-    *hp++ = tup;
-
-    ks  = map_get_keys(mp);
-    vs  = map_get_values(mp);
-
-    ASSERT(n >= 0);
-
-    /* copy map in order */
-    while (n && ((c = CMP_TERM(*ks, key)) < 0)) {
-	*shp++ = *ks++;
-	*hp++  = *vs++;
-	n--;
-    }
-
-    *shp++ = key;
-    *hp++  = value;
-
-    ASSERT(n >= 0);
-
-    while(n--) {
-	*shp++ = *ks++;
-	*hp++  = *vs++;
-    }
-    /* we have one word remaining
-     * this will work out fine once we get the size word
-     * in the header.
-     */
-    *shp = make_pos_bignum_header(0);
-    return res;
-}
+/* maps:put/3 */
 
 BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
-    if (is_map(BIF_ARG_3)) {
+    if (is_map(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
 	BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3));
-    } else if (is_hashmap(BIF_ARG_3)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-	Eterm map;
-
-	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0);
-	ASSERT(is_hashmap(map));
-	BIF_RET(map);
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -694,82 +583,237 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* maps:update/3
- */
-
 int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
-    Sint n,i;
-    Eterm* hp,*shp;
-    Eterm *ks,*vs;
-    map_t *mp = (map_t*)map_val(map);
+    Uint32 hx;
+    if (is_map(map)) {
+	Sint n,i;
+	Eterm* hp,*shp;
+	Eterm *ks,*vs;
+	map_t *mp = (map_t*)map_val(map);
+
+	if ((n = map_get_size(mp)) == 0) {
+	    return 0;
+	}
+
+	ks  = map_get_keys(mp);
+	vs  = map_get_values(mp);
+
+	/* only allocate for values,
+	 * assume key-tuple will be intact
+	 */
+
+	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
+	shp = hp;
+	*hp++ = MAP_HEADER;
+	*hp++ = n;
+	*hp++ = mp->keys;
+
+	if (is_immed(key)) {
+	    for( i = 0; i < n; i ++) {
+		if (ks[i] == key) {
+		    goto found_key;
+		} else {
+		    *hp++ = *vs++;
+		}
+	    }
+	} else {
+	    for( i = 0; i < n; i ++) {
+		if (EQ(ks[i], key)) {
+		    goto found_key;
+		} else {
+		    *hp++ = *vs++;
+		}
+	    }
+	}
 
-    if ((n = map_get_size(mp)) == 0) {
+	HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
 	return 0;
+
+found_key:
+	*hp++ = value;
+	vs++;
+	if (++i < n)
+	    sys_memcpy(hp, vs, (n - i)*sizeof(Eterm));
+	*res = make_map(shp);
+	return 1;
     }
 
-    ks  = map_get_keys(mp);
-    vs  = map_get_values(mp);
+    ASSERT(is_hashmap(map));
+    hx = hashmap_make_hash(key);
+    *res = hashmap_insert(p, hx, key, value, map, 1);
+    if (is_value(*res))
+	return 1;
 
-    /* only allocate for values,
-     * assume key-tuple will be intact
-     */
+    return 0;
+}
 
-    hp  = HAlloc(p, MAP_HEADER_SIZE + n);
-    shp = hp;
-    *hp++ = MAP_HEADER;
-    *hp++ = n;
-    *hp++ = mp->keys;
+Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
+    Uint32 hx;
+    Eterm res;
+    if (is_map(map)) {
+	Sint n,i;
+	Sint c = 0;
+	Eterm* hp, *shp;
+	Eterm *ks, *vs, tup;
+	map_t *mp = (map_t*)map_val(map);
 
-    if (is_immed(key)) {
-	for( i = 0; i < n; i ++) {
-	    if (ks[i] == key) {
-		goto found_key;
-	    } else {
-		*hp++ = *vs++;
+	n = map_get_size(mp);
+
+	if (n == 0) {
+	    hp    = HAlloc(p, MAP_HEADER_SIZE + 1 + 2);
+	    tup   = make_tuple(hp);
+	    *hp++ = make_arityval(1);
+	    *hp++ = key;
+	    res   = make_map(hp);
+	    *hp++ = MAP_HEADER;
+	    *hp++ = 1;
+	    *hp++ = tup;
+	    *hp++ = value;
+
+	    return res;
+	}
+
+	ks = map_get_keys(mp);
+	vs = map_get_values(mp);
+
+	/* only allocate for values,
+	 * assume key-tuple will be intact
+	 */
+
+	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
+	shp = hp; /* save hp, used if optimistic update fails */
+	res = make_map(hp);
+	*hp++ = MAP_HEADER;
+	*hp++ = n;
+	*hp++ = mp->keys;
+
+	if (is_immed(key)) {
+	    for( i = 0; i < n; i ++) {
+		if (ks[i] == key) {
+		    *hp++ = value;
+		    vs++;
+		    c = 1;
+		} else {
+		    *hp++ = *vs++;
+		}
+	    }
+	} else {
+	    for( i = 0; i < n; i ++) {
+		if (EQ(ks[i], key)) {
+		    *hp++ = value;
+		    vs++;
+		    c = 1;
+		} else {
+		    *hp++ = *vs++;
+		}
 	    }
 	}
-    } else {
-	for( i = 0; i < n; i ++) {
-	    if (EQ(ks[i], key)) {
-		goto found_key;
-	    } else {
-		*hp++ = *vs++;
+
+	if (c)
+	    return res;
+
+	/* the map will grow */
+
+	if (n >= MAP_SMALL_MAP_LIMIT) {
+	    hxnode_t *hxns;
+	    Uint32 sw, hx;
+	    Eterm tmp[2];
+
+	    HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
+	    hp = HAlloc(p, (2 * (n + 1)));
+	    ks = map_get_keys(mp);
+	    vs = map_get_values(mp);
+
+	    /* create tmp hx values and leaf ptrs */
+	    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(hxnode_t));
+
+	    for (i = 0; i < n; i++) {
+		hx = hashmap_restore_hash(tmp,0,ks[i]);
+		swizzle32(sw,hx);
+		hxns[i].hx   = sw;
+		hxns[i].val  = CONS(hp, ks[i], vs[i]); hp += 2;
+		hxns[i].skip = 1;
+		hxns[i].i    = i;
 	    }
+
+	    hx = hashmap_restore_hash(tmp,0,key);
+	    swizzle32(sw,hx);
+	    hxns[i].hx   = sw;
+	    hxns[i].val  = CONS(hp, key, value); hp += 2;
+	    hxns[i].skip = 1;
+	    hxns[i].i    = n;
+
+	    res = hashmap_from_unsorted_array(p, hxns, n + 1);
+
+	    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+	    return res;
+	}
+
+	/* still a small map. need to make a new tuple,
+	 * use old hp since it needs to be recreated anyway. */
+
+	tup    = make_tuple(shp);
+	*shp++ = make_arityval(n+1);
+
+	hp    = HAlloc(p, 3 + n + 1);
+	res   = make_map(hp);
+	*hp++ = MAP_HEADER;
+	*hp++ = n + 1;
+	*hp++ = tup;
+
+	ks = map_get_keys(mp);
+	vs = map_get_values(mp);
+
+	ASSERT(n >= 0);
+
+	/* copy map in order */
+	while (n && ((c = CMP_TERM(*ks, key)) < 0)) {
+	    *shp++ = *ks++;
+	    *hp++  = *vs++;
+	    n--;
 	}
+
+	*shp++ = key;
+	*hp++  = value;
+
+	ASSERT(n >= 0);
+
+	while(n--) {
+	    *shp++ = *ks++;
+	    *hp++  = *vs++;
+	}
+	/* we have one word remaining
+	 * this will work out fine once we get the size word
+	 * in the header.
+	 */
+	*shp = make_pos_bignum_header(0);
+	return res;
     }
+    ASSERT(is_hashmap(map));
 
-    HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
-    return 0;
+    hx  = hashmap_make_hash(key);
+    res = hashmap_insert(p, hx, key, value, map, 0);
+    ASSERT(is_hashmap(res));
 
-found_key:
-    *hp++ = value;
-    vs++;
-    if (++i < n)
-	sys_memcpy(hp, vs, (n - i)*sizeof(Eterm));
-    *res = make_map(shp);
-    return 1;
+    return res;
 }
 
+/* maps:update/3 */
+
 BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
-    if (is_map(BIF_ARG_3)) {
+    if (is_map(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
 	Eterm res;
 	if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) {
 	    BIF_RET(res);
 	}
-    } else if (is_hashmap(BIF_ARG_3)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-	Eterm map;
-
-	map = hashmap_insert(BIF_P, hx, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 1);
-	if (is_value(map))
-	    BIF_RET(map);
     }
     BIF_ERROR(BIF_P, BADARG);
 }
 
 
-/* maps:values/1
- */
+/* maps:values/1 */
 
 BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
-- 
cgit v1.2.3


From 1b963c425591b75410fe46d9af3b44e47692059e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 15:49:43 +0100
Subject: erts: Move hashmap:remove/2 to maps

---
 erts/emulator/beam/bif.tab       |   1 -
 erts/emulator/beam/erl_hashmap.c | 258 ---------------------------------------
 erts/emulator/beam/erl_map.c     | 257 +++++++++++++++++++++++++++++++++++++-
 3 files changed, 255 insertions(+), 261 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 55a7b62e7d..320cd39da8 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -615,7 +615,6 @@ bif erlang:fun_info_mfa/1
 bif erlang:get_keys/0
 
 # Hash Array Mappped Trie
-bif hashmap:remove/2
 bif hashmap:info/1
 bif hashmap:from_list/1
 bif hashmap:new/0
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 594a404e9f..9b16a5a88f 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -69,7 +69,6 @@ typedef struct {
     Eterm  val;
 } hxnode_t;
 
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_from_list(Process *p, Eterm node);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
@@ -115,14 +114,6 @@ BIF_RETTYPE hashmap_from_list_1(BIF_ALIST_1) {
 
 /* hashmap:remove/2 */
 
-BIF_RETTYPE hashmap_remove_2(BIF_ALIST_2) {
-    if (is_hashmap(BIF_ARG_2)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-
-	BIF_RET(hashmap_delete(BIF_P, hx, BIF_ARG_1, BIF_ARG_2));
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
 /* hashmap:size/1 */
 
 /* erlang:is_hashmap/1 */
@@ -143,255 +134,6 @@ BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
 /* impl. */
 
 
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
-    Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
-    Eterm th[2];
-    Eterm *ptr;
-    Eterm hdr,res = node;
-    Uint32 ix, bp, hval;
-    Uint slot, lvl = 0;
-    Uint size = 0, n = 0;
-    DECLARE_ESTACK(stack);
-
-    for (;;) {
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST:
-		if (EQ(CAR(list_val(node)), key)) {
-		    goto unroll;
-		}
-		goto not_found;
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(th,hx,lvl,key);
-			size += HAMT_NODE_ARRAY_SZ;
-			ESTACK_PUSH2(stack, ix, node);
-			node  = ptr[ix+1];
-			break;
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(th,hx,lvl,key);
-			size += HAMT_HEAD_ARRAY_SZ;
-			ESTACK_PUSH2(stack, ix, node);
-			node  = ptr[ix+2];
-			break;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-			n    = hashmap_bitcount(hval);
-
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+1];
-			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
-			    size += HAMT_NODE_BITMAP_SZ(n);
-			    break;
-			}
-			/* not occupied */
-			goto not_found;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-			n    = hashmap_bitcount(hval);
-
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+2];
-			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18);
-			    size += HAMT_HEAD_BITMAP_SZ(n);
-			    break;
-			}
-			/* not occupied */
-			goto not_found;
-		    default:
-			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
-			break;
-		}
-		break;
-	    default:
-		erl_exit(1, "bad primary tag %p\r\n", node);
-		break;
-	}
-    }
-
-unroll:
-    /* the size is bounded and atleast one less than the previous size */
-    size  -= 1;
-    hp     = HAlloc(p, size);
-    hp_end = hp + size;
-    res    = THE_NON_VALUE;
-
-    do {
-	node = ESTACK_POP(stack);
-
-	/* all nodes are things */
-	ptr = boxed_val(node);
-	hdr = *ptr;
-	ASSERT(is_header(hdr));
-
-	switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_NODE_ARRAY:
-		ix  = (Uint) ESTACK_POP(stack);
-		nhp = hp;
-		if (res == THE_NON_VALUE) {
-		    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(0xffff ^ (1 << ix)); ptr++;
-		    n     = 16;
-		    n    -= ix;
-		    while(ix--) { *hp++ = *ptr++; }
-		    ptr++; n--;
-		    while(n--) { *hp++ = *ptr++; }
-		    res = make_hashmap(nhp);
-		} else {
-		    n = HAMT_NODE_ARRAY_SZ;
-		    while(n--) { *hp++ = *ptr++; }
-		    nhp[ix+1] = res;
-		    res = make_hashmap(nhp);
-		}
-		break;
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ix  = (Uint) ESTACK_POP(stack);
-		nhp = hp;
-		if (res == THE_NON_VALUE) {
-		    n     = 16;
-		    n    -= ix;
-		    *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(0xffff ^ (1 << ix)); ptr++;
-		    *hp++ = (*ptr++) - 1;
-		    while(ix--) { *hp++ = *ptr++; }
-		    ptr++; n--;
-		    while(n--) { *hp++ = *ptr++; }
-		    res = make_hashmap(nhp);
-		} else {
-		    n     = 16;
-		    *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++;
-		    *hp++ = (*ptr++) - 1;
-		    while(n--) { *hp++ = *ptr++; }
-		    nhp[ix+2] = res;
-		    res = make_hashmap(nhp);
-		}
-		break;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		slot = (Uint)   ESTACK_POP(stack);
-		bp   = (Uint32) ESTACK_POP(stack);
-		n    = (Uint32) ESTACK_POP(stack);
-		nhp  = hp;
-
-		/* bitmap change matrix
-		 * res | none    leaf    bitmap
-		 * ----------------------------
-		 * n=1 | remove  remove  keep
-		 * n=2 | other   keep    keep
-		 * n>2 | shrink  keep    keep
-		 *
-		 * other: (remember, n is 2)
-		 *   shrink if the other bitmap value is a bitmap node
-		 *   remove if the other bitmap value is a leaf
-		 *
-		 * remove:
-		 *   this bitmap node is removed, res is moved up in tree (could be none)
-		 *   this is a special case of shrink
-		 *
-		 * keep:
-		 *   the current path index is still used down in the tree, need to keep it
-		 *   copy as usual with the updated res
-		 *
-		 * shrink:
-		 *   the current path index is no longer used down in the tree, remove it (shrink)
-		 */
-		if (res == THE_NON_VALUE) {
-		    if (n == 1) {
-			break;
-		    } else if (n == 2) {
-			if (slot == 0) {
-			    ix = 2; /* off by one 'cause hdr */
-			} else {
-			    ix = 1; /* off by one 'cause hdr */
-			}
-			if (primary_tag(ptr[ix]) == TAG_PRIMARY_LIST) {
-			    res = ptr[ix];
-			} else {
-			    hval  = MAP_HEADER_VAL(hdr);
-			    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp);
-			    *hp++ = ptr[ix];
-			    res = make_hashmap(nhp);
-			}
-		    } else {
-			/* n > 2 */
-			hval  = MAP_HEADER_VAL(hdr);
-			*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); ptr++;
-			n    -= slot;
-			while(slot--) { *hp++ = *ptr++; }
-			ptr++; n--;
-			while(n--) { *hp++ = *ptr++; }
-			res = make_hashmap(nhp);
-		    }
-		} else if (primary_tag(res) == TAG_PRIMARY_LIST && n == 1) {
-		    break;
-		} else {
-		    /* res is bitmap or leaf && n > 1, keep */
-		    n    -= slot;
-		    *hp++ = *ptr++;
-		    while(slot--) { *hp++ = *ptr++; }
-		    *hp++ = res;
-		    ptr++; n--;
-		    while(n--) { *hp++ = *ptr++; }
-		    res = make_hashmap(nhp);
-		}
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		slot = (Uint)   ESTACK_POP(stack);
-		bp   = (Uint32) ESTACK_POP(stack);
-		n    = (Uint32) ESTACK_POP(stack);
-		nhp  = hp;
-
-		if (res != THE_NON_VALUE) {
-		    *hp++ = *ptr++;
-		    *hp++ = (*ptr++) - 1;
-		    n    -= slot;
-		    while(slot--) { *hp++ = *ptr++; }
-		    *hp++ = res;
-		    ptr++; n--;
-		    while(n--) { *hp++ = *ptr++; }
-		} else {
-		    hval  = MAP_HEADER_VAL(hdr);
-		    *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval ^ bp); ptr++;
-		    *hp++ = (*ptr++) - 1;
-		    n    -= slot;
-		    while(slot--) { *hp++ = *ptr++; }
-		    ptr++; n--;
-		    while(n--) { *hp++ = *ptr++; }
-		}
-		res = make_hashmap(nhp);
-		break;
-	    default:
-		erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
-		break;
-	}
-    } while(!ESTACK_ISEMPTY(stack));
-    HRelease(p, hp_end, hp);
-not_found:
-    DESTROY_ESTACK(stack);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
-    ERTS_HOLE_CHECK(p);
-    return res;
-}
-
 #define swizzle32(D,S) \
     do { \
 	(D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20  \
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 8c44f45125..672ac9f513 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -68,6 +68,7 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 
 /* erlang:map_size/1
  * the corresponding instruction is implemented in:
@@ -495,8 +496,7 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* maps:remove/3
- */
+/* maps:remove/3 */
 
 int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     Sint n;
@@ -579,6 +579,9 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
 	if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) {
 	    BIF_RET(res);
 	}
+    } else if (is_hashmap(BIF_ARG_2)) {
+	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
+	BIF_RET(hashmap_delete(BIF_P, hx, BIF_ARG_1, BIF_ARG_2));
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -1245,6 +1248,256 @@ static Eterm hashmap_values(Process* p, Eterm node) {
     return res;
 }
 
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
+    Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
+    Eterm th[2];
+    Eterm *ptr;
+    Eterm hdr,res = node;
+    Uint32 ix, bp, hval;
+    Uint slot, lvl = 0;
+    Uint size = 0, n = 0;
+    DECLARE_ESTACK(stack);
+
+    for (;;) {
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		if (EQ(CAR(list_val(node)), key)) {
+		    goto unroll;
+		}
+		goto not_found;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
+			size += HAMT_NODE_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+1];
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			ix    = hashmap_index(hx);
+			hx    = hashmap_shift_hash(th,hx,lvl,key);
+			size += HAMT_HEAD_ARRAY_SZ;
+			ESTACK_PUSH2(stack, ix, node);
+			node  = ptr[ix+2];
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
+			    node  = ptr[slot+1];
+			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
+			    size += HAMT_NODE_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			goto not_found;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			hval = MAP_HEADER_VAL(hdr);
+			ix   = hashmap_index(hx);
+			bp   = 1 << ix;
+			slot = hashmap_bitcount(hval & (bp - 1));
+			n    = hashmap_bitcount(hval);
+
+			ESTACK_PUSH(stack, n);
+			ESTACK_PUSH3(stack, bp, slot, node);
+
+			/* occupied */
+			if (bp & hval) {
+			    hx    = hashmap_shift_hash(th,hx,lvl,key);
+			    node  = ptr[slot+2];
+			    ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18);
+			    size += HAMT_HEAD_BITMAP_SZ(n);
+			    break;
+			}
+			/* not occupied */
+			goto not_found;
+		    default:
+			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			break;
+		}
+		break;
+	    default:
+		erl_exit(1, "bad primary tag %p\r\n", node);
+		break;
+	}
+    }
+
+unroll:
+    /* the size is bounded and atleast one less than the previous size */
+    size  -= 1;
+    hp     = HAlloc(p, size);
+    hp_end = hp + size;
+    res    = THE_NON_VALUE;
+
+    do {
+	node = ESTACK_POP(stack);
+
+	/* all nodes are things */
+	ptr = boxed_val(node);
+	hdr = *ptr;
+	ASSERT(is_header(hdr));
+
+	switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		ix  = (Uint) ESTACK_POP(stack);
+		nhp = hp;
+		if (res == THE_NON_VALUE) {
+		    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(0xffff ^ (1 << ix)); ptr++;
+		    n     = 16;
+		    n    -= ix;
+		    while(ix--) { *hp++ = *ptr++; }
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		    res = make_hashmap(nhp);
+		} else {
+		    n = HAMT_NODE_ARRAY_SZ;
+		    while(n--) { *hp++ = *ptr++; }
+		    nhp[ix+1] = res;
+		    res = make_hashmap(nhp);
+		}
+		break;
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ix  = (Uint) ESTACK_POP(stack);
+		nhp = hp;
+		if (res == THE_NON_VALUE) {
+		    n     = 16;
+		    n    -= ix;
+		    *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(0xffff ^ (1 << ix)); ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    while(ix--) { *hp++ = *ptr++; }
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		    res = make_hashmap(nhp);
+		} else {
+		    n     = 16;
+		    *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    while(n--) { *hp++ = *ptr++; }
+		    nhp[ix+2] = res;
+		    res = make_hashmap(nhp);
+		}
+		break;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		slot = (Uint)   ESTACK_POP(stack);
+		bp   = (Uint32) ESTACK_POP(stack);
+		n    = (Uint32) ESTACK_POP(stack);
+		nhp  = hp;
+
+		/* bitmap change matrix
+		 * res | none    leaf    bitmap
+		 * ----------------------------
+		 * n=1 | remove  remove  keep
+		 * n=2 | other   keep    keep
+		 * n>2 | shrink  keep    keep
+		 *
+		 * other: (remember, n is 2)
+		 *   shrink if the other bitmap value is a bitmap node
+		 *   remove if the other bitmap value is a leaf
+		 *
+		 * remove:
+		 *   this bitmap node is removed, res is moved up in tree (could be none)
+		 *   this is a special case of shrink
+		 *
+		 * keep:
+		 *   the current path index is still used down in the tree, need to keep it
+		 *   copy as usual with the updated res
+		 *
+		 * shrink:
+		 *   the current path index is no longer used down in the tree, remove it (shrink)
+		 */
+		if (res == THE_NON_VALUE) {
+		    if (n == 1) {
+			break;
+		    } else if (n == 2) {
+			if (slot == 0) {
+			    ix = 2; /* off by one 'cause hdr */
+			} else {
+			    ix = 1; /* off by one 'cause hdr */
+			}
+			if (primary_tag(ptr[ix]) == TAG_PRIMARY_LIST) {
+			    res = ptr[ix];
+			} else {
+			    hval  = MAP_HEADER_VAL(hdr);
+			    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp);
+			    *hp++ = ptr[ix];
+			    res = make_hashmap(nhp);
+			}
+		    } else {
+			/* n > 2 */
+			hval  = MAP_HEADER_VAL(hdr);
+			*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); ptr++;
+			n    -= slot;
+			while(slot--) { *hp++ = *ptr++; }
+			ptr++; n--;
+			while(n--) { *hp++ = *ptr++; }
+			res = make_hashmap(nhp);
+		    }
+		} else if (primary_tag(res) == TAG_PRIMARY_LIST && n == 1) {
+		    break;
+		} else {
+		    /* res is bitmap or leaf && n > 1, keep */
+		    n    -= slot;
+		    *hp++ = *ptr++;
+		    while(slot--) { *hp++ = *ptr++; }
+		    *hp++ = res;
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		    res = make_hashmap(nhp);
+		}
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		slot = (Uint)   ESTACK_POP(stack);
+		bp   = (Uint32) ESTACK_POP(stack);
+		n    = (Uint32) ESTACK_POP(stack);
+		nhp  = hp;
+
+		if (res != THE_NON_VALUE) {
+		    *hp++ = *ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    n    -= slot;
+		    while(slot--) { *hp++ = *ptr++; }
+		    *hp++ = res;
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		} else {
+		    hval  = MAP_HEADER_VAL(hdr);
+		    *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval ^ bp); ptr++;
+		    *hp++ = (*ptr++) - 1;
+		    n    -= slot;
+		    while(slot--) { *hp++ = *ptr++; }
+		    ptr++; n--;
+		    while(n--) { *hp++ = *ptr++; }
+		}
+		res = make_hashmap(nhp);
+		break;
+	    default:
+		erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+		break;
+	}
+    } while(!ESTACK_ISEMPTY(stack));
+    HRelease(p, hp_end, hp);
+not_found:
+    DESTROY_ESTACK(stack);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+
+
 int erts_validate_and_sort_map(map_t* mp)
 {
     Eterm *ks  = map_get_keys(mp);
-- 
cgit v1.2.3


From bb05557ff27e16e1e079e8f99b68713c7ca012c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 24 Feb 2015 16:44:51 +0100
Subject: erts: Refactor maps:remove/2

---
 erts/emulator/beam/erl_map.c | 182 +++++++++++++++++++++++++++----------------
 1 file changed, 113 insertions(+), 69 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 672ac9f513..69dd91d079 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -499,89 +499,93 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
 /* maps:remove/3 */
 
 int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
-    Sint n;
-    Uint need;
-    Eterm *hp_start;
-    Eterm *thp, *mhp;
-    Eterm *ks, *vs, tup;
-    map_t *mp = (map_t*)map_val(map);
-
-    n = map_get_size(mp);
+    Uint32 hx;
+    if (is_map(map)) {
+	Sint n;
+	Uint need;
+	Eterm *hp_start;
+	Eterm *thp, *mhp;
+	Eterm *ks, *vs, tup;
+	map_t *mp = (map_t*)map_val(map);
 
-    if (n == 0) {
-	*res = map;
-	return 1;
-    }
+	n = map_get_size(mp);
 
-    ks = map_get_keys(mp);
-    vs = map_get_values(mp);
-
-    /* Assume key exists.
-     * Release allocated if it didn't.
-     * Allocate key tuple first.
-     */
-
-    need   = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */
-    hp_start = HAlloc(p, need);
-    thp    = hp_start;
-    mhp    = thp + n;               /* offset with tuple heap size */
-
-    tup    = make_tuple(thp);
-    *thp++ = make_arityval(n - 1);
-
-    *res   = make_map(mhp);
-    *mhp++ = MAP_HEADER;
-    *mhp++ = n - 1;
-    *mhp++ = tup;
-
-    if (is_immed(key)) {
-	while (1) {
-	    if (*ks == key) {
-		goto found_key;
-	    } else if (--n) {
-		*mhp++ = *vs++;
-		*thp++ = *ks++;
-	    } else
-		break;
+	if (n == 0) {
+	    *res = map;
+	    return 1;
 	}
-    } else {
-	while(1) {
-	    if (EQ(*ks, key)) {
-		goto found_key;
-	    } else if (--n) {
-		*mhp++ = *vs++;
-		*thp++ = *ks++;
-	    } else
-		break;
+
+	ks = map_get_keys(mp);
+	vs = map_get_values(mp);
+
+	/* Assume key exists.
+	 * Release allocated if it didn't.
+	 * Allocate key tuple first.
+	 */
+
+	need   = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */
+	hp_start = HAlloc(p, need);
+	thp    = hp_start;
+	mhp    = thp + n;               /* offset with tuple heap size */
+
+	tup    = make_tuple(thp);
+	*thp++ = make_arityval(n - 1);
+
+	*res   = make_map(mhp);
+	*mhp++ = MAP_HEADER;
+	*mhp++ = n - 1;
+	*mhp++ = tup;
+
+	if (is_immed(key)) {
+	    while (1) {
+		if (*ks == key) {
+		    goto found_key;
+		} else if (--n) {
+		    *mhp++ = *vs++;
+		    *thp++ = *ks++;
+		} else
+		    break;
+	    }
+	} else {
+	    while(1) {
+		if (EQ(*ks, key)) {
+		    goto found_key;
+		} else if (--n) {
+		    *mhp++ = *vs++;
+		    *thp++ = *ks++;
+		} else
+		    break;
+	    }
 	}
-    }
 
-    /* Not found, remove allocated memory
-     * and return previous map.
-     */
-    HRelease(p, hp_start + need, hp_start);
+	/* Not found, remove allocated memory
+	 * and return previous map.
+	 */
+	HRelease(p, hp_start + need, hp_start);
 
-    *res = map;
-    return 1;
+	*res = map;
+	return 1;
 
 found_key:
-    /* Copy rest of keys and values */
-    if (--n) {
-	sys_memcpy(mhp, vs+1, n*sizeof(Eterm));
-	sys_memcpy(thp, ks+1, n*sizeof(Eterm));
+	/* Copy rest of keys and values */
+	if (--n) {
+	    sys_memcpy(mhp, vs+1, n*sizeof(Eterm));
+	    sys_memcpy(thp, ks+1, n*sizeof(Eterm));
+	}
+	return 1;
     }
+    ASSERT(is_hashmap(map));
+    hx = hashmap_make_hash(key);
+    *res = hashmap_delete(p, hx, key, map);
     return 1;
 }
 
 BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
+    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
 	Eterm res;
 	if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) {
 	    BIF_RET(res);
 	}
-    } else if (is_hashmap(BIF_ARG_2)) {
-	Uint32 hx = hashmap_make_hash(BIF_ARG_1);
-	BIF_RET(hashmap_delete(BIF_P, hx, BIF_ARG_1, BIF_ARG_2));
     }
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -1248,11 +1252,11 @@ static Eterm hashmap_values(Process* p, Eterm node) {
     return res;
 }
 
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
     Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
     Eterm th[2];
     Eterm *ptr;
-    Eterm hdr,res = node;
+    Eterm hdr, res = map, node = map;
     Uint32 ix, bp, hval;
     Uint slot, lvl = 0;
     Uint size = 0, n = 0;
@@ -1338,7 +1342,47 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node) {
 
 unroll:
     /* the size is bounded and atleast one less than the previous size */
-    size  -= 1;
+    size -= 1;
+    n     = hashmap_size(map) - 1;
+
+    if (n <= MAP_SMALL_MAP_LIMIT) {
+	DECLARE_WSTACK(wstack);
+	Eterm *kv, *ks, *vs;
+	map_t *mp;
+	Eterm keys;
+
+	DESTROY_ESTACK(stack);
+
+	/* build flat structure */
+	hp    = HAlloc(p, 3 + 1 + (2 * n));
+	keys  = make_tuple(hp);
+	*hp++ = make_arityval(n);
+	ks    = hp;
+	hp   += n;
+	mp    = (map_t*)hp;
+	hp   += MAP_HEADER_SIZE;
+	vs    = hp;
+
+	mp->thing_word = MAP_HEADER;
+	mp->size = n;
+	mp->keys = keys;
+
+	hashmap_iterator_init(&wstack, map);
+
+	while ((kv=hashmap_iterator_next(&wstack)) != NULL) {
+	    if (EQ(CAR(kv),key))
+		continue;
+	    *ks++ = CAR(kv);
+	    *vs++ = CDR(kv);
+	}
+
+	/* it cannot have multiple keys */
+	erts_validate_and_sort_map(mp);
+
+	DESTROY_WSTACK(wstack);
+	return make_map(mp);
+    }
+
     hp     = HAlloc(p, size);
     hp_end = hp + size;
     res    = THE_NON_VALUE;
-- 
cgit v1.2.3


From 73885d5d098a58aa6a4e226d075ee90b24045c66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 16:03:03 +0100
Subject: erts: Refactor maps:from_list/1

---
 erts/emulator/beam/erl_map.c | 163 ++++++++++++++++++++++---------------------
 1 file changed, 85 insertions(+), 78 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 69dd91d079..5e5e567642 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -69,6 +69,7 @@ static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
+static Eterm map_from_validated_list(Process *p, Eterm list, Uint size);
 
 /* erlang:map_size/1
  * the corresponding instruction is implemented in:
@@ -227,13 +228,8 @@ BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
  */
 
 BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) {
-    Eterm *kv, item = BIF_ARG_1;
-    Eterm *hp, *thp,*vs, *ks, keys, res;
-    map_t *mp;
-    Uint  size = 0, unused_size = 0;
-    Sint  c = 0;
-    Sint  idx = 0;
-
+    Eterm item = BIF_ARG_1, res, *kv;
+    Uint  size = 0;
     if (is_list(item) || is_nil(item)) {
 
 	/* Calculate size and check validity */
@@ -254,95 +250,106 @@ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) {
 	if (is_not_nil(item))
 	    goto error;
 
-	hp    = HAlloc(BIF_P, 3 + 1 + (2 * size));
-	thp   = hp;
-	keys  = make_tuple(hp);
-	*hp++ = make_arityval(size);
-	ks    = hp;
-	hp   += size;
-	mp    = (map_t*)hp;
-	res   = make_map(mp);
-	hp   += MAP_HEADER_SIZE;
-	vs    = hp;
+	BIF_RET(map_from_validated_list(BIF_P, BIF_ARG_1, size));
+    }
 
-	mp->thing_word = MAP_HEADER;
-	mp->size = size; /* set later, might shrink*/
-	mp->keys = keys;
+error:
 
-	if (size == 0)
-	    BIF_RET(res);
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+static Eterm map_from_validated_list(Process *p, Eterm list, Uint size) {
+    Eterm *kv, item = list;
+    Eterm *hp, *thp,*vs, *ks, keys, res;
+    map_t *mp;
+    Uint  unused_size = 0;
+    Sint  c = 0;
+    Sint  idx = 0;
 
-	item  = BIF_ARG_1;
 
-	/* first entry */
-	kv    = tuple_val(CAR(list_val(item)));
-	ks[0] = kv[1];
-	vs[0] = kv[2];
-	size  = 1;
-	item  = CDR(list_val(item));
+    hp    = HAlloc(p, 3 + 1 + (2 * size));
+    thp   = hp;
+    keys  = make_tuple(hp);
+    *hp++ = make_arityval(size);
+    ks    = hp;
+    hp   += size;
+    mp    = (map_t*)hp;
+    res   = make_map(mp);
+    hp   += MAP_HEADER_SIZE;
+    vs    = hp;
 
-	/* insert sort key/value pairs */
-	while(is_list(item)) {
+    mp->thing_word = MAP_HEADER;
+    mp->size = size; /* set later, might shrink*/
+    mp->keys = keys;
 
-	    kv = tuple_val(CAR(list_val(item)));
+    if (size == 0)
+	return res;
 
-	    /* compare ks backwards
-	     * idx represent word index to be written (hole position).
-	     * We cannot copy the elements when searching since we might
-	     * have an equal key. So we search for just the index first =(
-	     *
-	     * It is perhaps faster to move the values in the first pass.
-	     * Check for uniqueness during insert phase and then have a
-	     * second phace compacting the map if duplicates are found
-	     * during insert. .. or do someother sort .. shell-sort perhaps.
-	     */
+    /* first entry */
+    kv    = tuple_val(CAR(list_val(item)));
+    ks[0] = kv[1];
+    vs[0] = kv[2];
+    size  = 1;
+    item  = CDR(list_val(item));
+
+    /* insert sort key/value pairs */
+    while(is_list(item)) {
+
+	kv = tuple_val(CAR(list_val(item)));
+
+	/* compare ks backwards
+	 * idx represent word index to be written (hole position).
+	 * We cannot copy the elements when searching since we might
+	 * have an equal key. So we search for just the index first =(
+	 *
+	 * It is perhaps faster to move the values in the first pass.
+	 * Check for uniqueness during insert phase and then have a
+	 * second phace compacting the map if duplicates are found
+	 * during insert. .. or do someother sort .. shell-sort perhaps.
+	 */
 
-	    idx = size;
+	idx = size;
 
-	    while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; }
+	while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; }
 
-	    if (c == 0) {
-		/* last compare was equal,
-		 * i.e. we have to release memory
-		 * and overwrite that key/value
-		 */
-		ks[idx-1] = kv[1];
-		vs[idx-1] = kv[2];
-		unused_size++;
-	    } else {
-		Uint i = size;
-		while(i > idx) {
-		    ks[i] = ks[i-1];
-		    vs[i] = vs[i-1];
-		    i--;
-		}
-		ks[idx] = kv[1];
-		vs[idx] = kv[2];
-		size++;
+	if (c == 0) {
+	    /* last compare was equal,
+	     * i.e. we have to release memory
+	     * and overwrite that key/value
+	     */
+	    ks[idx-1] = kv[1];
+	    vs[idx-1] = kv[2];
+	    unused_size++;
+	} else {
+	    Uint i = size;
+	    while(i > idx) {
+		ks[i] = ks[i-1];
+		vs[i] = vs[i-1];
+		i--;
 	    }
-	    item = CDR(list_val(item));
+	    ks[idx] = kv[1];
+	    vs[idx] = kv[2];
+	    size++;
 	}
+	item = CDR(list_val(item));
+    }
 
-	if (unused_size) {
-	    /* the key tuple is embedded in the heap
-	     * write a bignum to clear it.
-	     */
-	    /* release values as normal since they are on the top of the heap */
-
-	    ks[size] = make_pos_bignum_header(unused_size - 1);
-	    HRelease(BIF_P, vs + size + unused_size, vs + size);
-	}
+    if (unused_size) {
+	/* the key tuple is embedded in the heap
+	 * write a bignum to clear it.
+	 */
+	/* release values as normal since they are on the top of the heap */
 
-	*thp = make_arityval(size);
-	mp->size = size;
-	BIF_RET(res);
+	ks[size] = make_pos_bignum_header(unused_size - 1);
+	HRelease(p, vs + size + unused_size, vs + size);
     }
 
-error:
-
-    BIF_ERROR(BIF_P, BADARG);
+    *thp = make_arityval(size);
+    mp->size = size;
+    return res;
 }
 
+
 /* maps:is_key/2
  */
 
-- 
cgit v1.2.3


From 9f0fccbd5e45cd11aecfd782c0dbc121c3895af6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 16:27:53 +0100
Subject: erts: Move hashmap:from_list/1 to maps

---
 erts/emulator/beam/bif.tab       |   1 -
 erts/emulator/beam/erl_hashmap.c | 472 ---------------------------------------
 erts/emulator/beam/erl_hashmap.h |  12 -
 erts/emulator/beam/erl_map.c     | 442 +++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_map.h     |  13 ++
 5 files changed, 454 insertions(+), 486 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 320cd39da8..7e92e58cab 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -616,7 +616,6 @@ bif erlang:get_keys/0
 
 # Hash Array Mappped Trie
 bif hashmap:info/1
-bif hashmap:from_list/1
 bif hashmap:new/0
 bif hashmap:merge/2
 
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 9b16a5a88f..06012950e1 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -61,22 +61,8 @@ static char *format_binary(Uint64 x, char *b) {
 #endif
 
 
-/* for hashmap_from_list/1 */
-typedef struct {
-    Uint32 hx;
-    Uint32 skip;
-    Uint i;
-    Eterm  val;
-} hxnode_t;
-
-static Eterm hashmap_from_list(Process *p, Eterm node);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
-static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n);
-static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
-static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
-static int hxnodecmp(hxnode_t* a, hxnode_t* b);
-static int hxnodecmpkey(hxnode_t* a, hxnode_t* b);
 
 /* hashmap:new/0 */
 
@@ -101,13 +87,6 @@ BIF_RETTYPE hashmap_new_0(BIF_ALIST_0) {
 
 /* hashmap:from_list/1 */
 
-BIF_RETTYPE hashmap_from_list_1(BIF_ALIST_1) {
-    if (is_list(BIF_ARG_1) || is_nil(BIF_ARG_1)) {
-	return hashmap_from_list(BIF_P, BIF_ARG_1);
-    }
-
-    BIF_ERROR(BIF_P, BADARG);
-}
 /* hashmap:get/2 */
 
 /* hashmap:find/2 */
@@ -134,434 +113,10 @@ BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
 /* impl. */
 
 
-#define swizzle32(D,S) \
-    do { \
-	(D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20  \
-	    | ((S) & 0x00000f00) << 12 | ((S) & 0x0000f000) << 4   \
-	    | ((S) & 0x000f0000) >> 4  | ((S) & 0x00f00000) >> 12  \
-	    | ((S) & 0x0f000000) >> 20 | ((S) & 0xf0000000) >> 28; \
-    } while(0)
-
-
-static Eterm hashmap_from_list(Process *p, Eterm list) {
-    Eterm *kv, res, item = list;
-    Eterm *hp;
-    Eterm tmp[2];
-    Uint32 sw, hx;
-    Uint ix = 0, n = 0;
-    hxnode_t *hxns;
-
-    /* Calculate size and check validity */
-
-    if (is_nil(list)) {
-	hp    = HAlloc(p, HAMT_HEAD_EMPTY_SZ);
-	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(0);
-	hp[1] = 0;
-	return make_hashmap(hp);
-    }
-
-    while(is_list(item)) {
-	res = CAR(list_val(item));
-	if (is_not_tuple(res))
-	    goto error;
-
-	kv = tuple_val(res);
-	if (*kv != make_arityval(2))
-	    goto error;
-
-	n++;
-	item = CDR(list_val(item));
-    }
-
-    if (is_not_nil(item))
-	goto error;
-
-    hp = HAlloc(p, (2 * n));
-
-    /* create tmp hx values and leaf ptrs */
-    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
-    item = list;
-
-    while(is_list(item)) {
-	res = CAR(list_val(item));
-	kv  = tuple_val(res);
-	hx  = hashmap_restore_hash(tmp,0,kv[1]);
-	swizzle32(sw,hx);
-	hxns[ix].hx   = sw;
-	hxns[ix].val  = CONS(hp, kv[1], kv[2]); hp += 2;
-	hxns[ix].skip = 1; /* will be reassigned in from_array */
-	hxns[ix].i    = ix;
-	ix++;
-	item = CDR(list_val(item));
-    }
-
-    ASSERT(n > 0);
-
-    res = hashmap_from_unsorted_array(p, hxns, n);
-
-    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
-
-    BIF_RET(res);
-error:
-    BIF_ERROR(p, BADARG);
-}
-
-Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
-    Eterm tmp[2];
-    Uint32 sw, hx;
-    Uint ix;
-    hxnode_t *hxns;
-    Eterm res;
-
-    /* create tmp hx values and leaf ptrs */
-    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
-
-    for (ix = 0; ix < n; ix++) {
-	hx  = hashmap_restore_hash(tmp,0,CAR(list_val(leafs[ix])));
-	swizzle32(sw,hx);
-	hxns[ix].hx   = sw;
-	hxns[ix].val  = leafs[ix];
-	hxns[ix].skip = 1;
-	hxns[ix].i    = ix;
-    }
-
-    res = hashmap_from_unsorted_array(p, hxns, n);
-
-    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
-
-    return res;
-}
-
-static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
-    Uint jx = 0, ix = 0, lx, cx;
-    Eterm res;
-
-    /* sort and compact array (remove non-unique entries) */
-    qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
-
-    ix = 0, cx = 0;
-    while(ix < n - 1) {
-	if (hxns[ix].hx == hxns[ix+1].hx) {
-
-	    /* find region of equal hash values */
-	    jx = ix + 1;
-	    while(jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; }
-	    /* find all correct keys from region
-	     * (last in list but now hash sorted so we check highest id instead) */
-
-	    /* resort with keys instead of hash value within region */
-	    
-	    qsort(&hxns[ix], jx - ix, sizeof(hxnode_t),
-		    (int (*)(const void *, const void *)) hxnodecmpkey);
-
-	    while(ix < jx) {
-		lx = ix;
-		while(ix < jx && EQ(CAR(list_val(hxns[ix].val)), CAR(list_val(hxns[lx].val)))) {
-		    if (hxns[ix].i > hxns[lx].i) {
-			lx = ix;
-		    }
-		    ix++;
-		}
-		hxns[cx].hx  = hxns[lx].hx;
-		hxns[cx].val = hxns[lx].val;
-		cx++;
-	    }
-	    ix = jx;
-	    continue;
-	}
-	if (ix > cx) {
-	    hxns[cx].hx  = hxns[ix].hx;
-	    hxns[cx].val = hxns[ix].val;
-	}
-	cx++;
-	ix++;
-    }
-
-    if (ix < n) {
-	hxns[cx].hx  = hxns[ix].hx;
-	hxns[cx].val = hxns[ix].val;
-	cx++;
-    }
-
-    if (cx > 1) {
-	/* recursive decompose array */
-	res = hashmap_from_sorted_unique_array(p, hxns, cx, 0);
-    } else {
-	Eterm *hp;
-	 /* hash value has been swizzled, need to drag it down to get the
-	 * correct slot. */
-	hp    = HAlloc(p, HAMT_HEAD_BITMAP_SZ(1));
-	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf));
-	hp[1] = 1;
-	hp[2] = hxns[0].val;
-	res   = make_hashmap(hp);
-    }
-
-    return res;
-}
-
-static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int lvl) {
-    Eterm res = NIL;
-    Uint i,ix,jx,elems;
-    Uint32 sw, hx;
-    Eterm val;
-    Eterm th[2];
-    hxnode_t *tmp;
-
-    ASSERT(lvl < 32);
-    ix = 0;
-    elems = 1;
-    while (ix < n - 1) {
-	if (hxns[ix].hx == hxns[ix+1].hx) {
-	    jx = ix + 1;
-	    while (jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; }
-	    tmp = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, ((jx - ix)) * sizeof(hxnode_t));
-
-	    for(i = 0; i < jx - ix; i++) {
-		val = hxns[i + ix].val;
-		hx  = hashmap_restore_hash(th, lvl + 8, CAR(list_val(val)));
-		swizzle32(sw,hx);
-		tmp[i].hx   = sw;
-		tmp[i].val  = val;
-		tmp[i].i    = i;
-		tmp[i].skip = 1;
-	    }
-
-	    qsort(tmp, jx - ix, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
-
-	    hxns[ix].skip = jx - ix;
-	    hxns[ix].val  = hashmap_from_sorted_unique_array(p, tmp, jx - ix, lvl + 8);
-	    erts_free(ERTS_ALC_T_TMP, (void *) tmp);
-	    ix = jx;
-	    if (ix < n) { elems++; }
-	    continue;
-	}
-	hxns[ix].skip = 1;
-	elems++;
-	ix++;
-    }
-
-    res = hashmap_from_chunked_array(p, hxns, elems, !lvl);
-
-    ERTS_HOLE_CHECK(p);
-
-    return res;
-}
-
-#define maskval(V,L)      (((V) >> ((7 - (L))*4)) & 0xf)
-#define cdepth(V1,V2)     (hashmap_clz((V1) ^ (V2)) >> 2)
-
 /* n must be > 1
  * hash values in hxns has to be unique
  */
 
-#define HALLOC_EXTRA 200
-static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root) {
-    Uint ix, d, dn, dc, slot, elems;
-    Uint32 v, vp, vn, hdr;
-    Uint bp, sz;
-    DECLARE_ESTACK(stack);
-    Eterm res = NIL, *hp = NULL, *nhp;
-
-    ASSERT(n > 1);
-
-    /* push initial nodes on the stack,
-     * this is the starting depth */
-
-    ix = 0;
-    d  = 0;
-    vp = hxns[ix].hx;
-    v  = hxns[ix + hxns[ix].skip].hx;
-
-    ASSERT(vp > v);
-    slot = maskval(vp,d);
-
-    while(slot == maskval(v,d)) {
-	ESTACK_PUSH(stack, 1 << slot);
-	d++;
-	slot = maskval(vp,d);
-    }
-
-    res = hxns[ix].val;
-
-    if (hxns[ix].skip > 1) {
-	dc = 7;
-	/* build collision nodes */
-	while (dc > d) {
-	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
-	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(vp,dc));
-	    hp[1] = res;
-	    res   = make_hashmap(hp);
-	    dc--;
-	}
-    }
-
-    ESTACK_PUSH(stack, res);
-    ESTACK_PUSH(stack, 1 << slot);
-
-    /* all of the other nodes .. */
-    elems = n - 2; /* remove first and last elements */
-    while(elems--) {
-	hdr = ESTACK_POP(stack);
-	ix  = ix + hxns[ix].skip;
-
-	/* determine if node or subtree should be built by looking
-	 * at the next value. */
-
-	vn = hxns[ix + hxns[ix].skip].hx;
-	dn = cdepth(v,vn);
-	ASSERT(v > vn);
-
-	res = hxns[ix].val;
-
-	if (hxns[ix].skip > 1) {
-	    int wat = (d > dn) ? d : dn;
-	    dc = 7;
-	    /* build collision nodes */
-	    while (dc > wat) {
-		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
-		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
-		hp[1] = res;
-		res   = make_hashmap(hp);
-		dc--;
-	    }
-	}
-
-	/* next depth is higher (implies collision) */
-	if (d < dn) {
-	    /* hdr is the popped one initially */
-	    while(d < dn) {
-		slot = maskval(v, d);
-		bp   = 1 << slot;
-		ESTACK_PUSH(stack, hdr | bp);
-		d++;
-		hdr = 0; /* clear hdr for all other collisions */
-	    }
-
-	    slot = maskval(v, d);
-	    bp   = 1 << slot;
-	    /* no more collisions */
-	    ESTACK_PUSH(stack,res);
-	    ESTACK_PUSH(stack,bp);
-	} else if (d == dn) {
-	    /* no collisions at all */
-	    slot = maskval(v, d);
-	    bp   = 1 << slot;
-	    ESTACK_PUSH(stack,res);
-	    ESTACK_PUSH(stack,hdr | bp);
-	} else {
-	    /* dn < n, we have a drop and we are done
-	     * build nodes and subtree */
-	    while (dn != d) {
-		slot  = maskval(v, d);
-		bp    = 1 << slot;
-		/* OR bitposition before sz calculation to handle
-		 * redundant collisions */
-		hdr  |= bp;
-		sz    = hashmap_bitcount(hdr);
-		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
-		nhp   = hp;
-		*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
-		*hp++ = res; sz--;
-		while (sz--) { *hp++ = ESTACK_POP(stack); }
-		ASSERT((hp - nhp) < 18);
-		res = make_hashmap(nhp);
-
-		/* we need to pop the next hdr and push if we don't need it */
-
-		hdr = ESTACK_POP(stack);
-		d--;
-	    }
-	    ESTACK_PUSH(stack, res);
-	    ESTACK_PUSH(stack, hdr);
-	}
-
-	vp = v;
-	v  = vn;
-	d  = dn;
-	ERTS_HOLE_CHECK(p);
-    }
-
-    /* v and vp are reused from above */
-    dn  = cdepth(vp,v);
-    ix  = ix + hxns[ix].skip;
-    res = hxns[ix].val;
-
-    if (hxns[ix].skip > 1) {
-	dc = 7;
-	/* build collision nodes */
-	while (dc > dn) {
-	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
-	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
-	    hp[1] = res;
-	    res   = make_hashmap(hp);
-	    dc--;
-	}
-    }
-
-    hdr = ESTACK_POP(stack);
-    /* pop remaining subtree if any */
-    while (dn) {
-	slot  = maskval(v, dn);
-	bp    = 1 << slot;
-	/* OR bitposition before sz calculation to handle
-	 * redundant collisions */
-	hdr  |= bp;
-	sz    = hashmap_bitcount(hdr);
-	hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
-	nhp   = hp;
-	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
-	*hp++ = res; sz--;
-
-	while (sz--) { *hp++ = ESTACK_POP(stack); }
-	res = make_hashmap(nhp);
-	hdr = ESTACK_POP(stack);
-	dn--;
-    }
-
-    /* and finally the root .. */
-
-    slot  = maskval(v, dn);
-    bp    = 1 << slot;
-    hdr  |= bp;
-    sz    = hashmap_bitcount(hdr);
-    hp    = HAlloc(p, sz + /* hdr + item */ (is_root ? 2 : 1));
-    nhp   = hp;
-
-    if (is_root) {
-	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr);
-	*hp++ = n;
-    } else {
-	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
-    }
-
-    *hp++ = res; sz--;
-    while (sz--) { *hp++ = ESTACK_POP(stack); }
-
-    res = make_hashmap(nhp);
-
-    ASSERT(ESTACK_COUNT(stack) == 0);
-    DESTROY_ESTACK(stack);
-    ERTS_HOLE_CHECK(p);
-    return res;
-}
-#undef HALLOC_EXTRA
-
-static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) {
-    return CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val)));
-}
-
-static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
-    if (a->hx < b->hx)
-	return 1;
-    else if (a->hx == b->hx)
-	return 0;
-    else
-	return -1;
-}
-
 #define HALLOC_EXTRA 200
 
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
@@ -938,30 +493,3 @@ BIF_RETTYPE hashmap_info_1(BIF_ALIST_1) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
-
-/* implementation of builtin emulations */
-
-#if !defined(__GNUC__)
-/* Count leading zeros emulation */
-Uint32 hashmap_clz(Uint32 x) {
-    Uint32 y;
-    int n = 32;
-    y = x >>16;  if (y != 0) {n = n -16;  x = y;}
-    y = x >> 8;  if (y != 0) {n = n - 8;  x = y;}
-    y = x >> 4;  if (y != 0) {n = n - 4;  x = y;}
-    y = x >> 2;  if (y != 0) {n = n - 2;  x = y;}
-    y = x >> 1;  if (y != 0) return n - 2;
-    return n - x;
-}
-const Uint32 SK5 = 0x55555555, SK3 = 0x33333333;
-const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF;
-
-/* CTPOP emulation */
-Uint32 hashmap_bitcount(Uint32 x) {
-    x -= ((x >> 1  ) & SK5);
-    x  =  (x & SK3 ) + ((x >> 2 ) & SK3 );
-    x  =  (x & SKF0) + ((x >> 4 ) & SKF0);
-    x +=   x >> 8;
-    return (x + (x >> 16)) & 0x3F;
-}
-#endif
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
index 5a9aa05f61..7ac33b34b0 100644
--- a/erts/emulator/beam/erl_hashmap.h
+++ b/erts/emulator/beam/erl_hashmap.h
@@ -24,20 +24,10 @@
 #include "sys.h"
 #include "erl_term.h"
 
-Eterm erts_hashmap_get(Eterm key, Eterm map);
 int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
-Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
 
 /* HASH */
 
-#if defined(__GNUC__)
-#define hashmap_clz(x)       ((Uint32) __builtin_clz((unsigned int)(x)))
-#define hashmap_bitcount(x)  ((Uint32) __builtin_popcount((unsigned int) (x)))
-#else
-Uint32 hashmap_clz(Uint32 x);
-Uint32 hashmap_bitcount(Uint32 x);
-#endif
-
 /* hamt nodes v2.0
  * 
  * node :: leaf | array | bitmap
@@ -49,8 +39,6 @@ typedef struct hashmap_head_s {
     Eterm items[1];
 } hashmap_head_t;
 
-
-
 /* thing_word tagscheme
  * Need two bits for map subtags
  *
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 5e5e567642..7281353af5 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -63,6 +63,14 @@
  * - erts_internal:map_to_tuple_keys/1
  */
 
+/* for hashmap_from_list/1 */
+typedef struct {
+    Uint32 hx;
+    Uint32 skip;
+    Uint i;
+    Eterm  val;
+} hxnode_t;
+
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
 static Eterm hashmap_to_list(Process *p, Eterm map);
@@ -70,6 +78,12 @@ static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm map_from_validated_list(Process *p, Eterm list, Uint size);
+static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
+static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n);
+static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
+static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
+static int hxnodecmp(hxnode_t* a, hxnode_t* b);
+static int hxnodecmpkey(hxnode_t* a, hxnode_t* b);
 
 /* erlang:map_size/1
  * the corresponding instruction is implemented in:
@@ -250,7 +264,11 @@ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) {
 	if (is_not_nil(item))
 	    goto error;
 
-	BIF_RET(map_from_validated_list(BIF_P, BIF_ARG_1, size));
+	if (size > MAP_SMALL_MAP_LIMIT) {
+	    BIF_RET(hashmap_from_validated_list(BIF_P, BIF_ARG_1, size));
+	} else {
+	    BIF_RET(map_from_validated_list(BIF_P, BIF_ARG_1, size));
+	}
     }
 
 error:
@@ -349,6 +367,401 @@ static Eterm map_from_validated_list(Process *p, Eterm list, Uint size) {
     return res;
 }
 
+#define swizzle32(D,S) \
+    do { \
+	(D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20  \
+	    | ((S) & 0x00000f00) << 12 | ((S) & 0x0000f000) << 4   \
+	    | ((S) & 0x000f0000) >> 4  | ((S) & 0x00f00000) >> 12  \
+	    | ((S) & 0x0f000000) >> 20 | ((S) & 0xf0000000) >> 28; \
+    } while(0)
+
+#define maskval(V,L)      (((V) >> ((7 - (L))*4)) & 0xf)
+#define cdepth(V1,V2)     (hashmap_clz((V1) ^ (V2)) >> 2)
+
+static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
+    Eterm item = list;
+    Eterm *hp;
+    Eterm *kv, res;
+    Eterm tmp[2];
+    Uint32 sw, hx;
+    Uint ix = 0;
+    hxnode_t *hxns;
+
+    ASSERT(size > 0);
+
+    hp = HAlloc(p, (2 * size));
+
+    /* create tmp hx values and leaf ptrs */
+    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, size * sizeof(hxnode_t));
+
+    while(is_list(item)) {
+	res = CAR(list_val(item));
+	kv  = tuple_val(res);
+	hx  = hashmap_restore_hash(tmp,0,kv[1]);
+	swizzle32(sw,hx);
+	hxns[ix].hx   = sw;
+	hxns[ix].val  = CONS(hp, kv[1], kv[2]); hp += 2;
+	hxns[ix].skip = 1; /* will be reassigned in from_array */
+	hxns[ix].i    = ix;
+	ix++;
+	item = CDR(list_val(item));
+    }
+
+    res = hashmap_from_unsorted_array(p, hxns, size);
+
+    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+    return res;
+}
+
+Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
+    Eterm tmp[2];
+    Uint32 sw, hx;
+    Uint ix;
+    hxnode_t *hxns;
+    Eterm res;
+
+    /* create tmp hx values and leaf ptrs */
+    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
+
+    for (ix = 0; ix < n; ix++) {
+	hx  = hashmap_restore_hash(tmp,0,CAR(list_val(leafs[ix])));
+	swizzle32(sw,hx);
+	hxns[ix].hx   = sw;
+	hxns[ix].val  = leafs[ix];
+	hxns[ix].skip = 1;
+	hxns[ix].i    = ix;
+    }
+
+    res = hashmap_from_unsorted_array(p, hxns, n);
+
+    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+    return res;
+}
+
+static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
+    Uint jx = 0, ix = 0, lx, cx;
+    Eterm res;
+
+    /* sort and compact array (remove non-unique entries) */
+    qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
+
+    ix = 0, cx = 0;
+    while(ix < n - 1) {
+	if (hxns[ix].hx == hxns[ix+1].hx) {
+
+	    /* find region of equal hash values */
+	    jx = ix + 1;
+	    while(jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; }
+	    /* find all correct keys from region
+	     * (last in list but now hash sorted so we check highest id instead) */
+
+	    /* resort with keys instead of hash value within region */
+
+	    qsort(&hxns[ix], jx - ix, sizeof(hxnode_t),
+		    (int (*)(const void *, const void *)) hxnodecmpkey);
+
+	    while(ix < jx) {
+		lx = ix;
+		while(ix < jx && EQ(CAR(list_val(hxns[ix].val)), CAR(list_val(hxns[lx].val)))) {
+		    if (hxns[ix].i > hxns[lx].i) {
+			lx = ix;
+		    }
+		    ix++;
+		}
+		hxns[cx].hx  = hxns[lx].hx;
+		hxns[cx].val = hxns[lx].val;
+		cx++;
+	    }
+	    ix = jx;
+	    continue;
+	}
+	if (ix > cx) {
+	    hxns[cx].hx  = hxns[ix].hx;
+	    hxns[cx].val = hxns[ix].val;
+	}
+	cx++;
+	ix++;
+    }
+
+    if (ix < n) {
+	hxns[cx].hx  = hxns[ix].hx;
+	hxns[cx].val = hxns[ix].val;
+	cx++;
+    }
+
+    if (cx > 1) {
+	/* recursive decompose array */
+	res = hashmap_from_sorted_unique_array(p, hxns, cx, 0);
+    } else {
+	Eterm *hp;
+	 /* hash value has been swizzled, need to drag it down to get the
+	 * correct slot. */
+	hp    = HAlloc(p, HAMT_HEAD_BITMAP_SZ(1));
+	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf));
+	hp[1] = 1;
+	hp[2] = hxns[0].val;
+	res   = make_hashmap(hp);
+    }
+
+    return res;
+}
+
+static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int lvl) {
+    Eterm res = NIL;
+    Uint i,ix,jx,elems;
+    Uint32 sw, hx;
+    Eterm val;
+    Eterm th[2];
+    hxnode_t *tmp;
+
+    ASSERT(lvl < 32);
+    ix = 0;
+    elems = 1;
+    while (ix < n - 1) {
+	if (hxns[ix].hx == hxns[ix+1].hx) {
+	    jx = ix + 1;
+	    while (jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; }
+	    tmp = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, ((jx - ix)) * sizeof(hxnode_t));
+
+	    for(i = 0; i < jx - ix; i++) {
+		val = hxns[i + ix].val;
+		hx  = hashmap_restore_hash(th, lvl + 8, CAR(list_val(val)));
+		swizzle32(sw,hx);
+		tmp[i].hx   = sw;
+		tmp[i].val  = val;
+		tmp[i].i    = i;
+		tmp[i].skip = 1;
+	    }
+
+	    qsort(tmp, jx - ix, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
+
+	    hxns[ix].skip = jx - ix;
+	    hxns[ix].val  = hashmap_from_sorted_unique_array(p, tmp, jx - ix, lvl + 8);
+	    erts_free(ERTS_ALC_T_TMP, (void *) tmp);
+	    ix = jx;
+	    if (ix < n) { elems++; }
+	    continue;
+	}
+	hxns[ix].skip = 1;
+	elems++;
+	ix++;
+    }
+
+    res = hashmap_from_chunked_array(p, hxns, elems, !lvl);
+
+    ERTS_HOLE_CHECK(p);
+
+    return res;
+}
+
+#define HALLOC_EXTRA 200
+static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root) {
+    Uint ix, d, dn, dc, slot, elems;
+    Uint32 v, vp, vn, hdr;
+    Uint bp, sz;
+    DECLARE_ESTACK(stack);
+    Eterm res = NIL, *hp = NULL, *nhp;
+
+    ASSERT(n > 1);
+
+    /* push initial nodes on the stack,
+     * this is the starting depth */
+
+    ix = 0;
+    d  = 0;
+    vp = hxns[ix].hx;
+    v  = hxns[ix + hxns[ix].skip].hx;
+
+    ASSERT(vp > v);
+    slot = maskval(vp,d);
+
+    while(slot == maskval(v,d)) {
+	ESTACK_PUSH(stack, 1 << slot);
+	d++;
+	slot = maskval(vp,d);
+    }
+
+    res = hxns[ix].val;
+
+    if (hxns[ix].skip > 1) {
+	dc = 7;
+	/* build collision nodes */
+	while (dc > d) {
+	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(vp,dc));
+	    hp[1] = res;
+	    res   = make_hashmap(hp);
+	    dc--;
+	}
+    }
+
+    ESTACK_PUSH(stack, res);
+    ESTACK_PUSH(stack, 1 << slot);
+
+    /* all of the other nodes .. */
+    elems = n - 2; /* remove first and last elements */
+    while(elems--) {
+	hdr = ESTACK_POP(stack);
+	ix  = ix + hxns[ix].skip;
+
+	/* determine if node or subtree should be built by looking
+	 * at the next value. */
+
+	vn = hxns[ix + hxns[ix].skip].hx;
+	dn = cdepth(v,vn);
+	ASSERT(v > vn);
+
+	res = hxns[ix].val;
+
+	if (hxns[ix].skip > 1) {
+	    int wat = (d > dn) ? d : dn;
+	    dc = 7;
+	    /* build collision nodes */
+	    while (dc > wat) {
+		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
+		hp[1] = res;
+		res   = make_hashmap(hp);
+		dc--;
+	    }
+	}
+
+	/* next depth is higher (implies collision) */
+	if (d < dn) {
+	    /* hdr is the popped one initially */
+	    while(d < dn) {
+		slot = maskval(v, d);
+		bp   = 1 << slot;
+		ESTACK_PUSH(stack, hdr | bp);
+		d++;
+		hdr = 0; /* clear hdr for all other collisions */
+	    }
+
+	    slot = maskval(v, d);
+	    bp   = 1 << slot;
+	    /* no more collisions */
+	    ESTACK_PUSH(stack,res);
+	    ESTACK_PUSH(stack,bp);
+	} else if (d == dn) {
+	    /* no collisions at all */
+	    slot = maskval(v, d);
+	    bp   = 1 << slot;
+	    ESTACK_PUSH(stack,res);
+	    ESTACK_PUSH(stack,hdr | bp);
+	} else {
+	    /* dn < n, we have a drop and we are done
+	     * build nodes and subtree */
+	    while (dn != d) {
+		slot  = maskval(v, d);
+		bp    = 1 << slot;
+		/* OR bitposition before sz calculation to handle
+		 * redundant collisions */
+		hdr  |= bp;
+		sz    = hashmap_bitcount(hdr);
+		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
+		nhp   = hp;
+		*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+		*hp++ = res; sz--;
+		while (sz--) { *hp++ = ESTACK_POP(stack); }
+		ASSERT((hp - nhp) < 18);
+		res = make_hashmap(nhp);
+
+		/* we need to pop the next hdr and push if we don't need it */
+
+		hdr = ESTACK_POP(stack);
+		d--;
+	    }
+	    ESTACK_PUSH(stack, res);
+	    ESTACK_PUSH(stack, hdr);
+	}
+
+	vp = v;
+	v  = vn;
+	d  = dn;
+	ERTS_HOLE_CHECK(p);
+    }
+
+    /* v and vp are reused from above */
+    dn  = cdepth(vp,v);
+    ix  = ix + hxns[ix].skip;
+    res = hxns[ix].val;
+
+    if (hxns[ix].skip > 1) {
+	dc = 7;
+	/* build collision nodes */
+	while (dc > dn) {
+	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
+	    hp[1] = res;
+	    res   = make_hashmap(hp);
+	    dc--;
+	}
+    }
+
+    hdr = ESTACK_POP(stack);
+    /* pop remaining subtree if any */
+    while (dn) {
+	slot  = maskval(v, dn);
+	bp    = 1 << slot;
+	/* OR bitposition before sz calculation to handle
+	 * redundant collisions */
+	hdr  |= bp;
+	sz    = hashmap_bitcount(hdr);
+	hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
+	nhp   = hp;
+	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+	*hp++ = res; sz--;
+
+	while (sz--) { *hp++ = ESTACK_POP(stack); }
+	res = make_hashmap(nhp);
+	hdr = ESTACK_POP(stack);
+	dn--;
+    }
+
+    /* and finally the root .. */
+
+    slot  = maskval(v, dn);
+    bp    = 1 << slot;
+    hdr  |= bp;
+    sz    = hashmap_bitcount(hdr);
+    hp    = HAlloc(p, sz + /* hdr + item */ (is_root ? 2 : 1));
+    nhp   = hp;
+
+    if (is_root) {
+	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr);
+	*hp++ = n;
+    } else {
+	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+    }
+
+    *hp++ = res; sz--;
+    while (sz--) { *hp++ = ESTACK_POP(stack); }
+
+    res = make_hashmap(nhp);
+
+    ASSERT(ESTACK_COUNT(stack) == 0);
+    DESTROY_ESTACK(stack);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+#undef HALLOC_EXTRA
+
+static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) {
+    return CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val)));
+}
+
+static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
+    if (a->hx < b->hx)
+	return 1;
+    else if (a->hx == b->hx)
+	return 0;
+    else
+	return -1;
+}
 
 /* maps:is_key/2
  */
@@ -1593,3 +2006,30 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
+
+/* implementation of builtin emulations */
+
+#if !defined(__GNUC__)
+/* Count leading zeros emulation */
+Uint32 hashmap_clz(Uint32 x) {
+    Uint32 y;
+    int n = 32;
+    y = x >>16;  if (y != 0) {n = n -16;  x = y;}
+    y = x >> 8;  if (y != 0) {n = n - 8;  x = y;}
+    y = x >> 4;  if (y != 0) {n = n - 4;  x = y;}
+    y = x >> 2;  if (y != 0) {n = n - 2;  x = y;}
+    y = x >> 1;  if (y != 0) return n - 2;
+    return n - x;
+}
+const Uint32 SK5 = 0x55555555, SK3 = 0x33333333;
+const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF;
+
+/* CTPOP emulation */
+Uint32 hashmap_bitcount(Uint32 x) {
+    x -= ((x >> 1  ) & SK5);
+    x  =  (x & SK3 ) + ((x >> 2 ) & SK3 );
+    x  =  (x & SKF0) + ((x >> 4 ) & SKF0);
+    x +=   x >> 8;
+    return (x + (x >> 16)) & 0x3F;
+}
+#endif
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 48bc316e3b..428cfe9b63 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -22,6 +22,16 @@
 #define __ERL_MAP_H__
 
 #include "sys.h"
+
+/* instrinsic wrappers */
+#if defined(__GNUC__)
+#define hashmap_clz(x)       ((Uint32) __builtin_clz((unsigned int)(x)))
+#define hashmap_bitcount(x)  ((Uint32) __builtin_popcount((unsigned int) (x)))
+#else
+Uint32 hashmap_clz(Uint32 x);
+Uint32 hashmap_bitcount(Uint32 x);
+#endif
+
 /* MAP */
 
 typedef struct map_s {
@@ -70,6 +80,7 @@ typedef struct map_s {
 #define map_get_keys(x)        (((Eterm *)tuple_val(((map_t *)(x))->keys)) + 1)
 #define map_get_size(x)        (((map_t*)(x))->size)
 
+#define MAP_SMALL_MAP_LIMIT    (32)
 #define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
 #define MAP_HEADER_SIZE        (sizeof(map_t) / sizeof(Eterm))
 
@@ -80,6 +91,8 @@ int   erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
 int   erts_validate_and_sort_map(map_t* map);
 void  hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
+Eterm erts_hashmap_get(Eterm key, Eterm map);
+Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
 
 #if HALFWORD_HEAP
 const Eterm *
-- 
cgit v1.2.3


From a80156026638b6605b636c16fa59e8206ff7635e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 16:32:59 +0100
Subject: erts: Remove hashmap:new/0

---
 erts/emulator/beam/bif.tab       |  1 -
 erts/emulator/beam/erl_hashmap.c | 13 -------------
 2 files changed, 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 7e92e58cab..e9c5e83203 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -616,7 +616,6 @@ bif erlang:get_keys/0
 
 # Hash Array Mappped Trie
 bif hashmap:info/1
-bif hashmap:new/0
 bif hashmap:merge/2
 
 #
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 06012950e1..29a14a9f20 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -66,19 +66,6 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
 /* hashmap:new/0 */
 
-BIF_RETTYPE hashmap_new_0(BIF_ALIST_0) {
-    Eterm* hp;
-    hashmap_head_t *head;
-
-    hp   = HAlloc(BIF_P, HAMT_HEAD_EMPTY_SZ);
-    head = (hashmap_head_t *) hp;
-
-    head->thing_word = MAP_HEADER_HAMT_HEAD_BITMAP(0);
-    head->size       = 0;
-
-    BIF_RET(make_hashmap(head));
-}
-
 /* hashmap:put/3 */
 
 /* hashmap:update/3 */
-- 
cgit v1.2.3


From 746d2d846ebf20fc4dc303c53bcd1e908aee924b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 16:45:23 +0100
Subject: erts: Move hashmap:info/1 to erts_debug:map_info/1

---
 erts/emulator/beam/bif.tab       |   2 +-
 erts/emulator/beam/erl_hashmap.c | 135 ---------------------------------------
 erts/emulator/beam/erl_map.c     | 133 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 134 insertions(+), 136 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index e9c5e83203..f9127ef955 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -613,9 +613,9 @@ bif erlang:fun_info_mfa/1
 #
 
 bif erlang:get_keys/0
+bif erts_debug:map_info/1
 
 # Hash Array Mappped Trie
-bif hashmap:info/1
 bif hashmap:merge/2
 
 #
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index 29a14a9f20..fecdc2ef31 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -45,10 +45,6 @@
 #include "erl_map.h"
 #include "erl_hashmap.h"
 
-#ifndef DECL_AM
-#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
-#endif
-
 #if 0
 static char *format_binary(Uint64 x, char *b) {
     int z;
@@ -62,7 +58,6 @@ static char *format_binary(Uint64 x, char *b) {
 
 
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
-static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 
 /* hashmap:new/0 */
 
@@ -100,10 +95,6 @@ BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
 /* impl. */
 
 
-/* n must be > 1
- * hash values in hxns has to be unique
- */
-
 #define HALLOC_EXTRA 200
 
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
@@ -354,129 +345,3 @@ int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
 }
 
 /* hashmap:info/0 */
-
-static Eterm hashmap_info(Process *p, Eterm node) {
-    Eterm *hp;
-    Eterm res = NIL, info = NIL;
-    Eterm *ptr, tup, hdr;
-    Uint sz;
-    DECL_AM(depth);
-    DECL_AM(leafs);
-    DECL_AM(bitmaps);
-    DECL_AM(arrays);
-    Uint nleaf=0, nbitmap=0, narray=0;
-    Uint bitmap_usage[16], leaf_usage[16];
-    Uint lvl = 0, clvl;
-    DECLARE_ESTACK(stack);
-
-    for (sz = 0; sz < 16; sz++) {
-	bitmap_usage[sz] = 0;
-	leaf_usage[sz] = 0;
-    }
-
-    ptr = boxed_val(node);
-    ESTACK_PUSH(stack, 0);
-    ESTACK_PUSH(stack, node);
-    do {
-	node = ESTACK_POP(stack);
-	clvl = ESTACK_POP(stack);
-	lvl  = MAX(lvl,clvl);
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST:
-		nleaf++;
-		leaf_usage[clvl] += 1;
-		break;
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			narray++;
-			sz = 16;
-			while(sz--) {
-			    ESTACK_PUSH(stack, clvl + 1);
-			    ESTACK_PUSH(stack, ptr[sz+1]);
-			}
-			break;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			nbitmap++;
-			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-			ASSERT(sz < 17);
-			bitmap_usage[sz-1] += 1;
-			while(sz--) {
-			    ESTACK_PUSH(stack, clvl + 1);
-			    ESTACK_PUSH(stack, ptr[sz+1]);
-			}
-			break;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			nbitmap++;
-			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-			bitmap_usage[sz-1] += 1;
-			while(sz--) {
-			    ESTACK_PUSH(stack, clvl + 1);
-			    ESTACK_PUSH(stack, ptr[sz+2]);
-			}
-			break;
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			narray++;
-			sz = 16;
-			while(sz--) {
-			    ESTACK_PUSH(stack, clvl + 1);
-			    ESTACK_PUSH(stack, ptr[sz+2]);
-			}
-			break;
-		    default:
-			erl_exit(1, "bad header\r\n");
-			break;
-		}
-	}
-    } while(!ESTACK_ISEMPTY(stack));
-
-
-    /* size */
-    sz = 0;
-    hashmap_bld_tuple_uint(NULL,&sz,16,leaf_usage);
-    hashmap_bld_tuple_uint(NULL,&sz,16,bitmap_usage);
-
-    /* alloc */
-    hp   = HAlloc(p, 2+3 + 3*(2+4) + sz);
-
-    info = hashmap_bld_tuple_uint(&hp,NULL,16,leaf_usage);
-    tup  = TUPLE3(hp, AM_leafs, make_small(nleaf),info); hp += 4;
-    res  = CONS(hp, tup, res); hp += 2;
-
-    info = hashmap_bld_tuple_uint(&hp,NULL,16,bitmap_usage);
-    tup  = TUPLE3(hp, AM_bitmaps, make_small(nbitmap), info); hp += 4;
-    res  = CONS(hp, tup, res); hp += 2;
-
-    tup  = TUPLE3(hp, AM_arrays, make_small(narray),NIL); hp += 4;
-    res  = CONS(hp, tup, res); hp += 2;
-
-    tup  = TUPLE2(hp, AM_depth, make_small(lvl)); hp += 3;
-    res  = CONS(hp, tup, res); hp += 2;
-
-    DESTROY_ESTACK(stack);
-    ERTS_HOLE_CHECK(p);
-    return res;
-}
-
-static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]) {
-    Eterm res = THE_NON_VALUE;
-    Eterm *ts = (Eterm *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
-    Uint i;
-
-    for (i = 0; i < n; i++) {
-	ts[i] = erts_bld_uint(hpp, szp, nums[i]);
-    }
-    res = erts_bld_tuplev(hpp, szp, n, ts);
-    erts_free(ERTS_ALC_T_TMP, (void *) ts);
-    return res;
-}
-
-BIF_RETTYPE hashmap_info_1(BIF_ALIST_1) {
-    if (is_hashmap(BIF_ARG_1)) {
-	BIF_RET(hashmap_info(BIF_P,BIF_ARG_1));
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 7281353af5..14f16e9050 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -63,6 +63,10 @@
  * - erts_internal:map_to_tuple_keys/1
  */
 
+#ifndef DECL_AM
+#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
+#endif
+
 /* for hashmap_from_list/1 */
 typedef struct {
     Uint32 hx;
@@ -82,6 +86,8 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n);
 static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
 static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
+static Eterm hashmap_info(Process *p, Eterm node);
+static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 static int hxnodecmp(hxnode_t* a, hxnode_t* b);
 static int hxnodecmpkey(hxnode_t* a, hxnode_t* b);
 
@@ -1993,6 +1999,13 @@ int erts_validate_and_sort_map(map_t* mp)
     return 1;
 }
 
+BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+	BIF_RET(hashmap_info(BIF_P,BIF_ARG_1));
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 /*
  * erts_internal:map_to_tuple_keys/1
  *
@@ -2007,6 +2020,126 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
+static Eterm hashmap_info(Process *p, Eterm node) {
+    Eterm *hp;
+    Eterm res = NIL, info = NIL;
+    Eterm *ptr, tup, hdr;
+    Uint sz;
+    DECL_AM(depth);
+    DECL_AM(leafs);
+    DECL_AM(bitmaps);
+    DECL_AM(arrays);
+    Uint nleaf=0, nbitmap=0, narray=0;
+    Uint bitmap_usage[16], leaf_usage[16];
+    Uint lvl = 0, clvl;
+    DECLARE_ESTACK(stack);
+
+    for (sz = 0; sz < 16; sz++) {
+	bitmap_usage[sz] = 0;
+	leaf_usage[sz] = 0;
+    }
+
+    ptr = boxed_val(node);
+    ESTACK_PUSH(stack, 0);
+    ESTACK_PUSH(stack, node);
+    do {
+	node = ESTACK_POP(stack);
+	clvl = ESTACK_POP(stack);
+	lvl  = MAX(lvl,clvl);
+	switch(primary_tag(node)) {
+	    case TAG_PRIMARY_LIST:
+		nleaf++;
+		leaf_usage[clvl] += 1;
+		break;
+	    case TAG_PRIMARY_BOXED:
+		ptr = boxed_val(node);
+		hdr = *ptr;
+		ASSERT(is_header(hdr));
+		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+		    case HAMT_SUBTAG_NODE_ARRAY:
+			narray++;
+			sz = 16;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+1]);
+			}
+			break;
+		    case HAMT_SUBTAG_NODE_BITMAP:
+			nbitmap++;
+			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			ASSERT(sz < 17);
+			bitmap_usage[sz-1] += 1;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+1]);
+			}
+			break;
+		    case HAMT_SUBTAG_HEAD_BITMAP:
+			nbitmap++;
+			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+			bitmap_usage[sz-1] += 1;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+2]);
+			}
+			break;
+		    case HAMT_SUBTAG_HEAD_ARRAY:
+			narray++;
+			sz = 16;
+			while(sz--) {
+			    ESTACK_PUSH(stack, clvl + 1);
+			    ESTACK_PUSH(stack, ptr[sz+2]);
+			}
+			break;
+		    default:
+			erl_exit(1, "bad header\r\n");
+			break;
+		}
+	}
+    } while(!ESTACK_ISEMPTY(stack));
+
+
+    /* size */
+    sz = 0;
+    hashmap_bld_tuple_uint(NULL,&sz,16,leaf_usage);
+    hashmap_bld_tuple_uint(NULL,&sz,16,bitmap_usage);
+
+    /* alloc */
+    hp   = HAlloc(p, 2+3 + 3*(2+4) + sz);
+
+    info = hashmap_bld_tuple_uint(&hp,NULL,16,leaf_usage);
+    tup  = TUPLE3(hp, AM_leafs, make_small(nleaf),info); hp += 4;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    info = hashmap_bld_tuple_uint(&hp,NULL,16,bitmap_usage);
+    tup  = TUPLE3(hp, AM_bitmaps, make_small(nbitmap), info); hp += 4;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    tup  = TUPLE3(hp, AM_arrays, make_small(narray),NIL); hp += 4;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    tup  = TUPLE2(hp, AM_depth, make_small(lvl)); hp += 3;
+    res  = CONS(hp, tup, res); hp += 2;
+
+    DESTROY_ESTACK(stack);
+    ERTS_HOLE_CHECK(p);
+    return res;
+}
+
+static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]) {
+    Eterm res = THE_NON_VALUE;
+    Eterm *ts = (Eterm *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
+    Uint i;
+
+    for (i = 0; i < n; i++) {
+	ts[i] = erts_bld_uint(hpp, szp, nums[i]);
+    }
+    res = erts_bld_tuplev(hpp, szp, n, ts);
+    erts_free(ERTS_ALC_T_TMP, (void *) ts);
+    return res;
+}
+
+
 /* implementation of builtin emulations */
 
 #if !defined(__GNUC__)
-- 
cgit v1.2.3


From 7ac42c5d31f7722907b44d2df3421f8cd88d48c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 17:27:41 +0100
Subject: erts: Move hashmap:merge/2 to maps

---
 erts/emulator/beam/bif.tab       |   3 -
 erts/emulator/beam/erl_hashmap.c | 262 ----------------------
 erts/emulator/beam/erl_map.c     | 467 ++++++++++++++++++++++++++++++++-------
 3 files changed, 392 insertions(+), 340 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f9127ef955..b4e821a986 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -615,9 +615,6 @@ bif erlang:fun_info_mfa/1
 bif erlang:get_keys/0
 bif erts_debug:map_info/1
 
-# Hash Array Mappped Trie
-bif hashmap:merge/2
-
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
index fecdc2ef31..0c146137f5 100644
--- a/erts/emulator/beam/erl_hashmap.c
+++ b/erts/emulator/beam/erl_hashmap.c
@@ -56,9 +56,6 @@ static char *format_binary(Uint64 x, char *b) {
 }
 #endif
 
-
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
-
 /* hashmap:new/0 */
 
 /* hashmap:put/3 */
@@ -85,263 +82,4 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 
 /* hashmap:values/1 */
 
-BIF_RETTYPE hashmap_merge_2(BIF_ALIST_2) {
-    if (is_hashmap(BIF_ARG_1) && is_hashmap(BIF_ARG_2)) {
-	BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
-
-/* impl. */
-
-
-#define HALLOC_EXTRA 200
-
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
-#define PSTACK_TYPE struct HashmapMergePStackType
-    struct HashmapMergePStackType {
-	Eterm *srcA, *srcB;
-	Uint32 abm, bbm, rbm;        /* node bitmaps */
-	int keepA;
-	int ix;
-	Eterm array[16];
-    };
-    PSTACK_DECLARE(s, 4);
-    struct HashmapMergePStackType* sp = PSTACK_PUSH(s);
-    Eterm *hp, *nhp;
-    Eterm hdrA, hdrB;
-    Eterm th[2];
-    Uint32 ahx, bhx;
-    Uint size;  /* total key-value counter */
-    int keepA = 0;
-    unsigned lvl = 0;
-    Eterm res = THE_NON_VALUE;
-
-    /*
-     * Strategy: Do depth-first traversal of both trees (at the same time)
-     * and merge each pair of nodes.
-     */
-
-    {
-	hashmap_head_t* a = (hashmap_head_t*) hashmap_val(nodeA);
-	hashmap_head_t* b = (hashmap_head_t*) hashmap_val(nodeB);
-	size = a->size + b->size;
-    }
-
-recurse:
-
-    if (primary_tag(nodeA) == TAG_PRIMARY_BOXED &&
-	primary_tag(nodeB) == TAG_PRIMARY_LIST) {
-	/* Avoid implementing this combination by switching places */
-	Eterm tmp = nodeA;
-	nodeA = nodeB;
-	nodeB = tmp;
-	keepA = !keepA;
-    }
-
-    switch (primary_tag(nodeA)) {
-    case TAG_PRIMARY_LIST: {
-	sp->srcA = list_val(nodeA);
-	switch (primary_tag(nodeB)) {
-	case TAG_PRIMARY_LIST: { /* LEAF + LEAF */
-	    sp->srcB = list_val(nodeB);
-
-	    if (EQ(CAR(sp->srcA), CAR(sp->srcB))) {
-		--size;
-		res = keepA ? nodeA : nodeB;
-	    } else {
-		ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
-		bhx = hashmap_restore_hash(th, lvl, CAR(sp->srcB));
-		sp->abm = 1 << hashmap_index(ahx);
-		sp->bbm = 1 << hashmap_index(bhx);
-
-		sp->srcA = &nodeA;
-		sp->srcB = &nodeB;
-	    }
-	    break;
-	}
-	case TAG_PRIMARY_BOXED: { /* LEAF + NODE */
-	    sp->srcB = boxed_val(nodeB);
-	    ASSERT(is_header(*sp->srcB));
-	    hdrB = *sp->srcB++;
-
-	    ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
-	    sp->abm = 1 << hashmap_index(ahx);
-	    sp->srcA = &nodeA;
-	    switch(hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
-		sp->bbm = 0xffff;
-		break;
-
-	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sp->bbm = MAP_HEADER_VAL(hdrB);
-		break;
-
-	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
-		break;
-	    }
-	    break;
-	}
-	default:
-	    erl_exit(1, "bad primary tag %ld\r\n", nodeB);
-	}
-	break;
-    }
-    case TAG_PRIMARY_BOXED: { /* NODE + NODE */
-	sp->srcA = boxed_val(nodeA);
-	hdrA = *sp->srcA++;
-	ASSERT(is_header(hdrA));
-	switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
-	case HAMT_SUBTAG_HEAD_ARRAY: sp->srcA++;
-	case HAMT_SUBTAG_NODE_ARRAY: {
-	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
-	    sp->abm = 0xffff;
-	    sp->srcB = boxed_val(nodeB);
-	    hdrB = *sp->srcB++;
-	    ASSERT(is_header(hdrB));
-	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
-		sp->bbm = 0xffff;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sp->bbm = MAP_HEADER_VAL(hdrB);
-		break;
-	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
-	    }
-	    break;
-	}
-	case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++;
-	case HAMT_SUBTAG_NODE_BITMAP: {
-	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
-	    sp->abm = MAP_HEADER_VAL(hdrA);
-	    sp->srcB = boxed_val(nodeB);
-	    hdrB = *sp->srcB++;
-	    ASSERT(is_header(hdrB));
-	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
-		sp->bbm = 0xffff;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sp->bbm = MAP_HEADER_VAL(hdrB);
-		break;
-
-	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
-	    }
-	    break;
-	}
-	default:
-	    erl_exit(1, "bad primary tag %ld\r\n", nodeA);
-	}
-	break;
-    }
-    default:
-	erl_exit(1, "bad primary tag %ld\r\n", nodeA);
-    }
-
-    for (;;) {
-	if (is_value(res)) { /* We have a complete (sub-)tree or leaf */
-	    if (lvl == 0)
-		break;
-
-	    /* Pop from stack and continue build parent node */
-	    lvl--;
-	    sp = PSTACK_POP(s);
-	    sp->array[sp->ix++] = res;
-	    res = THE_NON_VALUE;
-	    if (sp->rbm) {
-		sp->srcA++;
-		sp->srcB++;
-		keepA = sp->keepA;
-	    }
-	} else { /* Start build a node */
-	    sp->ix = 0;
-	    sp->rbm = sp->abm | sp->bbm;
-	    ASSERT(!(sp->rbm == 0 && lvl > 0));
-	}
-
-	while (sp->rbm) {
-	    Uint32 next = sp->rbm & (sp->rbm-1);
-	    Uint32 bit = sp->rbm ^ next;
-	    sp->rbm = next;
-	    if (sp->abm & bit) {
-		if (sp->bbm & bit) {
-		    /* Bit clash. Push and resolve by recursive merge */
-		    if (sp->rbm) {
-			sp->keepA = keepA;
-		    }
-		    nodeA = *sp->srcA;
-		    nodeB = *sp->srcB;
-		    lvl++;
-		    sp = PSTACK_PUSH(s);
-		    goto recurse;
-		} else {
-		    sp->array[sp->ix++] = *sp->srcA++;
-		}
-	    } else {
-		ASSERT(sp->bbm & bit);
-		sp->array[sp->ix++] = *sp->srcB++;
-	    }
-	}
-
-	ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm));
-	if (lvl == 0) {
-	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
-	    hp = nhp;
-	    *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
-		     : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm));
-	    *hp++ = size;
-	} else {
-	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
-	    hp = nhp;
-	    *hp++ = (sp->ix == 16 ? make_arityval(16)
-		     : MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm));
-	}
-	memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
-	res = make_boxed(nhp);
-    }
-    PSTACK_DESTROY(s);
-    return res;
-}
-
-static int hash_cmp(Uint32 ha, Uint32 hb)
-{
-    int i;
-    for (i=0; i<8; i++) {
-	int cmp = (int)(ha & 0xF) - (int)(hb & 0xF);
-	if (cmp)
-	    return cmp;
-	ha >>= 4;
-	hb >>= 4;
-    }
-    return 0;
-}
-
-int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
-{
-    Eterm th[2];
-    unsigned lvl = 0;
-
-    if (ap && bp) {
-	ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0);
-	for (;;) {
-	    Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap));
-	    Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp));
-	    int cmp = hash_cmp(ha, hb);
-	    if (cmp)
-		return cmp;
-	    lvl += 8;
-	}
-    }
-    return ap ? -1 : 1;
-}
-
 /* hashmap:info/0 */
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 14f16e9050..8ae02e0183 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -77,6 +77,9 @@ typedef struct {
 
 static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
+static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB);
+static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -452,6 +455,15 @@ static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
     Uint jx = 0, ix = 0, lx, cx;
     Eterm res;
 
+    if (n == 0) {
+	Eterm *hp;
+	hp = HAlloc(p, 2);
+	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(0);
+	hp[1] = 0;
+
+	return make_hashmap(hp);
+    }
+
     /* sort and compact array (remove non-unique entries) */
     qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
 
@@ -504,8 +516,13 @@ static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
 	res = hashmap_from_sorted_unique_array(p, hxns, cx, 0);
     } else {
 	Eterm *hp;
-	 /* hash value has been swizzled, need to drag it down to get the
+
+	/* we only have one item, either because n was 1 or
+	 * because we hade multiples of the same key.
+	 *
+	 * hash value has been swizzled, need to drag it down to get the
 	 * correct slot. */
+
 	hp    = HAlloc(p, HAMT_HEAD_BITMAP_SZ(1));
 	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf));
 	hp[1] = 1;
@@ -769,8 +786,7 @@ static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
 	return -1;
 }
 
-/* maps:is_key/2
- */
+/* maps:is_key/2 */
 
 BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
@@ -779,8 +795,7 @@ BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* maps:keys/1
- */
+/* maps:keys/1 */
 
 BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
@@ -807,94 +822,396 @@ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
     }
     BIF_ERROR(BIF_P, BADARG);
 }
-/* maps:merge/2
- */
+/* maps:merge/2 */
 
 BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_1) && is_map(BIF_ARG_2)) {
-	Eterm *hp,*thp;
-	Eterm tup;
-	Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2;
-	map_t *mp1,*mp2,*mp_new;
-	Uint n1,n2,i1,i2,need,unused_size=0;
-	int c = 0;
-
-	mp1  = (map_t*)map_val(BIF_ARG_1);
-	mp2  = (map_t*)map_val(BIF_ARG_2);
-	n1   = map_get_size(mp1);
-	n2   = map_get_size(mp2);
-
-	need = MAP_HEADER_SIZE + 1 + 2*(n1 + n2);
-
-	hp     = HAlloc(BIF_P, need);
-	thp    = hp;
-	tup    = make_tuple(thp);
-	ks     = hp + 1; hp += 1 + n1 + n2;
-	mp_new = (map_t*)hp; hp += MAP_HEADER_SIZE;
-	vs     = hp; hp += n1 + n2;
-
-	mp_new->thing_word = MAP_HEADER;
-	mp_new->size = 0;
-	mp_new->keys = tup;
-
-	i1  = 0; i2 = 0;
-	ks1 = map_get_keys(mp1);
-	vs1 = map_get_values(mp1);
-	ks2 = map_get_keys(mp2);
-	vs2 = map_get_values(mp2);
-
-	while(i1 < n1 && i2 < n2) {
-	    c = CMP_TERM(ks1[i1],ks2[i2]);
-	    if ( c == 0) {
-		/* use righthand side arguments map value,
-		 * but advance both maps */
-		*ks++ = ks2[i2];
-		*vs++ = vs2[i2];
-		i1++, i2++, unused_size++;
-	    } else if ( c < 0) {
-		*ks++ = ks1[i1];
-		*vs++ = vs1[i1];
-		i1++;
-	    } else {
-		*ks++ = ks2[i2];
-		*vs++ = vs2[i2];
-		i2++;
-	    }
+    if (is_map(BIF_ARG_1)) {
+	if (is_map(BIF_ARG_2)) {
+	    BIF_RET(map_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
+	} else if (is_hashmap(BIF_ARG_2)) {
+	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
+	}
+    } else if (is_hashmap(BIF_ARG_1)) {
+	if (is_hashmap(BIF_ARG_2)) {
+	    BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
+	} else if (is_map(BIF_ARG_2)) {
+	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1));
 	}
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
 
-	/* copy remaining */
-	while (i1 < n1) {
+static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
+    Eterm *hp,*thp;
+    Eterm tup;
+    Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2;
+    map_t *mp1,*mp2,*mp_new;
+    Uint n1,n2,i1,i2,need,unused_size=0;
+    int c = 0;
+
+    mp1  = (map_t*)map_val(nodeA);
+    mp2  = (map_t*)map_val(nodeB);
+    n1   = map_get_size(mp1);
+    n2   = map_get_size(mp2);
+
+    need = MAP_HEADER_SIZE + 1 + 2*(n1 + n2);
+
+    hp     = HAlloc(p, need);
+    thp    = hp;
+    tup    = make_tuple(thp);
+    ks     = hp + 1; hp += 1 + n1 + n2;
+    mp_new = (map_t*)hp; hp += MAP_HEADER_SIZE;
+    vs     = hp; hp += n1 + n2;
+
+    mp_new->thing_word = MAP_HEADER;
+    mp_new->size = 0;
+    mp_new->keys = tup;
+
+    i1  = 0; i2 = 0;
+    ks1 = map_get_keys(mp1);
+    vs1 = map_get_values(mp1);
+    ks2 = map_get_keys(mp2);
+    vs2 = map_get_values(mp2);
+
+    while(i1 < n1 && i2 < n2) {
+	c = CMP_TERM(ks1[i1],ks2[i2]);
+	if ( c == 0) {
+	    /* use righthand side arguments map value,
+	     * but advance both maps */
+	    *ks++ = ks2[i2];
+	    *vs++ = vs2[i2];
+	    i1++, i2++, unused_size++;
+	} else if ( c < 0) {
 	    *ks++ = ks1[i1];
 	    *vs++ = vs1[i1];
 	    i1++;
-	}
-
-	while (i2 < n2) {
+	} else {
 	    *ks++ = ks2[i2];
 	    *vs++ = vs2[i2];
 	    i2++;
 	}
+    }
 
-	if (unused_size) {
-	    /* the key tuple is embedded in the heap, write a bignum to clear it.
-	     *
-	     * release values as normal since they are on the top of the heap
-	     * size = n1 + n1 - unused_size
-	     */
+    /* copy remaining */
+    while (i1 < n1) {
+	*ks++ = ks1[i1];
+	*vs++ = vs1[i1];
+	i1++;
+    }
+
+    while (i2 < n2) {
+	*ks++ = ks2[i2];
+	*vs++ = vs2[i2];
+	i2++;
+    }
+
+    if (unused_size) {
+	/* the key tuple is embedded in the heap, write a bignum to clear it.
+	 *
+	 * release values as normal since they are on the top of the heap
+	 * size = n1 + n1 - unused_size
+	 */
+
+	*ks = make_pos_bignum_header(unused_size - 1);
+	HRelease(p, vs + unused_size, vs);
+    }
+
+    mp_new->size = n1 + n2 - unused_size;
+    *thp = make_arityval(n1 + n2 - unused_size);
+
+    return make_map(mp_new);
+}
+
+static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) {
+    Eterm *ks, *vs, *hp, res;
+    map_t *mp;
+    Uint n, i;
+    hxnode_t *hxns;
+    Uint32 sw, hx;
+    Eterm tmp[2];
+
+    /* convert flat to tree */
+
+    ASSERT(is_map(flat));
+    ASSERT(is_hashmap(tree));
+
+    mp = (map_t*)map_val(flat);
+    n  = map_get_size(mp);
+
+    ks = map_get_keys(mp);
+    vs = map_get_values(mp);
+
+    hp = HAlloc(p, 2 * n);
+
+    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
+
+    for (i = 0; i < n; i++) {
+	hx = hashmap_restore_hash(tmp,0,ks[i]);
+	swizzle32(sw,hx);
+	hxns[i].hx   = sw;
+	hxns[i].val  = CONS(hp, ks[i], vs[i]); hp += 2;
+	hxns[i].skip = 1;
+	hxns[i].i    = i;
+    }
+
+    res = hashmap_from_unsorted_array(p, hxns, n);
+
+    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+    return swap_args ? hashmap_merge(p, tree, res) : hashmap_merge(p, res, tree);
+}
+
+#define HALLOC_EXTRA 200
+
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
+#define PSTACK_TYPE struct HashmapMergePStackType
+    struct HashmapMergePStackType {
+	Eterm *srcA, *srcB;
+	Uint32 abm, bbm, rbm;        /* node bitmaps */
+	int keepA;
+	int ix;
+	Eterm array[16];
+    };
+    PSTACK_DECLARE(s, 4);
+    struct HashmapMergePStackType* sp = PSTACK_PUSH(s);
+    Eterm *hp, *nhp;
+    Eterm hdrA, hdrB;
+    Eterm th[2];
+    Uint32 ahx, bhx;
+    Uint size;  /* total key-value counter */
+    int keepA = 0;
+    unsigned lvl = 0;
+    Eterm res = THE_NON_VALUE;
+
+    /*
+     * Strategy: Do depth-first traversal of both trees (at the same time)
+     * and merge each pair of nodes.
+     */
 
-	    *ks = make_pos_bignum_header(unused_size - 1);
-	    HRelease(BIF_P, vs + unused_size, vs);
+    {
+	hashmap_head_t* a = (hashmap_head_t*) hashmap_val(nodeA);
+	hashmap_head_t* b = (hashmap_head_t*) hashmap_val(nodeB);
+	size = a->size + b->size;
+    }
+
+recurse:
+
+    if (primary_tag(nodeA) == TAG_PRIMARY_BOXED &&
+	primary_tag(nodeB) == TAG_PRIMARY_LIST) {
+	/* Avoid implementing this combination by switching places */
+	Eterm tmp = nodeA;
+	nodeA = nodeB;
+	nodeB = tmp;
+	keepA = !keepA;
+    }
+
+    switch (primary_tag(nodeA)) {
+    case TAG_PRIMARY_LIST: {
+	sp->srcA = list_val(nodeA);
+	switch (primary_tag(nodeB)) {
+	case TAG_PRIMARY_LIST: { /* LEAF + LEAF */
+	    sp->srcB = list_val(nodeB);
+
+	    if (EQ(CAR(sp->srcA), CAR(sp->srcB))) {
+		--size;
+		res = keepA ? nodeA : nodeB;
+	    } else {
+		ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
+		bhx = hashmap_restore_hash(th, lvl, CAR(sp->srcB));
+		sp->abm = 1 << hashmap_index(ahx);
+		sp->bbm = 1 << hashmap_index(bhx);
+
+		sp->srcA = &nodeA;
+		sp->srcB = &nodeB;
+	    }
+	    break;
 	}
+	case TAG_PRIMARY_BOXED: { /* LEAF + NODE */
+	    sp->srcB = boxed_val(nodeB);
+	    ASSERT(is_header(*sp->srcB));
+	    hdrB = *sp->srcB++;
+
+	    ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
+	    sp->abm = 1 << hashmap_index(ahx);
+	    sp->srcA = &nodeA;
+	    switch(hdrB & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		sp->bbm = 0xffff;
+		break;
 
-	mp_new->size = n1 + n2 - unused_size;
-	*thp = make_arityval(n1 + n2 - unused_size);
+	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		sp->bbm = MAP_HEADER_VAL(hdrB);
+		break;
 
-	BIF_RET(make_map(mp_new));
+	    default:
+		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
+		break;
+	    }
+	    break;
+	}
+	default:
+	    erl_exit(1, "bad primary tag %ld\r\n", nodeB);
+	}
+	break;
     }
-    BIF_ERROR(BIF_P, BADARG);
+    case TAG_PRIMARY_BOXED: { /* NODE + NODE */
+	sp->srcA = boxed_val(nodeA);
+	hdrA = *sp->srcA++;
+	ASSERT(is_header(hdrA));
+	switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
+	case HAMT_SUBTAG_HEAD_ARRAY: sp->srcA++;
+	case HAMT_SUBTAG_NODE_ARRAY: {
+	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
+	    sp->abm = 0xffff;
+	    sp->srcB = boxed_val(nodeB);
+	    hdrB = *sp->srcB++;
+	    ASSERT(is_header(hdrB));
+	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		sp->bbm = 0xffff;
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		sp->bbm = MAP_HEADER_VAL(hdrB);
+		break;
+	    default:
+		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
+	    }
+	    break;
+	}
+	case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++;
+	case HAMT_SUBTAG_NODE_BITMAP: {
+	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
+	    sp->abm = MAP_HEADER_VAL(hdrA);
+	    sp->srcB = boxed_val(nodeB);
+	    hdrB = *sp->srcB++;
+	    ASSERT(is_header(hdrB));
+	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		sp->bbm = 0xffff;
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		sp->bbm = MAP_HEADER_VAL(hdrB);
+		break;
+
+	    default:
+		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
+	    }
+	    break;
+	}
+	default:
+	    erl_exit(1, "bad primary tag %ld\r\n", nodeA);
+	}
+	break;
+    }
+    default:
+	erl_exit(1, "bad primary tag %ld\r\n", nodeA);
+    }
+
+    for (;;) {
+	if (is_value(res)) { /* We have a complete (sub-)tree or leaf */
+	    if (lvl == 0)
+		break;
+
+	    /* Pop from stack and continue build parent node */
+	    lvl--;
+	    sp = PSTACK_POP(s);
+	    sp->array[sp->ix++] = res;
+	    res = THE_NON_VALUE;
+	    if (sp->rbm) {
+		sp->srcA++;
+		sp->srcB++;
+		keepA = sp->keepA;
+	    }
+	} else { /* Start build a node */
+	    sp->ix = 0;
+	    sp->rbm = sp->abm | sp->bbm;
+	    ASSERT(!(sp->rbm == 0 && lvl > 0));
+	}
+
+	while (sp->rbm) {
+	    Uint32 next = sp->rbm & (sp->rbm-1);
+	    Uint32 bit = sp->rbm ^ next;
+	    sp->rbm = next;
+	    if (sp->abm & bit) {
+		if (sp->bbm & bit) {
+		    /* Bit clash. Push and resolve by recursive merge */
+		    if (sp->rbm) {
+			sp->keepA = keepA;
+		    }
+		    nodeA = *sp->srcA;
+		    nodeB = *sp->srcB;
+		    lvl++;
+		    sp = PSTACK_PUSH(s);
+		    goto recurse;
+		} else {
+		    sp->array[sp->ix++] = *sp->srcA++;
+		}
+	    } else {
+		ASSERT(sp->bbm & bit);
+		sp->array[sp->ix++] = *sp->srcB++;
+	    }
+	}
+
+	ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm));
+	if (lvl == 0) {
+	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
+	    hp = nhp;
+	    *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
+		     : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm));
+	    *hp++ = size;
+	} else {
+	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
+	    hp = nhp;
+	    *hp++ = (sp->ix == 16 ? make_arityval(16)
+		     : MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm));
+	}
+	memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
+	res = make_boxed(nhp);
+    }
+    PSTACK_DESTROY(s);
+    return res;
 }
-/* maps:new/2
- */
+
+static int hash_cmp(Uint32 ha, Uint32 hb)
+{
+    int i;
+    for (i=0; i<8; i++) {
+	int cmp = (int)(ha & 0xF) - (int)(hb & 0xF);
+	if (cmp)
+	    return cmp;
+	ha >>= 4;
+	hb >>= 4;
+    }
+    return 0;
+}
+
+int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
+{
+    Eterm th[2];
+    unsigned lvl = 0;
+
+    if (ap && bp) {
+	ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0);
+	for (;;) {
+	    Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap));
+	    Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp));
+	    int cmp = hash_cmp(ha, hb);
+	    if (cmp)
+		return cmp;
+	    lvl += 8;
+	}
+    }
+    return ap ? -1 : 1;
+}
+
+/* maps:new/0 */
 
 BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
     Eterm* hp;
-- 
cgit v1.2.3


From efb521c69baccb8ab905595c222abf353c5c3283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Feb 2015 17:47:26 +0100
Subject: erts: Remove erl_hashmap.[ch] files

---
 erts/emulator/beam/copy.c            |  1 -
 erts/emulator/beam/erl_bif_guard.c   |  1 -
 erts/emulator/beam/erl_gc.c          |  1 -
 erts/emulator/beam/erl_hashmap.c     | 85 -------------------------------
 erts/emulator/beam/erl_hashmap.h     | 96 ------------------------------------
 erts/emulator/beam/erl_map.c         |  4 +-
 erts/emulator/beam/erl_map.h         | 67 +++++++++++++++++++++++++
 erts/emulator/beam/erl_printf_term.c |  1 -
 erts/emulator/beam/external.c        |  1 -
 erts/emulator/beam/utils.c           |  1 -
 10 files changed, 70 insertions(+), 188 deletions(-)
 delete mode 100644 erts/emulator/beam/erl_hashmap.c
 delete mode 100644 erts/emulator/beam/erl_hashmap.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 6294ba9412..5901c00d0a 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -33,7 +33,6 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "dtrace-wrapper.h"
-#include "erl_hashmap.h"
 
 static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*);
 
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index a5d1d3a5cb..bc0891422b 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -34,7 +34,6 @@
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_map.h"
-#include "erl_hashmap.h"
 
 static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live);
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index bdf7aa362e..d1a7ee113b 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -31,7 +31,6 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "erl_map.h"
-#include "erl_hashmap.h"
 #include "error.h"
 #include "big.h"
 #include "erl_gc.h"
diff --git a/erts/emulator/beam/erl_hashmap.c b/erts/emulator/beam/erl_hashmap.c
deleted file mode 100644
index 0c146137f5..0000000000
--- a/erts/emulator/beam/erl_hashmap.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2011. All Rights Reserved.
- *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- *
- * %CopyrightEnd%
- *
- * hashmaps are an adaption of Rich Hickeys Persistent HashMaps
- *   which were an adaption of Phil Bagwells - Hash Array Mapped Tries
- *
- * Author: Björn-Egil Dahlberg
- */
-/*
- * Ls = lists:seq(1,188888).
- * A = lists:foldl(fun(I,O) -> hashmap:put(I,I,O) end, hashmap:new(), Ls).
- * lists:foreach(fun(I) -> io:format("looking up ~p got ~p~n", [I, hashmap:get(I, A)]), I = hashmap:get(I,A) end, Ls).
- *
- * lists:foldl(fun(I,O) -> hashmap:put(I,I,O) end, hashmap:new(), lists:seq(1,7)).
- * lists:foldl(fun(I,O) -> hashmap:info(O), hashmap:put(I,I,O) end, hashmap:new(), lists:seq(1,5)).
- *
- */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_vm.h"
-#include "global.h"
-#include "erl_process.h"
-#include "error.h"
-#include "bif.h"
-
-#include "erl_map.h"
-#include "erl_hashmap.h"
-
-#if 0
-static char *format_binary(Uint64 x, char *b) {
-    int z;
-    b[64] = '\0';
-    for (z = 0; z < 64; z++) { 
-	b[63-z] = ((x>>z) & 0x1) ? '1' : '0'; 
-    }
-    return b;
-}
-#endif
-
-/* hashmap:new/0 */
-
-/* hashmap:put/3 */
-
-/* hashmap:update/3 */
-
-/* hashmap:to_list/1 */
-
-/* hashmap:from_list/1 */
-
-/* hashmap:get/2 */
-
-/* hashmap:find/2 */
-
-/* hashmap:remove/2 */
-
-/* hashmap:size/1 */
-
-/* erlang:is_hashmap/1 */
-
-/* hashmap:is_key/2 */
-
-/* hashmap:keys/1 */
-
-/* hashmap:values/1 */
-
-/* hashmap:info/0 */
diff --git a/erts/emulator/beam/erl_hashmap.h b/erts/emulator/beam/erl_hashmap.h
deleted file mode 100644
index 7ac33b34b0..0000000000
--- a/erts/emulator/beam/erl_hashmap.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2011. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-
-#ifndef __ERL_HASH_H__
-#define __ERL_HASH_H__
-
-#include "sys.h"
-#include "erl_term.h"
-
-int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
-
-/* HASH */
-
-/* hamt nodes v2.0
- * 
- * node :: leaf | array | bitmap
- * head
- */
-typedef struct hashmap_head_s {
-    Eterm thing_word;
-    Uint size;
-    Eterm items[1];
-} hashmap_head_t;
-
-/* thing_word tagscheme
- * Need two bits for map subtags
- *
- * Original HEADER representation:
- *
- *     aaaaaaaaaaaaaaaa aaaaaaaaaatttt00       arity:26, tag:4
- *
- * For maps we have:
- *
- *     vvvvvvvvvvvvvvvv aaaaaaaamm111100       val:16, arity:8, mtype:2
- *
- * unsure about trailing zeros
- *
- * map-tag:
- *     00 - flat map tag (non-hamt) -> val:16 = #items
- *     01 - map-node bitmap tag     -> val:16 = bitmap
- *     10 - map-head (array-node)   -> val:16 = 0xffff
- *     11 - map-head (bitmap-node)  -> val:16 = bitmap
- */
-
-/* erl_map.h stuff */
-
-#define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2)))
-
-#define MAKE_MAP_HEADER(Type,Arity,Val) \
-    (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_HASHMAP))
-
-#define MAP_HEADER_HAMT_HEAD_ARRAY \
-    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff)
-
-#define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \
-    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp)
-
-#define MAP_HEADER_HAMT_NODE_ARRAY \
-    make_arityval(16)
-
-#define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \
-    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp)
-
-#define HAMT_HEAD_EMPTY_SZ     (2)
-#define HAMT_NODE_ARRAY_SZ     (17)
-#define HAMT_HEAD_ARRAY_SZ     (18)
-#define HAMT_NODE_BITMAP_SZ(n) (1 + n)
-#define HAMT_HEAD_BITMAP_SZ(n) (2 + n)
-
-#define _HEADER_MAP_SUBTAG_MASK    (0xfc) /* 2 bits maps tag + 4 bits subtag + 2 ignore bits */
-/* SUBTAG_NODE_ARRAY is in fact a tuple with 16 elements */
-#define HAMT_SUBTAG_NODE_ARRAY   (((16 << _HEADER_ARITY_OFFS) | ARITYVAL_SUBTAG) & _HEADER_MAP_SUBTAG_MASK)
-#define HAMT_SUBTAG_NODE_BITMAP  ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
-#define HAMT_SUBTAG_HEAD_ARRAY   ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
-#define HAMT_SUBTAG_HEAD_BITMAP  ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
-
-#define hashmap_index(hash)      (((Uint32)hash) & 0xf)
-
-#endif
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 8ae02e0183..4242807933 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -16,6 +16,9 @@
  *
  * %CopyrightEnd%
  *
+ * hashmaps are an adaption of Rich Hickeys Persistent HashMaps
+ *   which were an adaption of Phil Bagwells - Hash Array Mapped Tries
+ *
  * Author: Björn-Egil Dahlberg
  */
 
@@ -31,7 +34,6 @@
 #include "bif.h"
 
 #include "erl_map.h"
-#include "erl_hashmap.h"
 
 /* BIFs
  *
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 428cfe9b63..7fddc9c240 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -91,6 +91,7 @@ int   erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
 int   erts_validate_and_sort_map(map_t* map);
 void  hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
+int   hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 Eterm erts_hashmap_get(Eterm key, Eterm map);
 Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
 
@@ -104,4 +105,70 @@ erts_maps_get(Eterm key, Eterm map);
 #  define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B)
 #endif
 
+/* hamt nodes v2.0
+ *
+ * node :: leaf | array | bitmap
+ * head
+ */
+typedef struct hashmap_head_s {
+    Eterm thing_word;
+    Uint size;
+    Eterm items[1];
+} hashmap_head_t;
+
+/* thing_word tagscheme
+ * Need two bits for map subtags
+ *
+ * Original HEADER representation:
+ *
+ *     aaaaaaaaaaaaaaaa aaaaaaaaaatttt00       arity:26, tag:4
+ *
+ * For maps we have:
+ *
+ *     vvvvvvvvvvvvvvvv aaaaaaaamm111100       val:16, arity:8, mtype:2
+ *
+ * unsure about trailing zeros
+ *
+ * map-tag:
+ *     00 - flat map tag (non-hamt) -> val:16 = #items
+ *     01 - map-node bitmap tag     -> val:16 = bitmap
+ *     10 - map-head (array-node)   -> val:16 = 0xffff
+ *     11 - map-head (bitmap-node)  -> val:16 = bitmap
+ */
+
+/* erl_map.h stuff */
+
+#define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2)))
+
+#define MAKE_MAP_HEADER(Type,Arity,Val) \
+    (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_HASHMAP))
+
+#define MAP_HEADER_HAMT_HEAD_ARRAY \
+    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff)
+
+#define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \
+    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp)
+
+#define MAP_HEADER_HAMT_NODE_ARRAY \
+    make_arityval(16)
+
+#define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \
+    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp)
+
+#define HAMT_HEAD_EMPTY_SZ     (2)
+#define HAMT_NODE_ARRAY_SZ     (17)
+#define HAMT_HEAD_ARRAY_SZ     (18)
+#define HAMT_NODE_BITMAP_SZ(n) (1 + n)
+#define HAMT_HEAD_BITMAP_SZ(n) (2 + n)
+
+#define _HEADER_MAP_SUBTAG_MASK    (0xfc) /* 2 bits maps tag + 4 bits subtag + 2 ignore bits */
+/* SUBTAG_NODE_ARRAY is in fact a tuple with 16 elements */
+#define HAMT_SUBTAG_NODE_ARRAY   (((16 << _HEADER_ARITY_OFFS) | ARITYVAL_SUBTAG) & _HEADER_MAP_SUBTAG_MASK)
+#define HAMT_SUBTAG_NODE_BITMAP  ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
+#define HAMT_SUBTAG_HEAD_ARRAY   ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
+#define HAMT_SUBTAG_HEAD_BITMAP  ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
+
+#define hashmap_index(hash)      (((Uint32)hash) & 0xf)
+
+
 #endif
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index b07b7785dd..81fd19693a 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -26,7 +26,6 @@
 #include "big.h"
 #include "erl_map.h"
 #include "erl_binary.h"
-#include "erl_hashmap.h"
 
 #define PRINT_CHAR(CNT, FN, ARG, C)					\
 do {									\
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9030b528a4..af8db4c265 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -45,7 +45,6 @@
 #include "erl_bits.h"
 #include "erl_zlib.h"
 #include "erl_map.h"
-#include "erl_hashmap.h"
 
 #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f595cfbacd..d234e8fc68 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -32,7 +32,6 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "erl_map.h"
-#include "erl_hashmap.h"
 #include "packet_parser.h"
 #include "erl_gc.h"
 #define ERTS_WANT_DB_INTERNAL__
-- 
cgit v1.2.3


From f7ee6181417f05670e2e733085dd7f038d6f30f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 26 Feb 2015 14:45:38 +0100
Subject: erts: Handle hashmap in get_map_element(s) instructions

---
 erts/emulator/beam/beam_emu.c | 160 +++++++++++++++++++++++++-----------------
 erts/emulator/beam/erl_map.c  |   5 +-
 erts/emulator/beam/erl_map.h  |   3 +-
 3 files changed, 99 insertions(+), 69 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 034436e975..9a05e26859 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2460,12 +2460,8 @@ do {								\
 
  OpCase(i_get_map_elements_fsI): {
     Eterm map;
-    map_t *mp;
-    Eterm field;
-    Eterm *ks;
-    Eterm *vs;
     BeamInstr *fs;
-    Uint sz,n;
+    Uint sz, n;
 
     GetArg1(1, map);
 
@@ -2473,36 +2469,56 @@ do {								\
      * i.e. that it follows a test is_map if needed.
      */
 
-    mp = (map_t *)map_val(map);
-    sz = map_get_size(mp);
-
-    if (sz == 0) {
-	SET_I((BeamInstr *) Arg(0));
-	goto get_map_elements_fail;
-    }
-
     n  = (Uint)Arg(2) / 2;
     fs = &Arg(3); /* pattern fields and target registers */
-    ks = map_get_keys(mp);
-    vs = map_get_values(mp);
 
-    while(sz) {
-	field = (Eterm)*fs;
-	if (EQ(field,*ks)) {
-	    PUT_TERM_REG(*vs, fs[1]);
-	    n--;
+    if (is_map(map)) {
+	map_t *mp;
+	Eterm *ks;
+	Eterm *vs;
+
+	mp = (map_t *)map_val(map);
+	sz = map_get_size(mp);
+
+	if (sz == 0) {
+	    SET_I((BeamInstr *) Arg(0));
+	    goto get_map_elements_fail;
+	}
+
+	ks = map_get_keys(mp);
+	vs = map_get_values(mp);
+
+	while(sz) {
+	    if (EQ((Eterm)*fs,*ks)) {
+		PUT_TERM_REG(*vs, fs[1]);
+		n--;
+		fs += 2;
+		/* no more values to fetch, we are done */
+		if (n == 0) break;
+	    }
+	    ks++; sz--;
+	    vs++;
+	}
+
+	if (n) {
+	    SET_I((BeamInstr *) Arg(0));
+	    goto get_map_elements_fail;
+	}
+    } else {
+	const Eterm *v;
+	Uint32 hx;
+	ASSERT(is_hashmap(map));
+	while(n--) {
+	    hx = hashmap_make_hash((Eterm)*fs);
+	    if ((v = erts_hashmap_get(hx,(Eterm)*fs, map)) == NULL) {
+		SET_I((BeamInstr *) Arg(0));
+		goto get_map_elements_fail;
+	    }
+	    PUT_TERM_REG(*v, fs[1]);
 	    fs += 2;
-	    /* no more values to fetch, we are done */
-	    if (n == 0) break;
 	}
-	ks++; sz--;
-	vs++;
     }
 
-    if (n) {
-	SET_I((BeamInstr *) Arg(0));
-	goto get_map_elements_fail;
-    }
 
     I += 4 + Arg(2);
 get_map_elements_fail:
@@ -6444,55 +6460,69 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
 
 static int has_not_map_field(Eterm map, Eterm key)
 {
-    map_t* mp;
-    Eterm* keys;
-    Uint i;
-    Uint n;
-
-    mp   = (map_t *)map_val(map);
-    keys = map_get_keys(mp);
-    n    = map_get_size(mp);
-    if (is_immed(key)) {
-	for (i = 0; i < n; i++) {
-	    if (keys[i] == key) {
-		return 0;
+    Uint32 hx;
+    if (is_map(map)) {
+	map_t* mp;
+	Eterm* keys;
+	Uint i;
+	Uint n;
+
+	mp   = (map_t *)map_val(map);
+	keys = map_get_keys(mp);
+	n    = map_get_size(mp);
+	if (is_immed(key)) {
+	    for (i = 0; i < n; i++) {
+		if (keys[i] == key) {
+		    return 0;
+		}
 	    }
-	}
-    } else {
-	for (i = 0; i <  n; i++) {
-	    if (EQ(keys[i], key)) {
-		return 0;
+	} else {
+	    for (i = 0; i <  n; i++) {
+		if (EQ(keys[i], key)) {
+		    return 0;
+		}
 	    }
 	}
+	return 1;
     }
-    return 1;
+    ASSERT(is_hashmap(map));
+    hx = hashmap_make_hash(key);
+    return erts_hashmap_get(hx,key,map) ? 0 : 1;
 }
 
 static Eterm get_map_element(Eterm map, Eterm key)
 {
-    map_t *mp;
-    Eterm* ks, *vs;
-    Uint i;
-    Uint n;
-
-    mp = (map_t *)map_val(map);
-    ks = map_get_keys(mp);
-    vs = map_get_values(mp);
-    n  = map_get_size(mp);
-    if (is_immed(key)) {
-	for (i = 0; i < n; i++) {
-	    if (ks[i] == key) {
-		return vs[i];
+    Uint32 hx;
+    const Eterm *vs;
+    if (is_map(map)) {
+	map_t *mp;
+	Eterm *ks;
+	Uint i;
+	Uint n;
+
+	mp = (map_t *)map_val(map);
+	ks = map_get_keys(mp);
+	vs = map_get_values(mp);
+	n  = map_get_size(mp);
+	if (is_immed(key)) {
+	    for (i = 0; i < n; i++) {
+		if (ks[i] == key) {
+		    return vs[i];
+		}
 	    }
-	}
-    } else {
-	for (i = 0; i < n; i++) {
-	    if (EQ(ks[i], key)) {
-		return vs[i];
+	} else {
+	    for (i = 0; i < n; i++) {
+		if (EQ(ks[i], key)) {
+		    return vs[i];
+		}
 	    }
 	}
+	return THE_NON_VALUE;
     }
-    return THE_NON_VALUE;
+    ASSERT(is_hashmap(map));
+    hx = hashmap_make_hash(key);
+    vs = erts_hashmap_get(hx,key,map);
+    return vs ? *vs : THE_NON_VALUE;
 }
 
 #define GET_TERM(term, dest)					\
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4242807933..2be4c9a730 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -77,7 +77,6 @@ typedef struct {
     Eterm  val;
 } hxnode_t;
 
-static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node);
 static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
 static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
@@ -199,7 +198,7 @@ erts_maps_get(Eterm key, Eterm map)
     ASSERT(is_hashmap(map));
     hx = hashmap_make_hash(key);
 
-    return hashmap_get(hx, key, map);
+    return erts_hashmap_get(hx, key, map);
 }
 
 BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
@@ -1660,7 +1659,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
     }
 }
 
-static const Eterm *hashmap_get(Uint32 hx, Eterm key, Eterm node) {
+const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     Eterm *ptr, hdr;
     Eterm th[2];
     Uint ix,slot, lvl = 0;
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 7fddc9c240..7175e653d9 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -92,8 +92,9 @@ int   erts_validate_and_sort_map(map_t* map);
 void  hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int   hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
-Eterm erts_hashmap_get(Eterm key, Eterm map);
 Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
+Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm val, Eterm node, int is_update);
+const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
 #if HALFWORD_HEAP
 const Eterm *
-- 
cgit v1.2.3


From 6507ec20f00acac025252bf7b7609eee1f593a84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 26 Feb 2015 16:53:23 +0100
Subject: erts: Make ESTACK usable through pointer

---
 erts/emulator/beam/global.h | 92 +++++++++++++++++++++++----------------------
 erts/emulator/beam/utils.c  |  4 +-
 2 files changed, 49 insertions(+), 47 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ad7e6fb89d..09bcf71a28 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -376,12 +376,13 @@ typedef struct ErtsEStack_ {
     Eterm* start;
     Eterm* sp;
     Eterm* end;
+    Eterm* edefault;
     ErtsAlcType_t alloc_type;
 }ErtsEStack;
 
 #define DEF_ESTACK_SIZE (16)
 
-void erl_grow_estack(ErtsEStack*, Eterm* def_stack, Uint need);
+void erl_grow_estack(ErtsEStack*, Uint need);
 #define ESTK_CONCAT(a,b) a##b
 #define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack)
 
@@ -391,22 +392,23 @@ void erl_grow_estack(ErtsEStack*, Eterm* def_stack, Uint need);
         ESTK_DEF_STACK(s),  /* start */ 		\
         ESTK_DEF_STACK(s),  /* sp */			\
         ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */	\
+        ESTK_DEF_STACK(s),  /* default */		\
         ERTS_ALC_T_ESTACK /* alloc_type */		\
     }
 
 #define ESTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
-    if (s.start != ESTK_DEF_STACK(s)) {					\
+    if ((s).start != ESTK_DEF_STACK(s)) {				\
 	erl_exit(1, "Internal error - trying to change allocator "	\
 		 "type of active estack\n");				\
     }									\
-    s.alloc_type = (t);							\
+    (s).alloc_type = (t);						\
  } while (0)
 
 #define DESTROY_ESTACK(s)				\
 do {							\
-    if (s.start != ESTK_DEF_STACK(s)) {			\
-	erts_free(s.alloc_type, s.start); 		\
+    if ((s).start != ESTK_DEF_STACK(s)) {		\
+	erts_free((s).alloc_type, (s).start); 		\
     }							\
 } while(0)
 
@@ -417,16 +419,16 @@ do {							\
  */
 #define ESTACK_SAVE(s,dst)\
 do {\
-    if (s.start == ESTK_DEF_STACK(s)) {\
+    if ((s).start == ESTK_DEF_STACK(s)) {\
 	UWord _wsz = ESTACK_COUNT(s);\
-	(dst)->start = erts_alloc(s.alloc_type,\
+	(dst)->start = erts_alloc((s).alloc_type,\
 				  DEF_ESTACK_SIZE * sizeof(Eterm));\
-	memcpy((dst)->start, s.start,_wsz*sizeof(Eterm));\
+	memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\
 	(dst)->sp = (dst)->start + _wsz;\
 	(dst)->end = (dst)->start + DEF_ESTACK_SIZE;\
-	(dst)->alloc_type = s.alloc_type;\
+	(dst)->alloc_type = (s).alloc_type;\
     } else\
-        *(dst) = s;\
+        *(dst) = (s);\
  } while (0)
 
 #define DESTROY_SAVED_ESTACK(estack)\
@@ -445,70 +447,70 @@ do {\
  */
 #define ESTACK_RESTORE(s, src)			\
 do {						\
-    ASSERT(s.start == ESTK_DEF_STACK(s));	\
-    s = *(src);  /* struct copy */		\
+    ASSERT((s).start == ESTK_DEF_STACK(s));	\
+    (s) = *(src);  /* struct copy */		\
     (src)->start = NULL;			\
-    ASSERT(s.sp >= s.start);			\
-    ASSERT(s.sp <= s.end);			\
+    ASSERT((s).sp >= (s).start);		\
+    ASSERT((s).sp <= (s).end);			\
 } while (0)
 
-#define ESTACK_IS_STATIC(s) (s.start == ESTK_DEF_STACK(s)))
+#define ESTACK_IS_STATIC(s) ((s).start == ESTK_DEF_STACK(s))
 
-#define ESTACK_PUSH(s, x)				\
-do {							\
-    if (s.sp == s.end) {				\
-	erl_grow_estack(&s, ESTK_DEF_STACK(s), 1); 	\
-    }							\
-    *s.sp++ = (x);					\
+#define ESTACK_PUSH(s, x)			\
+do {						\
+    if ((s).sp == (s).end) {			\
+	erl_grow_estack(&(s), 1); 		\
+    }						\
+    *(s).sp++ = (x);				\
 } while(0)
 
 #define ESTACK_PUSH2(s, x, y)			\
 do {						\
-    if (s.sp > s.end - 2) {			\
-	erl_grow_estack(&s, ESTK_DEF_STACK(s), 2); \
+    if ((s).sp > (s).end - 2) {			\
+	erl_grow_estack(&(s), 2);		\
     }						\
-    *s.sp++ = (x);				\
-    *s.sp++ = (y);				\
+    *(s).sp++ = (x);				\
+    *(s).sp++ = (y);				\
 } while(0)
 
 #define ESTACK_PUSH3(s, x, y, z)		\
 do {						\
-    if (s.sp > s.end - 3) {			\
-	erl_grow_estack(&s, ESTK_DEF_STACK(s), 3); \
+    if ((s).sp > (s).end - 3) {			\
+	erl_grow_estack(&s, 3); 		\
     }						\
-    *s.sp++ = (x);				\
-    *s.sp++ = (y);				\
-    *s.sp++ = (z);				\
+    *(s).sp++ = (x);				\
+    *(s).sp++ = (y);				\
+    *(s).sp++ = (z);				\
 } while(0)
 
 #define ESTACK_PUSH4(s, E1, E2, E3, E4)		\
 do {						\
-    if (s.sp > s.end - 4) {			\
-	erl_grow_estack(&s, ESTK_DEF_STACK(s)); \
+    if ((s).sp > (s).end - 4) {			\
+	erl_grow_estack(&s, 4);                 \
     }						\
-    *s.sp++ = (E1);				\
-    *s.sp++ = (E2);				\
-    *s.sp++ = (E3);				\
-    *s.sp++ = (E4);				\
+    *(s).sp++ = (E1);				\
+    *(s).sp++ = (E2);				\
+    *(s).sp++ = (E3);				\
+    *(s).sp++ = (E4);				\
 } while(0)
 
-#define ESTACK_RESERVE(s, push_cnt)                         \
-do {						            \
-    if (s.sp > s.end - (push_cnt)) {	                    \
-	erl_grow_estack(&s, ESTK_DEF_STACK(s), (push_cnt)); \
-    }						            \
+#define ESTACK_RESERVE(s, push_cnt)             \
+do {					        \
+    if ((s).sp > (s).end - (push_cnt)) {	\
+	erl_grow_estack(&(s), (push_cnt));	\
+    }					        \
 } while(0)
 
 /* Must be preceded by ESTACK_RESERVE */
 #define ESTACK_FAST_PUSH(s, x)				\
 do {							\
-    ASSERT(s.sp < s.end);                               \
+    ASSERT((s).sp < (s).end);                           \
     *s.sp++ = (x);					\
 } while(0)
 
-#define ESTACK_COUNT(s) (s.sp - s.start)
-#define ESTACK_ISEMPTY(s) (s.sp == s.start)
-#define ESTACK_POP(s) (*(--s.sp))
+#define ESTACK_COUNT(s) ((s).sp - (s).start)
+#define ESTACK_ISEMPTY(s) ((s).sp == (s).start)
+#define ESTACK_POP(s) (*(--(s).sp))
 
 
 /*
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d234e8fc68..c8a21adf17 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -190,7 +190,7 @@ erts_set_hole_marker(Eterm* ptr, Uint sz)
  * Helper function for the ESTACK macros defined in global.h.
  */
 void
-erl_grow_estack(ErtsEStack* s, Eterm* default_estack, Uint need)
+erl_grow_estack(ErtsEStack* s, Uint need)
 {
     Uint old_size = (s->end - s->start);
     Uint new_size;
@@ -201,7 +201,7 @@ erl_grow_estack(ErtsEStack* s, Eterm* default_estack, Uint need)
     else
 	new_size = ((need / old_size) + 2) * old_size;
 
-    if (s->start != default_estack) {
+    if (s->start != s->edefault) {
 	s->start = erts_realloc(s->alloc_type, s->start,
 				new_size*sizeof(Eterm));
     } else {
-- 
cgit v1.2.3


From cdfa304b1da0eb9fb270fe78ee6acfc0dad864fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 26 Feb 2015 17:10:40 +0100
Subject: erts: Split hashmap_insert to down and up

---
 erts/emulator/beam/erl_map.c | 113 +++++++++++++++++++++++++------------------
 erts/emulator/beam/erl_map.h |  27 +++++++----
 2 files changed, 85 insertions(+), 55 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 2be4c9a730..4ad33f98a7 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -77,7 +77,7 @@ typedef struct {
     Eterm  val;
 } hxnode_t;
 
-static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, Eterm node, int is_update);
+
 static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
@@ -1391,7 +1391,7 @@ found_key:
 
     ASSERT(is_hashmap(map));
     hx = hashmap_make_hash(key);
-    *res = hashmap_insert(p, hx, key, value, map, 1);
+    *res = erts_hashmap_insert(p, hx, key, value, map, 1);
     if (is_value(*res))
 	return 1;
 
@@ -1545,7 +1545,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
     ASSERT(is_hashmap(map));
 
     hx  = hashmap_make_hash(key);
-    res = hashmap_insert(p, hx, key, value, map, 0);
+    res = erts_hashmap_insert(p, hx, key, value, map, 0);
     ASSERT(is_hashmap(res));
 
     return res;
@@ -1730,16 +1730,34 @@ const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     return NULL;
 }
 
-static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
-			    Eterm node, int is_update) {
-    Eterm *hp = NULL, *nhp = NULL;
+Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
+			  Eterm map, int is_update) {
+    Uint size, upsz;
+    Eterm *hp, res = THE_NON_VALUE;
+    DECLARE_ESTACK(stack);
+    if (erts_hashmap_insert_down(hx, key, map, &size, &upsz, &stack, is_update)) {
+	hp  = HAlloc(p, size);
+	res = erts_hashmap_insert_up(hp, key, value, &upsz, &stack);
+    }
+
+    DESTROY_ESTACK(stack);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+    ERTS_HOLE_CHECK(p);
+
+    return res;
+}
+
+
+int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
+			     Uint *update_size, ErtsEStack *sp, int is_update) {
     Eterm *ptr;
-    Eterm hdr,res,ckey,fake;
+    Eterm hdr, ckey;
     Eterm th[2];
     Uint32 ix, cix, bp, hval, chx;
     Uint slot, lvl = 0, clvl;
-    Uint size = 0, n = 0, update_size = 1;
-    DECLARE_ESTACK(stack);
+    Uint size = 0, n = 0;
+
+    *update_size = 1;
 
     for (;;) {
 	switch(primary_tag(node)) {
@@ -1747,12 +1765,11 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 		ptr  = list_val(node);
 		ckey = CAR(ptr);
 		if (EQ(ckey, key)) {
-		    update_size = 0;
+		    *update_size = 0;
 		    goto unroll;
 		}
 		if (is_update) {
-		    res = THE_NON_VALUE;
-		    goto bail_out;
+		    return 0;
 		}
 		goto insert_subnodes;
 	    case TAG_PRIMARY_BOXED:
@@ -1765,14 +1782,14 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 			ix    = hashmap_index(hx);
 			hx    = hashmap_shift_hash(th,hx,lvl,key);
 			size += HAMT_NODE_ARRAY_SZ;
-			ESTACK_PUSH2(stack, ix, node);
+			ESTACK_PUSH2(*sp, ix, node);
 			node  = ptr[ix+1];
 			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			ix    = hashmap_index(hx);
 			hx    = hashmap_shift_hash(th,hx,lvl,key);
 			size += HAMT_HEAD_ARRAY_SZ;
-			ESTACK_PUSH2(stack, ix, node);
+			ESTACK_PUSH2(*sp, ix, node);
 			node  = ptr[ix+2];
 			break;
 		    case HAMT_SUBTAG_NODE_BITMAP:
@@ -1782,8 +1799,8 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 			slot = hashmap_bitcount(hval & (bp - 1));
 			n    = hashmap_bitcount(hval);
 
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
+			ESTACK_PUSH(*sp, n);
+			ESTACK_PUSH3(*sp, bp, slot, node);
 
 			/* occupied */
 			if (bp & hval) {
@@ -1795,8 +1812,7 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 			}
 			/* not occupied */
 			if (is_update) {
-			    res = THE_NON_VALUE;
-			    goto bail_out;
+			    return 0;
 			}
 			size += HAMT_NODE_BITMAP_SZ(n+1);
 			goto unroll;
@@ -1807,8 +1823,8 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 			slot = hashmap_bitcount(hval & (bp - 1));
 			n    = hashmap_bitcount(hval);
 
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
+			ESTACK_PUSH(*sp, n);
+			ESTACK_PUSH3(*sp, bp, slot, node);
 
 			/* occupied */
 			if (bp & hval) {
@@ -1820,8 +1836,7 @@ static Eterm hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 			}
 			/* not occupied */
 			if (is_update) {
-			    res = THE_NON_VALUE;
-			    goto bail_out;
+			    return 0;
 			}
 			size += HAMT_HEAD_BITMAP_SZ(n+1);
 			goto unroll;
@@ -1843,27 +1858,37 @@ insert_subnodes:
     cix   = hashmap_index(chx);
 
     while (cix == ix) {
-	ESTACK_PUSH(stack, 0);
-	ESTACK_PUSH3(stack, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
+	ESTACK_PUSH(*sp, 0);
+	ESTACK_PUSH3(*sp, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
 	size += HAMT_NODE_BITMAP_SZ(1);
 	hx    = hashmap_shift_hash(th,hx,lvl,key);
 	chx   = hashmap_shift_hash(th,chx,clvl,ckey);
 	ix    = hashmap_index(hx);
 	cix   = hashmap_index(chx);
     }
-    ESTACK_PUSH3(stack, cix, ix, node);
+    ESTACK_PUSH3(*sp, cix, ix, node);
 
 unroll:
-    size += 2;
-    hp  = HAlloc(p, size);
+    *sz = size + /* res cons */ 2;
+    return 1;
+}
+
+Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
+			     Uint *update_size, ErtsEStack *sp) {
+    Eterm node, fake, *ptr, hdr;
+    Eterm res;
+    Eterm *nhp = NULL;
+    Uint32 ix, cix, bp, hval;
+    Uint slot, n;
+
     res = CONS(hp, key, value); hp += 2;
 
     do {
-	node = ESTACK_POP(stack);
+	node = ESTACK_POP(*sp);
 	switch(primary_tag(node)) {
 	    case TAG_PRIMARY_LIST:
-		ix   = (Uint32) ESTACK_POP(stack);
-		cix  = (Uint32) ESTACK_POP(stack);
+		ix  = (Uint32) ESTACK_POP(*sp);
+		cix = (Uint32) ESTACK_POP(*sp);
 
 		nhp   = hp;
 		*hp++ = MAP_HEADER_HAMT_NODE_BITMAP((1 << ix) | (1 << cix));
@@ -1887,7 +1912,7 @@ unroll:
 
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
 		    case HAMT_SUBTAG_NODE_ARRAY:
-			slot  = (Uint)   ESTACK_POP(stack);
+			slot  = (Uint) ESTACK_POP(*sp);
 			nhp   = hp;
 			n     = HAMT_NODE_ARRAY_SZ;
 			while(n--) { *hp++ = *ptr++; }
@@ -1895,19 +1920,19 @@ unroll:
 			res = make_hashmap(nhp);
 			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
-			slot  = (Uint)   ESTACK_POP(stack);
+			slot  = (Uint) ESTACK_POP(*sp);
 			nhp   = hp;
 			n     = HAMT_HEAD_ARRAY_SZ - 2;
 			*hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++;
-			*hp++ = (*ptr++) + update_size;
+			*hp++ = (*ptr++) + *update_size;
 			while(n--) { *hp++ = *ptr++; }
 			nhp[slot+2] = res;
 			res = make_hashmap(nhp);
 			break;
 		    case HAMT_SUBTAG_NODE_BITMAP:
-			slot  = (Uint)   ESTACK_POP(stack);
-			bp    = (Uint32) ESTACK_POP(stack);
-			n     = (Uint32) ESTACK_POP(stack);
+			slot  = (Uint)   ESTACK_POP(*sp);
+			bp    = (Uint32) ESTACK_POP(*sp);
+			n     = (Uint32) ESTACK_POP(*sp);
 			hval  = MAP_HEADER_VAL(hdr);
 			nhp   = hp;
 			*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval | bp); ptr++;
@@ -1924,13 +1949,13 @@ unroll:
 			res = make_hashmap(nhp);
 			break;
 		    case HAMT_SUBTAG_HEAD_BITMAP:
-			slot  = (Uint)   ESTACK_POP(stack);
-			bp    = (Uint32) ESTACK_POP(stack);
-			n     = (Uint32) ESTACK_POP(stack);
+			slot  = (Uint)   ESTACK_POP(*sp);
+			bp    = (Uint32) ESTACK_POP(*sp);
+			n     = (Uint32) ESTACK_POP(*sp);
 			hval  = MAP_HEADER_VAL(hdr);
 			nhp   = hp;
 			*hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++;
-			*hp++ = (*ptr++) + update_size;
+			*hp++ = (*ptr++) + *update_size;
 
 			n -= slot;
 			while(slot--) { *hp++ = *ptr++; }
@@ -1953,13 +1978,9 @@ unroll:
 		break;
 	}
 
-    } while(!ESTACK_ISEMPTY(stack));
+    } while(!ESTACK_ISEMPTY(*sp));
 
-bail_out:
-    DESTROY_ESTACK(stack);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
-    ERTS_HOLE_CHECK(p);
-    return res;
+   return res;
 }
 
 static Eterm hashmap_keys(Process* p, Eterm node) {
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 7175e653d9..2d5c5c958c 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -85,16 +85,25 @@ typedef struct map_s {
 #define MAP_HEADER_SIZE        (sizeof(map_t) / sizeof(Eterm))
 
 struct ErtsWStack_;
-Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
-int   erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
-int   erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
-int   erts_validate_and_sort_map(map_t* map);
-void  hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
+struct ErtsEStack_;
+
+Eterm  erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
+int    erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
+int    erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
+
+Eterm  erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
+			   Eterm node, int is_update);
+int    erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
+			        Uint *upsz, struct ErtsEStack_ *sp, int is_update);
+Eterm  erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
+			      Uint *upsz, struct ErtsEStack_ *sp);
+
+int    erts_validate_and_sort_map(map_t* map);
+void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
-int   hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
-Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
-Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm val, Eterm node, int is_update);
-const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
+int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
+Eterm  erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
+const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
 #if HALFWORD_HEAP
 const Eterm *
-- 
cgit v1.2.3


From dea2faefbbea2b2e80a43600f47833d47a208b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 27 Feb 2015 14:51:57 +0100
Subject: erts: Handle hashmap in update/assoc instructions

---
 erts/emulator/beam/beam_emu.c | 79 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9a05e26859..1166b32a3b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6598,11 +6598,43 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Eterm new_key;
     Eterm* kp;
 
+    new_p = &Arg(5);
+    num_updates = Arg(4) / 2;
+
     if (is_not_map(map)) {
-	return THE_NON_VALUE;
+	Uint32 hx;
+	Eterm val;
+
+	/* apparently the compiler does not emit is_map instructions,
+	 * bad compiler */
+
+	if (is_not_hashmap(map))
+	    return THE_NON_VALUE;
+
+	res = map;
+	E = p->stop;
+	while(num_updates--) {
+	    /* assoc can't fail */
+	    GET_TERM(new_p[0], new_key);
+	    GET_TERM(new_p[1], val);
+	    hx = hashmap_make_hash(new_key);
+
+	    res = erts_hashmap_insert(p, hx, new_key, val, res,  0);
+	    if (p->mbuf) {
+		Uint live = Arg(3);
+		reg[live] = res;
+		erts_garbage_collect(p, 0, reg, live+1);
+		res       = reg[live];
+	    }
+
+	    E = p->stop;
+
+	    new_p += 2;
+	}
+	return res;
     }
 
-    old_mp = (map_t *) map_val(map);
+    old_mp  = (map_t *) map_val(map);
     num_old = map_get_size(old_mp);
 
     /*
@@ -6618,7 +6650,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      * update list are new).
      */
 
-    num_updates = Arg(4) / 2;
     need = 2*(num_old+num_updates) + 1 + MAP_HEADER_SIZE;
     if (HeapWordsLeft(p) < need) {
 	Uint live = Arg(3);
@@ -6665,7 +6696,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     old_vals = map_get_values(old_mp);
     old_keys = map_get_keys(old_mp);
 
-    new_p = &Arg(5);
     GET_TERM(*new_p, new_key);
     n = num_updates;
 
@@ -6776,8 +6806,44 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     BeamInstr* new_p;
     Eterm new_key;
 
+    new_p = &Arg(5);
+    n = Arg(4) / 2;		/* Number of values to be updated */
+    ASSERT(n > 0);
+
     if (is_not_map(map)) {
-	return THE_NON_VALUE;
+	Uint32 hx;
+	Eterm val;
+
+	/* apparently the compiler does not emit is_map instructions,
+	 * bad compiler */
+
+	if (is_not_hashmap(map))
+	    return THE_NON_VALUE;
+
+	res = map;
+	E = p->stop;
+	while(n--) {
+	    /* assoc can't fail */
+	    GET_TERM(new_p[0], new_key);
+	    GET_TERM(new_p[1], val);
+	    hx = hashmap_make_hash(new_key);
+
+	    res = erts_hashmap_insert(p, hx, new_key, val, res,  1);
+	    if (is_non_value(res))
+		return res;
+
+	    if (p->mbuf) {
+		Uint live = Arg(3);
+		reg[live] = res;
+		erts_garbage_collect(p, 0, reg, live+1);
+		res       = reg[live];
+	    }
+
+	    E = p->stop;
+
+	    new_p += 2;
+	}
+	return res;
     }
 
     old_mp = (map_t *) map_val(map);
@@ -6822,12 +6888,9 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     mp->keys = old_mp->keys;
 
     /* Get array of key/value pairs to be updated */
-    new_p = &Arg(5);
     GET_TERM(*new_p, new_key);
 
     /* Update all values */
-    n = Arg(4) / 2;		/* Number of values to be updated */
-    ASSERT(n > 0);
     for (i = 0; i < num_old; i++) {
 	if (!EQ(*old_keys, new_key)) {
 	    /* Not same keys */
-- 
cgit v1.2.3


From 861a865cea33e0f8d84148dfbd64ab00beb1a54a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 2 Mar 2015 15:46:37 +0100
Subject: erts: Add make_internal_hash

---
 erts/emulator/beam/erl_db_hash.c |   2 +-
 erts/emulator/beam/erl_map.h     |   2 +-
 erts/emulator/beam/erl_utils.h   |   1 +
 erts/emulator/beam/utils.c       | 394 ++++++++++++++++++++++++++++++++++++++-
 4 files changed, 393 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index c2157457a0..6cb2ecac15 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -174,7 +174,7 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
 /* optimised version of make_hash (normal case? atomic key) */
 #define MAKE_HASH(term) \
     ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
-      make_hash2(term)) % MAX_HASH)
+      make_internal_hash(term)) % MAX_HASH)
 
 #ifdef ERTS_SMP
 #  define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1)
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 2d5c5c958c..3544189936 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -57,7 +57,7 @@ typedef struct map_s {
 
 #define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
 #define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
-#define hashmap_make_hash(Key) make_hash2(Key)
+#define hashmap_make_hash(Key) make_internal_hash(Key)
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
     (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7)))
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index c32f8fd61c..c772a750f1 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -119,6 +119,7 @@ Uint32 make_broken_hash(Eterm);
 Uint32 block_hash(byte *, unsigned, Uint32);
 Uint32 make_hash2(Eterm);
 Uint32 make_hash(Eterm);
+Uint32 make_internal_hash(Eterm);
 
 void erts_save_emu_args(int argc, char **argv);
 Eterm erts_get_emu_args(struct process *c_p);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index c8a21adf17..67331a37f6 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1154,6 +1154,11 @@ make_hash2(Eterm term)
 #define HCONST_14 0xa708a81eUL
 #define HCONST_15 0x454021d7UL
 #define HCONST_16 0xe3779b90UL
+#define HCONST_17 0x81af1549UL
+#define HCONST_18 0x1fe68f02UL
+#define HCONST_19 0xbe1e08bbUL
+#define HCONST_20 0x5c558274UL
+#define HCONST_21 0xfa8cfc2dUL
 
 #define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF))
 #define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF))
@@ -1180,6 +1185,13 @@ make_hash2(Eterm term)
 	} while(0)
 
 #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
+
+#ifdef ARCH_64
+#  define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst)
+#else
+#  define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, Const)
+#endif
+
     /* Optimization. Simple cases before declaration of estack. */
     if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
 	switch (term & _TAG_IMMED1_MASK) {
@@ -1339,7 +1351,6 @@ make_hash2(Eterm term)
 	    case EXPORT_SUBTAG:
 	    {
 		Export* ep = *((Export **) (export_val(term) + 1));
-
 		UINT32_HASH_2
 		    (ep->code[2], 
 		     atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue,
@@ -1354,7 +1365,6 @@ make_hash2(Eterm term)
 	    {
 		ErlFunThing* funp = (ErlFunThing *) fun_val(term);
 		Uint num_free = funp->num_free;
-
 		UINT32_HASH_2
 		    (num_free, 
 		     atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue,
@@ -1558,10 +1568,386 @@ make_hash2(Eterm term)
 	}
     }
     }
+}
+
+/* Term hash function for internal use.
+ * Is not "portable" in any way betweem different VM instances.
+ *
+ * One IMPORTANT property must hold (for hamt).
+ * EVERY BIT of the term that is significant for equality (see EQ)
+ * must be used as input for the hash. Two different terms must always have a
+ * chance of hashing different when salted: h([Salt|A]) vs h([Salt|B]).
+ *
+ * This is why we can not use cached hash values for atoms for example.
+ */
+
+#define CONST_HASH(AConst)                              \
+do {  /* Lightweight mixing of constant (type info) */  \
+    hash ^= AConst;                                     \
+    hash = (hash << 17) ^ (hash >> (32-17));            \
+} while (0)
+
+Uint32
+make_internal_hash(Eterm term)
+{
+    Uint32 hash;
+    Uint32 hash_xor_pairs;
+
+    ERTS_UNDEF(hash_xor_pairs, 0);
+
+    /* Optimization. Simple cases before declaration of estack. */
+    if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
+        hash = 0;
+    #if ERTS_SIZEOF_ETERM == 8
+        UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST);
+    #elif ERTS_SIZEOF_ETERM == 4
+        UINT32_HASH(term, HCONST);
+    #else
+    #  error "No you don't"
+    #endif
+        return hash;
+    }
+    {
+    Eterm tmp;
+    DECLARE_ESTACK(s);
+
+    UseTmpHeapNoproc(2);
+    hash = 0;
+    for (;;) {
+	switch (primary_tag(term)) {
+	case TAG_PRIMARY_LIST:
+	{
+	    int c = 0;
+	    Uint32 sh = 0;
+	    Eterm* ptr = list_val(term);
+	    while (is_byte(*ptr)) {
+		/* Optimization for strings. */
+		sh = (sh << 8) + unsigned_val(*ptr);
+		if (c == 3) {
+		    UINT32_HASH(sh, HCONST_4);
+		    c = sh = 0;
+		} else {
+		    c++;
+		}
+		term = CDR(ptr);
+		if (is_not_list(term))
+		    break;
+		ptr = list_val(term);
+	    }
+	    if (c > 0)
+		UINT32_HASH(sh, HCONST_4);
+	    if (is_list(term)) {
+		tmp = CDR(ptr);
+                CONST_HASH(HCONST_17);  /* Hash CAR in cons cell */
+                ESTACK_PUSH(s, tmp);
+                if (is_not_list(tmp)) {
+                    ESTACK_PUSH(s, HASH_CDR);
+                }
+		term = CAR(ptr);
+	    }
+	}
+	break;
+	case TAG_PRIMARY_BOXED:
+	{
+	    Eterm hdr = *boxed_val(term);
+	    ASSERT(is_header(hdr));
+	    switch (hdr & _TAG_HEADER_MASK) {
+	    case ARITYVAL_SUBTAG:
+	    {
+		int i;
+		int arity = header_arity(hdr);
+		Eterm* elem = tuple_val(term);
+		UINT32_HASH(arity, HCONST_9);
+		if (arity == 0) /* Empty tuple */
+		    goto pop_next;
+		for (i = arity; ; i--) {
+		    term = elem[i];
+                    if (i == 1)
+                        break;
+                    ESTACK_PUSH(s, term);
+		}
+	    }
+	    break;
+	    case MAP_SUBTAG:
+	    {
+		map_t *mp = (map_t *)map_val(term);
+		int i;
+		int size  = map_get_size(mp);
+		Eterm *ks = map_get_keys(mp);
+		Eterm *vs = map_get_values(mp);
+		UINT32_HASH(size, HCONST_16);
+		if (size == 0) {
+		    goto pop_next;
+		}
+		/* We want a hash function that is *independent* of
+		 * the order in which keys and values are encountered.
+		 * We therefore calculate context independent hashes for all    				      .
+		 * key-value pairs and then xor them together.
+		 */
+		ESTACK_PUSH(s, hash_xor_pairs);
+		ESTACK_PUSH(s, hash);
+		ESTACK_PUSH(s, HASH_MAP_TAIL);
+		hash = 0;
+		hash_xor_pairs = 0;
+		for (i = size - 1; i >= 0; i--) {
+		    ESTACK_PUSH(s, HASH_MAP_PAIR);
+		    ESTACK_PUSH(s, vs[i]);
+                    ESTACK_PUSH(s, ks[i]);
+		}
+		goto pop_next;
+	    }
+	    break;
+            case HASHMAP_SUBTAG:
+            {
+                Eterm* ptr = boxed_val(term) + 1;
+                Uint size;
+                int i;
+                switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+                case HAMT_SUBTAG_HEAD_ARRAY:
+                case HAMT_SUBTAG_HEAD_BITMAP:
+                    size = *ptr++;
+                    UINT32_HASH(size, HCONST_16);
+                    if (size == 0)
+                        goto pop_next;
+                    ESTACK_PUSH(s, hash_xor_pairs);
+                    ESTACK_PUSH(s, hash);
+                    ESTACK_PUSH(s, HASH_MAP_TAIL);
+                    hash = 0;
+                    hash_xor_pairs = 0;
+                }
+                switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+                case HAMT_SUBTAG_HEAD_ARRAY:
+                case HAMT_SUBTAG_NODE_ARRAY:
+                    i = 16;
+                    break;
+                case HAMT_SUBTAG_HEAD_BITMAP:
+                case HAMT_SUBTAG_NODE_BITMAP:
+                    i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                    break;
+                default:
+                    erl_exit(1, "bad header");
+                }
+                while (i) {
+                    if (is_list(*ptr)) {
+                        Eterm* cons = list_val(*ptr);
+                        ESTACK_PUSH(s, HASH_MAP_PAIR);
+                        ESTACK_PUSH(s, CDR(cons));
+                        ESTACK_PUSH(s, CAR(cons));
+                    }
+                    else {
+                        ASSERT(is_boxed(*ptr));
+                        ESTACK_PUSH(s, *ptr);
+                    }
+                    i--; ptr++;
+                }
+                goto pop_next;
+            }
+            break;
+	    case EXPORT_SUBTAG:
+	    {
+		Export* ep = *((Export **) (export_val(term) + 1));
+                /* Assumes Export entries never moves */
+                POINTER_HASH(ep, HCONST_14);
+		goto pop_next;
+	    }
+
+	    case FUN_SUBTAG:
+	    {
+		ErlFunThing* funp = (ErlFunThing *) fun_val(term);
+		Uint num_free = funp->num_free;
+                UINT32_HASH_2(num_free, funp->fe->module, HCONST_20);
+                UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21);
+		if (num_free == 0) {
+		    goto pop_next;
+		} else {
+		    Eterm* bptr = funp->env + num_free - 1;
+		    while (num_free-- > 1) {
+			term = *bptr--;
+			ESTACK_PUSH(s, term);
+		    }
+		    term = *bptr;
+		}
+	    }
+	    break;
+	    case REFC_BINARY_SUBTAG:
+	    case HEAP_BINARY_SUBTAG:
+	    case SUB_BINARY_SUBTAG:
+	    {
+		byte* bptr;
+		unsigned sz = binary_size(term);
+		Uint32 con = HCONST_13 + hash;
+		Uint bitoffs;
+		Uint bitsize;
+
+		ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize);
+		if (sz == 0 && bitsize == 0) {
+		    hash = con;
+		} else {
+		    if (bitoffs == 0) {
+			hash = block_hash(bptr, sz, con);
+			if (bitsize > 0) {
+			    UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)),
+					  HCONST_15);
+			}
+		    } else {
+			byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP,
+							sz + (bitsize != 0));
+			erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize);
+			hash = block_hash(buf, sz, con);
+			if (bitsize > 0) {
+			    UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)),
+					  HCONST_15);
+			}
+			erts_free(ERTS_ALC_T_TMP, (void *) buf);
+		    }
+		}
+		goto pop_next;
+	    }
+	    break;
+	    case POS_BIG_SUBTAG:
+	    case NEG_BIG_SUBTAG:
+	    {
+		Eterm* ptr = big_val(term);
+		Uint i = 0;
+		Uint n = BIG_SIZE(ptr);
+		Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11;
+#if D_EXP == 16
+		do {
+		    Uint32 x, y;
+		    x = i < n ? BIG_DIGIT(ptr, i++) : 0;
+		    x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
+		    y = i < n ? BIG_DIGIT(ptr, i++) : 0;
+		    y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
+		    UINT32_HASH_2(x, y, con);
+		} while (i < n);
+#elif D_EXP == 32
+		do {
+		    Uint32 x, y;
+		    x = i < n ? BIG_DIGIT(ptr, i++) : 0;
+		    y = i < n ? BIG_DIGIT(ptr, i++) : 0;
+		    UINT32_HASH_2(x, y, con);
+		} while (i < n);
+#elif D_EXP == 64
+		do {
+		    Uint t;
+		    Uint32 x, y;
+                    ASSERT(i < n);
+		    t = BIG_DIGIT(ptr, i++);
+		    x = t & 0xffffffff;
+		    y = t >> 32;
+		    UINT32_HASH_2(x, y, con);
+		} while (i < n);
+#else
+#error "unsupported D_EXP size"
+#endif
+		goto pop_next;
+	    }
+	    break;
+	    case REF_SUBTAG:
+		UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7);
+                ASSERT(internal_ref_no_of_numbers(term) == 3);
+                UINT32_HASH_2(internal_ref_numbers(term)[1],
+                              internal_ref_numbers(term)[2], HCONST_8);
+                goto pop_next;
+
+            case EXTERNAL_REF_SUBTAG:
+            {
+                ExternalThing* thing = external_thing_ptr(term);
+
+                ASSERT(external_thing_ref_no_of_numbers(thing) == 3);
+                /*SVERK Is it really ok to hash node POINTER? */
+            #ifdef ARCH_64
+                POINTER_HASH(thing->node, HCONST_7);
+                UINT32_HASH(external_thing_ref_numbers(thing)[0], HCONST_7);
+            #else
+                UINT32_HASH_2(thing->node,
+                              external_thing_ref_numbers(thing)[0], HCONST_7);
+            #endif
+                UINT32_HASH_2(external_thing_ref_numbers(thing)[1],
+                              external_thing_ref_numbers(thing)[2], HCONST_8);
+                goto pop_next;
+            }
+            case EXTERNAL_PID_SUBTAG: {
+                ExternalThing* thing = external_thing_ptr(term);
+                /*SVERK Is it really ok to hash node POINTER? */
+            #ifdef ARCH_64
+                POINTER_HASH(thing->node, HCONST_5);
+                UINT32_HASH(thing->data.ui[0], HCONST_5);
+            #else
+                UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_5);
+            #endif
+		goto pop_next;
+            }
+	    case EXTERNAL_PORT_SUBTAG: {
+                ExternalThing* thing = external_thing_ptr(term);
+                /*SVERK Is it really ok to hash node POINTER? */
+            #ifdef ARCH_64
+                POINTER_HASH(thing->node, HCONST_6);
+                UINT32_HASH(thing->data.ui[0], HCONST_6);
+            #else
+                UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_6);
+            #endif
+		goto pop_next;
+            }
+	    case FLOAT_SUBTAG:
+	    {
+		FloatDef ff;
+		GET_DOUBLE(term, ff);
+		UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
+		goto pop_next;
+	    }
+
+	    default:
+		erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+	    }
+	}
+	break;
+        case TAG_PRIMARY_IMMED1:
+        #if ERTS_SIZEOF_ETERM == 8
+            UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST);
+        #else
+            UINT32_HASH(term, HCONST);
+        #endif
+            goto pop_next;
+
+	default:
+	    erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+
+	pop_next:
+	    if (ESTACK_ISEMPTY(s)) {
+		DESTROY_ESTACK(s);
+		UnUseTmpHeapNoproc(2);
+		return hash;
+	    }
+
+	    term = ESTACK_POP(s);
+
+	    switch (term) {
+		case HASH_MAP_TAIL: {
+		    hash = (Uint32) ESTACK_POP(s);
+                    UINT32_HASH(hash_xor_pairs, HCONST_19);
+		    hash_xor_pairs = (Uint32) ESTACK_POP(s);
+		    goto pop_next;
+		}
+		case HASH_MAP_PAIR:
+		    hash_xor_pairs ^= hash;
+		    hash = 0;
+		    goto pop_next;
+
+	        case HASH_CDR:
+		    CONST_HASH(HCONST_18);   /* Hash CDR i cons cell */
+		    goto pop_next;
+		default:
+		    break;
+	    }
+	}
+    }
+    }
 
+#undef CONST_HASH
 #undef HASH_MAP_TAIL
-#undef HASH_MAP_KEY
-#undef HASH_MAP_VAL
+#undef HASH_MAP_PAIR
+#undef HASH_CDR
 
 #undef UINT32_HASH_2
 #undef UINT32_HASH
-- 
cgit v1.2.3


From 18eabd06e64d21f83f466e99de35302b238ffec7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 2 Mar 2015 20:54:42 +0100
Subject: erts: Fix compare order of hamt vs other types

MAP_DEF and HASHMAP_DEF must have adjacent values
---
 erts/emulator/beam/erl_term.h | 30 ++++++++++++++++--------------
 erts/emulator/beam/utils.c    |  8 ++++----
 2 files changed, 20 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 264bf8bd74..953525a45c 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1111,20 +1111,22 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define LIST_DEF		0x1
 #define NIL_DEF			0x2
 #define MAP_DEF			0x3
-#define TUPLE_DEF		0x4
-#define PID_DEF			0x5
-#define EXTERNAL_PID_DEF	0x6
-#define PORT_DEF		0x7
-#define EXTERNAL_PORT_DEF	0x8
-#define EXPORT_DEF		0x9
-#define FUN_DEF			0xa
-#define REF_DEF			0xb
-#define EXTERNAL_REF_DEF	0xc
-#define ATOM_DEF		0xd
-#define FLOAT_DEF		0xe
-#define BIG_DEF			0xf
-#define SMALL_DEF		0x10
-#define HASHMAP_DEF		0x11
+#define HASHMAP_DEF		0x4
+#define TUPLE_DEF		0x5
+#define PID_DEF			0x6
+#define EXTERNAL_PID_DEF	0x7
+#define PORT_DEF		0x8
+#define EXTERNAL_PORT_DEF	0x9
+#define EXPORT_DEF		0xa
+#define FUN_DEF			0xb
+#define REF_DEF			0xc
+#define EXTERNAL_REF_DEF	0xd
+#define ATOM_DEF		0xe
+#define FLOAT_DEF		0xf
+#define BIG_DEF			0x10
+#define SMALL_DEF		0x11
+
+#define FIRST_VACANT_TAG_DEF    0x12
 
 #if ET_DEBUG
 extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 67331a37f6..3b35750fd7 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -830,10 +830,10 @@ Uint32 make_hash(Eterm term_arg)
     unsigned op;
 
     /* Must not collide with the real tag_val_def's: */
-#define MAKE_HASH_TUPLE_OP 0x11
-#define MAKE_HASH_TERM_ARRAY_OP 0x12
-#define MAKE_HASH_CDR_PRE_OP 0x13
-#define	MAKE_HASH_CDR_POST_OP 0x14
+#define MAKE_HASH_TUPLE_OP      (FIRST_VACANT_TAG_DEF)
+#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1)
+#define MAKE_HASH_CDR_PRE_OP    (FIRST_VACANT_TAG_DEF+2)
+#define MAKE_HASH_CDR_POST_OP   (FIRST_VACANT_TAG_DEF+3)
 
     /* 
     ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit 
-- 
cgit v1.2.3


From f316f5f718a9e60ed8dc394a6a0a1cb15147dd2a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 3 Mar 2015 10:46:10 +0100
Subject: erts: Fix erlang:hash and erlang:phash for hamt

by calling make_hash2.
---
 erts/emulator/beam/utils.c | 32 ++++----------------------------
 1 file changed, 4 insertions(+), 28 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 3b35750fd7..674dddc86f 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1013,21 +1013,9 @@ tail_recur:
 	    break;
 	}	
     case MAP_DEF:
+    case HASHMAP_DEF:
 	{
-	    map_t *mp = (map_t *)map_val(term);
-	    int size  = map_get_size(mp);
-	    Eterm *ks = map_get_keys(mp);
-	    Eterm *vs = map_get_values(mp);
-
-	    /* Use a prime with size to remedy some of
-	     * the {} and <<>> hash problems */
-	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size;
-	    if (size == 0)
-		break;
-
-	    /* push values first */
-	    WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
-	    WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
+	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
 	    break;
 	}
     case TUPLE_DEF: 
@@ -2164,21 +2152,9 @@ tail_recur:
 	break;
 
     case MAP_DEF:
+    case HASHMAP_DEF:
 	{
-	    map_t *mp = (map_t *)map_val(term);
-	    int size  = map_get_size(mp);
-	    Eterm *ks = map_get_keys(mp);
-	    Eterm *vs = map_get_values(mp);
-
-	    /* Use a prime with size to remedy some of
-	     * the {} and <<>> hash problems */
-	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size;
-	    if (size == 0)
-		break;
-
-	    /* push values first */
-	    WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
-	    WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP);
+	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
 	    break;
 	}
     case TUPLE_DEF: 
-- 
cgit v1.2.3


From eb6f1d4923899fcb29a5d3c312f14489e7846a37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 3 Mar 2015 09:35:10 +0100
Subject: erts: Ensure maps becomes hashmaps in maps:merge/2

Maps should become hashmaps when merged size exceeds small limit size.
---
 erts/emulator/beam/erl_map.c | 49 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4ad33f98a7..1703f86b0e 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -830,12 +830,14 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
 	if (is_map(BIF_ARG_2)) {
 	    BIF_RET(map_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
 	} else if (is_hashmap(BIF_ARG_2)) {
+	    /* Will always become a tree */
 	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
 	}
     } else if (is_hashmap(BIF_ARG_1)) {
 	if (is_hashmap(BIF_ARG_2)) {
 	    BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
 	} else if (is_map(BIF_ARG_2)) {
+	    /* Will always become a tree */
 	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1));
 	}
     }
@@ -847,7 +849,7 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     Eterm tup;
     Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2;
     map_t *mp1,*mp2,*mp_new;
-    Uint n1,n2,i1,i2,need,unused_size=0;
+    Uint n,n1,n2,i1,i2,need,unused_size=0;
     int c = 0;
 
     mp1  = (map_t*)map_val(nodeA);
@@ -876,13 +878,13 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 
     while(i1 < n1 && i2 < n2) {
 	c = CMP_TERM(ks1[i1],ks2[i2]);
-	if ( c == 0) {
+	if (c == 0) {
 	    /* use righthand side arguments map value,
 	     * but advance both maps */
 	    *ks++ = ks2[i2];
 	    *vs++ = vs2[i2];
 	    i1++, i2++, unused_size++;
-	} else if ( c < 0) {
+	} else if (c < 0) {
 	    *ks++ = ks1[i1];
 	    *vs++ = vs1[i1];
 	    i1++;
@@ -917,8 +919,42 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 	HRelease(p, vs + unused_size, vs);
     }
 
-    mp_new->size = n1 + n2 - unused_size;
-    *thp = make_arityval(n1 + n2 - unused_size);
+    n = n1 + n2 - unused_size;
+    *thp = make_arityval(n);
+
+    /* Reshape map to a hashmap if the map exceeds the limit */
+
+    if (n > MAP_SMALL_MAP_LIMIT) {
+	Uint32 hx,sw;
+	Uint i;
+	Eterm res;
+	hxnode_t *hxns;
+
+	ks = map_get_keys(mp_new);
+	vs = map_get_values(mp_new);
+
+	hp = HAlloc(p, 2 * n);
+
+	hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP,n * sizeof(hxnode_t));
+
+	for (i = 0; i < n; i++) {
+	    hx = hashmap_make_hash(ks[i]);
+	    swizzle32(sw,hx);
+	    hxns[i].hx   = sw;
+	    hxns[i].val  = CONS(hp, ks[i], vs[i]); hp += 2;
+	    hxns[i].skip = 1;
+	    hxns[i].i    = i;
+	}
+
+	res = hashmap_from_unsorted_array(p, hxns, n);
+
+	erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+	return res;
+    }
+
+    mp_new->size = n;
 
     return make_map(mp_new);
 }
@@ -929,7 +965,6 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
     Uint n, i;
     hxnode_t *hxns;
     Uint32 sw, hx;
-    Eterm tmp[2];
 
     /* convert flat to tree */
 
@@ -947,7 +982,7 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
     hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
 
     for (i = 0; i < n; i++) {
-	hx = hashmap_restore_hash(tmp,0,ks[i]);
+	hx = hashmap_make_hash(ks[i]);
 	swizzle32(sw,hx);
 	hxns[i].hx   = sw;
 	hxns[i].val  = CONS(hp, ks[i], vs[i]); hp += 2;
-- 
cgit v1.2.3


From 9794b73998690178538a1dfc193565dcd477b4fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 3 Mar 2015 12:46:05 +0100
Subject: erts: Fix update_map_assoc instruction

Did not build a hashmap once the small limit was exceeded.
---
 erts/emulator/beam/beam_emu.c | 13 ++++++++++++-
 erts/emulator/beam/erl_map.c  | 31 +++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_map.h  |  1 +
 3 files changed, 44 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1166b32a3b..c753e57ddc 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6781,8 +6781,19 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     n = kp - p->htop - 1;	/* Actual number of keys/values */
     *p->htop = make_arityval(n);
+    p->htop  = hp;
     mp->size = n;
-    p->htop = hp;
+
+    /* The expensive case, need to build a hashmap */
+    if (n > MAP_SMALL_MAP_LIMIT) {
+	res = erts_hashmap_from_ks_and_vs(p,map_get_keys(mp),map_get_values(mp),n);
+	if (p->mbuf) {
+	    Uint live = Arg(3);
+	    reg[live] = res;
+	    erts_garbage_collect(p, 0, reg, live+1);
+	    res       = reg[live];
+	}
+    }
     return res;
 }
 
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 1703f86b0e..e26b97d75c 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -452,6 +452,37 @@ Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
     return res;
 }
 
+
+Eterm erts_hashmap_from_ks_and_vs(Process *p, Eterm *ks, Eterm *vs, Uint n) {
+    Uint32 sw, hx;
+    Uint i;
+    hxnode_t *hxns;
+    Eterm *hp, res;
+
+    ASSERT(n > 0);
+
+    hp = HAlloc(p, (2 * n));
+
+    /* create tmp hx values and leaf ptrs */
+    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
+
+    for(i = 0; i < n; i++) {
+	hx = hashmap_make_hash(ks[i]);
+	swizzle32(sw,hx);
+	hxns[i].hx   = sw;
+	hxns[i].val  = CONS(hp, ks[i], vs[i]); hp += 2;
+	hxns[i].skip = 1; /* will be reassigned in from_array */
+	hxns[i].i    = i;
+    }
+
+    res = hashmap_from_unsorted_array(p, hxns, n);
+
+    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+
+    return res;
+}
+
 static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
     Uint jx = 0, ix = 0, lx, cx;
     Eterm res;
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 3544189936..6a7b29fe86 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -103,6 +103,7 @@ void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 Eterm  erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
+Eterm  erts_hashmap_from_ks_and_vs(Process *p, Eterm *ks, Eterm *vs, Uint n);
 const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
 #if HALFWORD_HEAP
-- 
cgit v1.2.3


From de95f407aba2c8b7bf2a5827842722581aacfd6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 3 Mar 2015 13:34:14 +0100
Subject: erts: Refactor hashmap_from_ks_and_vs

Use extra key and value if needed.
---
 erts/emulator/beam/erl_map.c | 54 +++++++++++++++-----------------------------
 erts/emulator/beam/erl_map.h |  7 +++++-
 2 files changed, 24 insertions(+), 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index e26b97d75c..97f94d90ac 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -453,18 +453,18 @@ Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
 }
 
 
-Eterm erts_hashmap_from_ks_and_vs(Process *p, Eterm *ks, Eterm *vs, Uint n) {
+Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n,
+					Eterm key, Eterm value) {
     Uint32 sw, hx;
-    Uint i;
+    Uint i,sz;
     hxnode_t *hxns;
     Eterm *hp, res;
 
-    ASSERT(n > 0);
-
-    hp = HAlloc(p, (2 * n));
+    sz = (key == THE_NON_VALUE) ? n : (n + 1);
+    hp = HAlloc(p, 2 * sz);
 
     /* create tmp hx values and leaf ptrs */
-    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
+    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, sz * sizeof(hxnode_t));
 
     for(i = 0; i < n; i++) {
 	hx = hashmap_make_hash(ks[i]);
@@ -475,7 +475,16 @@ Eterm erts_hashmap_from_ks_and_vs(Process *p, Eterm *ks, Eterm *vs, Uint n) {
 	hxns[i].i    = i;
     }
 
-    res = hashmap_from_unsorted_array(p, hxns, n);
+    if (key != THE_NON_VALUE) {
+	hx = hashmap_make_hash(key);
+	swizzle32(sw,hx);
+	hxns[i].hx   = sw;
+	hxns[i].val  = CONS(hp, key, value); hp += 2;
+	hxns[i].skip = 1;
+	hxns[i].i    = i;
+    }
+
+    res = hashmap_from_unsorted_array(p, hxns, sz);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1531,39 +1540,12 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 
 	/* the map will grow */
 
-	if (n >= MAP_SMALL_MAP_LIMIT) {
-	    hxnode_t *hxns;
-	    Uint32 sw, hx;
-	    Eterm tmp[2];
-
+	if (n > MAP_SMALL_MAP_LIMIT) {
 	    HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
-	    hp = HAlloc(p, (2 * (n + 1)));
 	    ks = map_get_keys(mp);
 	    vs = map_get_values(mp);
 
-	    /* create tmp hx values and leaf ptrs */
-	    hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(hxnode_t));
-
-	    for (i = 0; i < n; i++) {
-		hx = hashmap_restore_hash(tmp,0,ks[i]);
-		swizzle32(sw,hx);
-		hxns[i].hx   = sw;
-		hxns[i].val  = CONS(hp, ks[i], vs[i]); hp += 2;
-		hxns[i].skip = 1;
-		hxns[i].i    = i;
-	    }
-
-	    hx = hashmap_restore_hash(tmp,0,key);
-	    swizzle32(sw,hx);
-	    hxns[i].hx   = sw;
-	    hxns[i].val  = CONS(hp, key, value); hp += 2;
-	    hxns[i].skip = 1;
-	    hxns[i].i    = n;
-
-	    res = hashmap_from_unsorted_array(p, hxns, n + 1);
-
-	    erts_free(ERTS_ALC_T_TMP, (void *) hxns);
-	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
+	    res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value);
 
 	    return res;
 	}
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 6a7b29fe86..d1e227183f 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -103,9 +103,14 @@ void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 Eterm  erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
-Eterm  erts_hashmap_from_ks_and_vs(Process *p, Eterm *ks, Eterm *vs, Uint n);
 const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
+#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \
+    erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE);
+
+Eterm  erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n,
+					 Eterm k, Eterm v);
+
 #if HALFWORD_HEAP
 const Eterm *
 erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base);
-- 
cgit v1.2.3


From 3697ee925251edfe7a7ac2ed483039f414830012 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 3 Mar 2015 17:16:51 +0100
Subject: erts: Fix instruction new_map

---
 erts/emulator/beam/beam_emu.c | 22 +++++++++++++++++++++-
 erts/emulator/beam/erl_map.c  |  6 +++---
 2 files changed, 24 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c753e57ddc..8288eb9798 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6557,6 +6557,27 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     BeamInstr *ptr;
     map_t *mp;
 
+    ptr = &Arg(4);
+
+    if (n > MAP_SMALL_MAP_LIMIT) {
+	if (HeapWordsLeft(p) < n) {
+	    erts_garbage_collect(p, n, reg, Arg(2));
+	}
+
+	mhp = p->htop;
+	thp = p->htop;
+	E   = p->stop;
+
+	for (i = 0; i < n/2; i++) {
+	    GET_TERM(*ptr++, *mhp++);
+	    GET_TERM(*ptr++, *mhp++);
+	}
+
+	p->htop = mhp;
+
+	return erts_hashmap_from_array(p, thp, n/2);
+    }
+
     if (HeapWordsLeft(p) < need) {
 	erts_garbage_collect(p, need, reg, Arg(2));
     }
@@ -6564,7 +6585,6 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     thp    = p->htop;
     mhp    = thp + 1 + n/2;
     E      = p->stop;
-    ptr    = &Arg(4);
     keys   = make_tuple(thp);
     *thp++ = make_arityval(n/2);
 
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 97f94d90ac..ea041e2303 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -426,7 +426,6 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
 }
 
 Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
-    Eterm tmp[2];
     Uint32 sw, hx;
     Uint ix;
     hxnode_t *hxns;
@@ -436,12 +435,13 @@ Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
     hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t));
 
     for (ix = 0; ix < n; ix++) {
-	hx  = hashmap_restore_hash(tmp,0,CAR(list_val(leafs[ix])));
+	hx  = hashmap_make_hash(*leafs);
 	swizzle32(sw,hx);
 	hxns[ix].hx   = sw;
-	hxns[ix].val  = leafs[ix];
+	hxns[ix].val  = make_list(leafs);
 	hxns[ix].skip = 1;
 	hxns[ix].i    = ix;
+	leafs += 2;
     }
 
     res = hashmap_from_unsorted_array(p, hxns, n);
-- 
cgit v1.2.3


From 412c30b4dcdb8388b7472c8abb328f2a2fce92c0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 3 Mar 2015 19:26:03 +0100
Subject: erts: Fix various map vs hamt size bugs

---
 erts/emulator/beam/beam_emu.c |  2 +-
 erts/emulator/beam/erl_map.c  | 36 ++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_term.h |  4 ++--
 3 files changed, 39 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8288eb9798..15591c3cad 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6559,7 +6559,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
 
     ptr = &Arg(4);
 
-    if (n > MAP_SMALL_MAP_LIMIT) {
+    if (n > 2*MAP_SMALL_MAP_LIMIT) {
 	if (HeapWordsLeft(p) < n) {
 	    erts_garbage_collect(p, n, reg, Arg(2));
 	}
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ea041e2303..0cf1c0e59d 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -422,6 +422,41 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
 
+    if (hashmap_size(res) <= MAP_SMALL_MAP_LIMIT) {
+        DECLARE_WSTACK(wstack);
+	Eterm *kv, *ks, *vs;
+	map_t *mp;
+	Eterm keys;
+        Uint n = hashmap_size(res);
+
+	/* build flat structure */
+	hp    = HAlloc(p, 3 + 1 + (2 * n));
+	keys  = make_tuple(hp);
+	*hp++ = make_arityval(n);
+	ks    = hp;
+	hp   += n;
+	mp    = (map_t*)hp;
+	hp   += MAP_HEADER_SIZE;
+	vs    = hp;
+
+	mp->thing_word = MAP_HEADER;
+	mp->size = n;
+	mp->keys = keys;
+
+	hashmap_iterator_init(&wstack, res);
+
+	while ((kv=hashmap_iterator_next(&wstack)) != NULL) {
+	    *ks++ = CAR(kv);
+	    *vs++ = CDR(kv);
+	}
+
+	/* it cannot have multiple keys */
+	erts_validate_and_sort_map(mp);
+
+	DESTROY_WSTACK(wstack);
+	return make_map(mp);
+    }
+
     return res;
 }
 
@@ -461,6 +496,7 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
     Eterm *hp, res;
 
     sz = (key == THE_NON_VALUE) ? n : (n + 1);
+    ASSERT(sz > MAP_SMALL_MAP_LIMIT);
     hp = HAlloc(p, 2 * sz);
 
     /* create tmp hx values and leaf ptrs */
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 953525a45c..6d97344868 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1110,8 +1110,8 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define BINARY_DEF		0x0
 #define LIST_DEF		0x1
 #define NIL_DEF			0x2
-#define MAP_DEF			0x3
-#define HASHMAP_DEF		0x4
+#define HASHMAP_DEF		0x3
+#define MAP_DEF			0x4
 #define TUPLE_DEF		0x5
 #define PID_DEF			0x6
 #define EXTERNAL_PID_DEF	0x7
-- 
cgit v1.2.3


From f8dbf0c0ff3bd68d720faca230356d281e4e3e42 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 3 Mar 2015 19:32:00 +0100
Subject: First stab at binary_to_term for hamt

with over estimation of heap size.
---
 erts/emulator/beam/beam_emu.c    |   4 +-
 erts/emulator/beam/erl_map.c     |  66 +++++++++-------
 erts/emulator/beam/erl_map.h     |   4 +-
 erts/emulator/beam/erl_message.c |  13 ++++
 erts/emulator/beam/erl_message.h |  17 ++++
 erts/emulator/beam/external.c    | 163 ++++++++++++++++++++++++++++-----------
 6 files changed, 189 insertions(+), 78 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 15591c3cad..4f57037507 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6556,6 +6556,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     Eterm *E;
     BeamInstr *ptr;
     map_t *mp;
+    ErtsHeapFactory factory;
 
     ptr = &Arg(4);
 
@@ -6575,7 +6576,8 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
 
 	p->htop = mhp;
 
-	return erts_hashmap_from_array(p, thp, n/2);
+        factory.p = p;
+	return erts_hashmap_from_array(&factory, thp, n/2);
     }
 
     if (HeapWordsLeft(p) < need) {
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 0cf1c0e59d..eacdd0b6c3 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -87,9 +87,9 @@ static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm map_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
-static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n);
-static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
-static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root);
+static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n);
+static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root);
+static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root);
 static Eterm hashmap_info(Process *p, Eterm node);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 static int hxnodecmp(hxnode_t* a, hxnode_t* b);
@@ -396,6 +396,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     Uint32 sw, hx;
     Uint ix = 0;
     hxnode_t *hxns;
+    ErtsHeapFactory factory;
 
     ASSERT(size > 0);
 
@@ -417,7 +418,8 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
 	item = CDR(list_val(item));
     }
 
-    res = hashmap_from_unsorted_array(p, hxns, size);
+    factory.p = p;
+    res = hashmap_from_unsorted_array(&factory, hxns, size);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -460,7 +462,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     return res;
 }
 
-Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
+Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n) {
     Uint32 sw, hx;
     Uint ix;
     hxnode_t *hxns;
@@ -479,10 +481,9 @@ Eterm erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n) {
 	leafs += 2;
     }
 
-    res = hashmap_from_unsorted_array(p, hxns, n);
+    res = hashmap_from_unsorted_array(factory, hxns, n);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
 
     return res;
 }
@@ -493,6 +494,7 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
     Uint32 sw, hx;
     Uint i,sz;
     hxnode_t *hxns;
+    ErtsHeapFactory factory;
     Eterm *hp, res;
 
     sz = (key == THE_NON_VALUE) ? n : (n + 1);
@@ -520,7 +522,8 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
 	hxns[i].i    = i;
     }
 
-    res = hashmap_from_unsorted_array(p, hxns, sz);
+    factory.p = p;
+    res = hashmap_from_unsorted_array(&factory, hxns, sz);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -528,13 +531,14 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
     return res;
 }
 
-static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
+static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory,
+                                         hxnode_t *hxns, Uint n) {
     Uint jx = 0, ix = 0, lx, cx;
     Eterm res;
 
     if (n == 0) {
 	Eterm *hp;
-	hp = HAlloc(p, 2);
+	hp = erts_produce_heap(factory, 2, 0);
 	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(0);
 	hp[1] = 0;
 
@@ -590,7 +594,7 @@ static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
 
     if (cx > 1) {
 	/* recursive decompose array */
-	res = hashmap_from_sorted_unique_array(p, hxns, cx, 0);
+	res = hashmap_from_sorted_unique_array(factory, hxns, cx, 0);
     } else {
 	Eterm *hp;
 
@@ -600,7 +604,7 @@ static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
 	 * hash value has been swizzled, need to drag it down to get the
 	 * correct slot. */
 
-	hp    = HAlloc(p, HAMT_HEAD_BITMAP_SZ(1));
+	hp    = erts_produce_heap(factory, HAMT_HEAD_BITMAP_SZ(1), 0);
 	hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf));
 	hp[1] = 1;
 	hp[2] = hxns[0].val;
@@ -610,7 +614,8 @@ static Eterm hashmap_from_unsorted_array(Process *p, hxnode_t *hxns, Uint n) {
     return res;
 }
 
-static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n, int lvl) {
+static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory,
+                                              hxnode_t *hxns, Uint n, int lvl) {
     Eterm res = NIL;
     Uint i,ix,jx,elems;
     Uint32 sw, hx;
@@ -640,7 +645,7 @@ static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n
 	    qsort(tmp, jx - ix, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp);
 
 	    hxns[ix].skip = jx - ix;
-	    hxns[ix].val  = hashmap_from_sorted_unique_array(p, tmp, jx - ix, lvl + 8);
+	    hxns[ix].val  = hashmap_from_sorted_unique_array(factory, tmp, jx - ix, lvl + 8);
 	    erts_free(ERTS_ALC_T_TMP, (void *) tmp);
 	    ix = jx;
 	    if (ix < n) { elems++; }
@@ -651,15 +656,16 @@ static Eterm hashmap_from_sorted_unique_array(Process *p, hxnode_t *hxns, Uint n
 	ix++;
     }
 
-    res = hashmap_from_chunked_array(p, hxns, elems, !lvl);
+    res = hashmap_from_chunked_array(factory, hxns, elems, !lvl);
 
-    ERTS_HOLE_CHECK(p);
+    ERTS_FACTORY_HOLE_CHECK(factory);
 
     return res;
 }
 
 #define HALLOC_EXTRA 200
-static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int is_root) {
+static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
+                                        hxnode_t *hxns, Uint n, int is_root) {
     Uint ix, d, dn, dc, slot, elems;
     Uint32 v, vp, vn, hdr;
     Uint bp, sz;
@@ -691,7 +697,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
 	dc = 7;
 	/* build collision nodes */
 	while (dc > d) {
-	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+	    hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
 	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(vp,dc));
 	    hp[1] = res;
 	    res   = make_hashmap(hp);
@@ -722,7 +728,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
 	    dc = 7;
 	    /* build collision nodes */
 	    while (dc > wat) {
-		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+		hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
 		hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
 		hp[1] = res;
 		res   = make_hashmap(hp);
@@ -762,7 +768,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
 		 * redundant collisions */
 		hdr  |= bp;
 		sz    = hashmap_bitcount(hdr);
-		hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
+		hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
 		nhp   = hp;
 		*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
 		*hp++ = res; sz--;
@@ -782,7 +788,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
 	vp = v;
 	v  = vn;
 	d  = dn;
-	ERTS_HOLE_CHECK(p);
+	ERTS_FACTORY_HOLE_CHECK(factory);
     }
 
     /* v and vp are reused from above */
@@ -794,7 +800,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
 	dc = 7;
 	/* build collision nodes */
 	while (dc > dn) {
-	    hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+	    hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
 	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc));
 	    hp[1] = res;
 	    res   = make_hashmap(hp);
@@ -811,7 +817,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
 	 * redundant collisions */
 	hdr  |= bp;
 	sz    = hashmap_bitcount(hdr);
-	hp    = HAllocX(p, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
+	hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
 	nhp   = hp;
 	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
 	*hp++ = res; sz--;
@@ -828,7 +834,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
     bp    = 1 << slot;
     hdr  |= bp;
     sz    = hashmap_bitcount(hdr);
-    hp    = HAlloc(p, sz + /* hdr + item */ (is_root ? 2 : 1));
+    hp    = erts_produce_heap(factory, sz + /* hdr + item */ (is_root ? 2 : 1), 0);
     nhp   = hp;
 
     if (is_root) {
@@ -845,7 +851,7 @@ static Eterm hashmap_from_chunked_array(Process *p, hxnode_t *hxns, Uint n, int
 
     ASSERT(ESTACK_COUNT(stack) == 0);
     DESTROY_ESTACK(stack);
-    ERTS_HOLE_CHECK(p);
+    ERTS_FACTORY_HOLE_CHECK(factory);
     return res;
 }
 #undef HALLOC_EXTRA
@@ -1005,6 +1011,7 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 	Uint i;
 	Eterm res;
 	hxnode_t *hxns;
+        ErtsHeapFactory factory;
 
 	ks = map_get_keys(mp_new);
 	vs = map_get_values(mp_new);
@@ -1022,7 +1029,8 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 	    hxns[i].i    = i;
 	}
 
-	res = hashmap_from_unsorted_array(p, hxns, n);
+        factory.p = p;
+	res = hashmap_from_unsorted_array(&factory, hxns, n);
 
 	erts_free(ERTS_ALC_T_TMP, (void *) hxns);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1041,6 +1049,7 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
     Uint n, i;
     hxnode_t *hxns;
     Uint32 sw, hx;
+    ErtsHeapFactory factory;
 
     /* convert flat to tree */
 
@@ -1066,7 +1075,8 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
 	hxns[i].i    = i;
     }
 
-    res = hashmap_from_unsorted_array(p, hxns, n);
+    factory.p = p;
+    res = hashmap_from_unsorted_array(&factory, hxns, n);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1576,7 +1586,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 
 	/* the map will grow */
 
-	if (n > MAP_SMALL_MAP_LIMIT) {
+	if (n >= MAP_SMALL_MAP_LIMIT) {
 	    HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
 	    ks = map_get_keys(mp);
 	    vs = map_get_values(mp);
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index d1e227183f..35dbed86a9 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -80,7 +80,7 @@ typedef struct map_s {
 #define map_get_keys(x)        (((Eterm *)tuple_val(((map_t *)(x))->keys)) + 1)
 #define map_get_size(x)        (((map_t*)(x))->size)
 
-#define MAP_SMALL_MAP_LIMIT    (32)
+#define MAP_SMALL_MAP_LIMIT    (2) /*SVERK (32) */
 #define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
 #define MAP_HEADER_SIZE        (sizeof(map_t) / sizeof(Eterm))
 
@@ -102,7 +102,7 @@ int    erts_validate_and_sort_map(map_t* map);
 void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
-Eterm  erts_hashmap_from_array(Process *p, Eterm *leafs, Uint n);
+Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n);
 const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
 #define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 8870fac7d9..e4cbd8477d 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1146,3 +1146,16 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
     }
 }
 
+Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
+{
+    Eterm* res;
+    if (factory->p) {
+        res = HAllocX(factory->p, need, xtra);
+    } else {
+        res = factory->hp;
+        factory->hp += need;
+        ASSERT(factory->hp <= factory->hp_end);
+    }
+    return res;
+}
+
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 0f3bb8d281..ece75a5ee4 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -68,6 +68,23 @@ struct erl_heap_fragment {
     Eterm mem[1];		/* Data */
 };
 
+typedef struct {
+    Process* p;
+    Eterm* hp;
+    Eterm* hp_end;
+    /* more to come... */
+} ErtsHeapFactory;
+
+Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
+#ifdef CHECK_FOR_HOLES
+# define ERTS_FACTORY_HOLE_CHECK(f) do {    \
+        if ((f)->p) erts_check_for_holes((f)->p); \
+    } while (0)
+#else
+# define ERTS_FACTORY_HOLE_CHECK(p)
+#endif
+
+
 typedef struct erl_mesg {
     struct erl_mesg* next;	/* Next message */
     union {
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index af8db4c265..0e012e9eec 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1182,7 +1182,8 @@ typedef struct {
     Eterm* hp_end;
     int remaining_n;
     char* remaining_bytes;
-    Eterm* maps_head;
+    Eterm* maps_list;
+    struct dec_term_hamt_placeholder* hamt_list;
 } B2TDecodeContext;
 
 typedef struct {
@@ -1508,7 +1509,8 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
             ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size);
             ctx->u.dc.hp       = ctx->u.dc.hp_start;
             ctx->u.dc.hp_end   = ctx->u.dc.hp_start + ctx->heap_size;
-	    ctx->u.dc.maps_head = NULL;
+	    ctx->u.dc.maps_list = NULL;
+	    ctx->u.dc.hamt_list = NULL;
             ctx->state = B2TDecode;
             /*fall through*/
 	case B2TDecode:
@@ -2938,9 +2940,21 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
 #endif /* DEBUG */
 }
 
+struct dec_term_hamt_placeholder
+{
+    struct dec_term_hamt_placeholder* next;
+    Eterm* objp; /* write result here */
+    Uint size;   /* nr of leafs */
+    Eterm* leaf_array;
+
+    Eterm _heap_capacity_[1];
+};
+
+#define DEC_TERM_HAMT_PLACEHOLDER_SIZE(SZ) \
+    (offsetof(struct dec_term_hamt_placeholder, _heap_capacity_) + (SZ))
 
 /* Decode term from external format into *objp.
-** On failure return NULL and (R13B04) *hpp will be unchanged.
+** On failure return NULL and *hpp will be unchanged.
 */
 static byte*
 dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
@@ -2950,7 +2964,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
     int n;
     ErtsAtomEncoding char_enc;
     register Eterm* hp;        /* Please don't take the address of hp */
-    Eterm *maps_head;   /* for validation of maps */
+    Eterm *maps_list;   /* for preprocessing of small maps */
+    struct dec_term_hamt_placeholder* hamt_list;   /* for preprocessing of big maps */
     Eterm* next;
     SWord reds;
 
@@ -2960,7 +2975,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
         next     = ctx->u.dc.next;
         ep       = ctx->u.dc.ep;
         hpp      = &ctx->u.dc.hp;
-	maps_head = ctx->u.dc.maps_head;
+	maps_list = ctx->u.dc.maps_list;
+        hamt_list = ctx->u.dc.hamt_list;
 
         if (ctx->state != B2TDecode) {
             int n_limit = reds;
@@ -3041,7 +3057,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
         reds = ERTS_SWORD_MAX;
         next = objp;
         *next = (Eterm) (UWord) NULL;
-	maps_head = NULL;
+	maps_list = NULL;
+        hamt_list = NULL;
     }
     hp = *hpp;
 
@@ -3577,46 +3594,68 @@ dec_term_atom_common:
 	    break;
 	case MAP_EXT:
 	    {
-		map_t *mp;
 		Uint32 size,n;
 		Eterm *kptr,*vptr;
 		Eterm keys;
 
 		size = get_int32(ep); ep += 4;
 
-		keys  = make_tuple(hp);
-		*hp++ = make_arityval(size);
-		hp   += size;
-		kptr = hp - 1;
-
-		mp    = (map_t*)hp;
-		hp   += MAP_HEADER_SIZE;
-		hp   += size;
-		vptr = hp - 1;
-
-		/* kptr, last word for keys
-		 * vptr, last word for values
-		 */
-
-		/*
-		 * Use thing_word to link through decoded maps.
-		 * The list of maps is for later validation.
-		 */
-
-		mp->thing_word = (Eterm) COMPRESS_POINTER(maps_head);
-		maps_head      = (Eterm *) mp;
-
-		mp->size       = size;
-		mp->keys       = keys;
-		*objp          = make_map(mp);
-
-		for (n = size; n; n--) {
-		    *vptr = (Eterm) COMPRESS_POINTER(next);
-		    *kptr = (Eterm) COMPRESS_POINTER(vptr);
-		    next  = kptr;
-		    vptr--;
-		    kptr--;
-		}
+                if (size <= MAP_SMALL_MAP_LIMIT) {
+                    map_t *mp;
+
+                    keys  = make_tuple(hp);
+                    *hp++ = make_arityval(size);
+                    hp   += size;
+                    kptr = hp - 1;
+
+                    mp    = (map_t*)hp;
+                    hp   += MAP_HEADER_SIZE;
+                    hp   += size;
+                    vptr = hp - 1;
+
+                    /* kptr, last word for keys
+                     * vptr, last word for values
+                     */
+
+                    /*
+                     * Use thing_word to link through decoded maps.
+                     * The list of maps is for later validation.
+                     */
+
+                    mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list);
+                    maps_list      = (Eterm *) mp;
+
+                    mp->size       = size;
+                    mp->keys       = keys;
+                    *objp          = make_map(mp);
+
+                    for (n = size; n; n--) {
+                        *vptr = (Eterm) COMPRESS_POINTER(next);
+                        *kptr = (Eterm) COMPRESS_POINTER(vptr);
+                        next  = kptr;
+                        vptr--;
+                        kptr--;
+                    }
+                }
+                else {  /* Make hamt */
+                    struct dec_term_hamt_placeholder* holder =
+                        (struct dec_term_hamt_placeholder*) hp;
+
+                    holder->next = hamt_list;
+                    hamt_list    = holder;
+                    holder->objp = objp;
+                    holder->size = size;
+
+                    hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE(size);
+                    holder->leaf_array = hp;
+
+                    for (n = size; n; n--) {
+                        CDR(hp) = (Eterm) COMPRESS_POINTER(next);
+                        CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp));
+                        next = &CAR(hp);
+                        hp += 2;
+                    }
+                }
 	    }
 	    break;
 	case NEW_FUN_EXT:
@@ -3837,7 +3876,7 @@ dec_term_atom_common:
                     ctx->u.dc.ep = ep;
                     ctx->u.dc.next = next;
                     ctx->u.dc.hp = hp;
-		    ctx->u.dc.maps_head = maps_head;
+		    ctx->u.dc.maps_list = maps_list;
                     ctx->reds = 0;
                     return NULL;
                 }
@@ -3852,12 +3891,38 @@ dec_term_atom_common:
      * - done here for when we know it is complete.
      */
 
-    while (maps_head) {
-	next  = (Eterm *)(EXPAND_POINTER(*maps_head));
-	*maps_head = MAP_HEADER;
-	if (!erts_validate_and_sort_map((map_t*)maps_head))
+    while (maps_list) {
+	next  = (Eterm *)(EXPAND_POINTER(*maps_list));
+	*maps_list = MAP_HEADER;
+	if (!erts_validate_and_sort_map((map_t*)maps_list))
 	    goto error;
-	maps_head  = next;
+	maps_list  = next;
+    }
+
+    while (hamt_list) {
+        ErtsHeapFactory factory;
+        int residue;
+
+        factory.p = NULL;
+        factory.hp = (Eterm*) hamt_list;
+        factory.hp_end = hamt_list->leaf_array;
+
+        next  = (Eterm *) hamt_list->next;
+        objp = hamt_list->objp;
+
+	/*SVERK Make it reject duplicate keys */
+        *objp = erts_hashmap_from_array(&factory,
+                                        hamt_list->leaf_array,
+                                        hamt_list->size);
+        if (is_non_value(*objp))
+            goto error;
+
+        residue = factory.hp_end - factory.hp;
+        if (residue) {
+            ASSERT(residue > 0);
+            *factory.hp = make_pos_bignum_header(residue-1);
+        }
+        hamt_list = (struct dec_term_hamt_placeholder*) next;
     }
 
     if (ctx) {
@@ -4420,7 +4485,11 @@ init_done:
 	    n = get_int32(ep);
 	    ep += 4;
 	    ADDTERMS(2*n);
-	    heap_size += 3 + n + 1 + n;
+            if (n <= MAP_SMALL_MAP_LIMIT) {
+                heap_size += 3 + n + 1 + n;
+            } else {
+                heap_size += DEC_TERM_HAMT_PLACEHOLDER_SIZE(n) + 2*n;
+            }
 	    break;
 	case STRING_EXT:
 	    CHKSIZE(2);
-- 
cgit v1.2.3


From 10fa0c3d1b1fbe473de385ee7ae2aa77807e0281 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 5 Mar 2015 16:16:30 +0100
Subject: erts: Fix instruction i_has_map_fields_fsI for hashmaps

---
 erts/emulator/beam/beam_emu.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4f57037507..9b4cd38d69 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2400,10 +2400,24 @@ void process_main(void)
     Uint sz,n;
 
     GetArg1(1, map);
+    n  = (Uint)Arg(2);
+    fs = &Arg(3); /* pattern fields */
 
-    /* this instruction assumes Arg1 is a map,
-     * i.e. that it follows a test is_map if needed.
-     */
+    /* get term from field? */
+    if (is_hashmap(map)) {
+	Uint32 hx;
+	while(n--) {
+	    field = *fs++;
+	    hx = hashmap_make_hash(field);
+	    if (!erts_hashmap_get(hx,field,map)) {
+		SET_I((BeamInstr *) Arg(0));
+		goto has_map_fields_fail;
+	    }
+	}
+	goto has_map_fields_ok;
+    }
+
+    ASSERT(is_map(map));
 
     mp = (map_t *)map_val(map);
     sz = map_get_size(mp);
@@ -2414,8 +2428,6 @@ void process_main(void)
     }
 
     ks = map_get_keys(mp);
-    n  = (Uint)Arg(2);
-    fs = &Arg(3); /* pattern fields */
 
     ASSERT(n>0);
 
@@ -2433,7 +2445,7 @@ void process_main(void)
 	SET_I((BeamInstr *) Arg(0));
 	goto has_map_fields_fail;
     }
-
+has_map_fields_ok:
     I += 4 + Arg(2);
 has_map_fields_fail:
     ASSERT(VALID_INSTR(*I));
-- 
cgit v1.2.3


From 01e843722aa39b3411d349c6fbea80ad87a6e9ef Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 5 Mar 2015 18:48:14 +0100
Subject: erts: Reject duplicate keys for hamt in binary_to_term

---
 erts/emulator/beam/beam_emu.c |  2 +-
 erts/emulator/beam/erl_map.c  | 23 ++++++++++++++---------
 erts/emulator/beam/erl_map.h  |  2 +-
 erts/emulator/beam/external.c |  4 ++--
 4 files changed, 18 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9b4cd38d69..d45aa93f73 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6589,7 +6589,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
 	p->htop = mhp;
 
         factory.p = p;
-	return erts_hashmap_from_array(&factory, thp, n/2);
+	return erts_hashmap_from_array(&factory, thp, n/2, 0);
     }
 
     if (HeapWordsLeft(p) < need) {
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index eacdd0b6c3..3ccfc61b08 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -87,7 +87,7 @@ static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
 static Eterm map_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
-static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n);
+static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys);
 static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root);
 static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root);
 static Eterm hashmap_info(Process *p, Eterm node);
@@ -419,7 +419,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     }
 
     factory.p = p;
-    res = hashmap_from_unsorted_array(&factory, hxns, size);
+    res = hashmap_from_unsorted_array(&factory, hxns, size, 0);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -462,7 +462,8 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     return res;
 }
 
-Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n) {
+Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n,
+                              int reject_dupkeys) {
     Uint32 sw, hx;
     Uint ix;
     hxnode_t *hxns;
@@ -481,7 +482,7 @@ Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n) {
 	leafs += 2;
     }
 
-    res = hashmap_from_unsorted_array(factory, hxns, n);
+    res = hashmap_from_unsorted_array(factory, hxns, n, reject_dupkeys);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
 
@@ -523,7 +524,7 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
     }
 
     factory.p = p;
-    res = hashmap_from_unsorted_array(&factory, hxns, sz);
+    res = hashmap_from_unsorted_array(&factory, hxns, sz, 0);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -532,7 +533,8 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
 }
 
 static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory,
-                                         hxnode_t *hxns, Uint n) {
+                                         hxnode_t *hxns, Uint n,
+                                         int reject_dupkeys) {
     Uint jx = 0, ix = 0, lx, cx;
     Eterm res;
 
@@ -566,7 +568,10 @@ static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory,
 	    while(ix < jx) {
 		lx = ix;
 		while(ix < jx && EQ(CAR(list_val(hxns[ix].val)), CAR(list_val(hxns[lx].val)))) {
-		    if (hxns[ix].i > hxns[lx].i) {
+                    if (reject_dupkeys)
+                        return THE_NON_VALUE;
+
+                    if (hxns[ix].i > hxns[lx].i) {
 			lx = ix;
 		    }
 		    ix++;
@@ -1030,7 +1035,7 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 	}
 
         factory.p = p;
-	res = hashmap_from_unsorted_array(&factory, hxns, n);
+	res = hashmap_from_unsorted_array(&factory, hxns, n, 0);
 
 	erts_free(ERTS_ALC_T_TMP, (void *) hxns);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1076,7 +1081,7 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
     }
 
     factory.p = p;
-    res = hashmap_from_unsorted_array(&factory, hxns, n);
+    res = hashmap_from_unsorted_array(&factory, hxns, n, 0);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 35dbed86a9..39d98b9e03 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -102,7 +102,7 @@ int    erts_validate_and_sort_map(map_t* map);
 void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
-Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n);
+Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys);
 const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
 #define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 0e012e9eec..9bcd9a7a50 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3910,10 +3910,10 @@ dec_term_atom_common:
         next  = (Eterm *) hamt_list->next;
         objp = hamt_list->objp;
 
-	/*SVERK Make it reject duplicate keys */
         *objp = erts_hashmap_from_array(&factory,
                                         hamt_list->leaf_array,
-                                        hamt_list->size);
+                                        hamt_list->size,
+                                        1);
         if (is_non_value(*objp))
             goto error;
 
-- 
cgit v1.2.3


From 7478569d0a7d619d600816f3a75e56922d8ed428 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Mar 2015 18:07:11 +0100
Subject: erts: Tweak over estimation of hashmap size for binary_to_term

Strategy: Calculate an over estimation of heap size that will give
such a low probability for overflow, that "it will not happen".

Scary assumption 1: Uniformly distributed hash values.
Scary assumption 2: Tree size is normally distributed (right?)
---
 erts/emulator/beam/erl_map.c   | 37 ++++++++++++++++++++++++++
 erts/emulator/beam/erl_map.h   |  1 +
 erts/emulator/beam/erl_utils.h |  1 +
 erts/emulator/beam/external.c  | 60 ++++++++++++++++++++++--------------------
 erts/emulator/beam/utils.c     | 11 ++++++++
 5 files changed, 81 insertions(+), 29 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 3ccfc61b08..5eafa69b06 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2437,6 +2437,43 @@ int erts_validate_and_sort_map(map_t* mp)
     return 1;
 }
 
+/* Really rough estimate of sqrt(x)
+ * Guaranteed not to be less than sqrt(x)
+ */
+static int int_sqrt_ceiling(Uint x)
+{
+    int n;
+
+    if (x <= 2)
+	return x;
+
+    n = erts_fit_in_bits_uint(x-1);
+    if (n & 1) {
+	/* Calc: sqrt(2^n) = 2^(n/2) * sqrt(2) ~= 2^(n/2) * 3 / 2 */
+	return (1 << (n/2 - 1)) * 3;
+    }
+    else {
+	/* Calc: sqrt(2^n) = 2^(n/2) */
+	return 1 << (n / 2);
+    }
+}
+
+Uint hashmap_over_estimated_heap_size(Uint n)
+{
+    /* n is nr of key-value pairs.
+       Average nr of nodes is about n/3.
+       Standard deviation is about sqrt(n)/3.
+       Assuming normal probability distribution,
+       we overestimate nr of nodes by 14 std.devs, which gives a probability
+       for overrun of 1.0e-49 (same magnitude as a git SHA1 collision).
+     */
+    Uint nodes = (n + int_sqrt_ceiling(n)*14) / 3;
+    return (n*2 +     /* leaf cons cells */
+	    n +       /* leaf list terms */
+	    nodes*2); /* headers + parent refs */
+}
+
+
 BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) {
     if (is_hashmap(BIF_ARG_1)) {
 	BIF_RET(hashmap_info(BIF_P,BIF_ARG_1));
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 39d98b9e03..659295184d 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -99,6 +99,7 @@ Eterm  erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 			      Uint *upsz, struct ErtsEStack_ *sp);
 
 int    erts_validate_and_sort_map(map_t* map);
+Uint   hashmap_over_estimated_heap_size(Uint n);
 void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index c772a750f1..7cb8972e29 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -113,6 +113,7 @@ void erts_silence_warn_unused_result(long unused);
 
 int erts_fit_in_bits_int64(Sint64);
 int erts_fit_in_bits_int32(Sint32);
+int erts_fit_in_bits_uint(Uint);
 int erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
 Uint32 make_broken_hash(Eterm);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9bcd9a7a50..bd9ad65086 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2945,13 +2945,11 @@ struct dec_term_hamt_placeholder
     struct dec_term_hamt_placeholder* next;
     Eterm* objp; /* write result here */
     Uint size;   /* nr of leafs */
-    Eterm* leaf_array;
-
-    Eterm _heap_capacity_[1];
+    Eterm leafs[1];
 };
 
-#define DEC_TERM_HAMT_PLACEHOLDER_SIZE(SZ) \
-    (offsetof(struct dec_term_hamt_placeholder, _heap_capacity_) + (SZ))
+#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \
+    (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm))
 
 /* Decode term from external format into *objp.
 ** On failure return NULL and *hpp will be unchanged.
@@ -3646,8 +3644,7 @@ dec_term_atom_common:
                     holder->objp = objp;
                     holder->size = size;
 
-                    hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE(size);
-                    holder->leaf_array = hp;
+                    hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE;
 
                     for (n = size; n; n--) {
                         CDR(hp) = (Eterm) COMPRESS_POINTER(next);
@@ -3899,30 +3896,33 @@ dec_term_atom_common:
 	maps_list  = next;
     }
 
-    while (hamt_list) {
-        ErtsHeapFactory factory;
-        int residue;
-
-        factory.p = NULL;
-        factory.hp = (Eterm*) hamt_list;
-        factory.hp_end = hamt_list->leaf_array;
+    /* Iterate through all the hamts and build tree nodes.
+     */
+    if (hamt_list) {
+        struct dec_term_hamt_placeholder* hamt = hamt_list;
+	ErtsHeapFactory factory;
+
+	factory.p = NULL;
+        factory.hp = hp;
+	/* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
+        factory.hp_end = hp + (ERTS_SWORD_MAX / sizeof(Eterm));
+
+        do {
+	    *hamt->objp = erts_hashmap_from_array(&factory,
+						  hamt->leafs,
+						  hamt->size,
+						  1);
+	    if (is_non_value(*hamt->objp))
+		goto error;
 
-        next  = (Eterm *) hamt_list->next;
-        objp = hamt_list->objp;
+	    hamt_list = hamt->next;
 
-        *objp = erts_hashmap_from_array(&factory,
-                                        hamt_list->leaf_array,
-                                        hamt_list->size,
-                                        1);
-        if (is_non_value(*objp))
-            goto error;
+	    /* Yes, we waste a couple of heap words per hamt
+	       for the temporary placeholder */
+	    *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1);
+        } while (hamt_list);
 
-        residue = factory.hp_end - factory.hp;
-        if (residue) {
-            ASSERT(residue > 0);
-            *factory.hp = make_pos_bignum_header(residue-1);
-        }
-        hamt_list = (struct dec_term_hamt_placeholder*) next;
+	hp = factory.hp;
     }
 
     if (ctx) {
@@ -4292,6 +4292,8 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
     return 0;
 }
 
+
+
 static Sint
 decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
 {
@@ -4488,7 +4490,7 @@ init_done:
             if (n <= MAP_SMALL_MAP_LIMIT) {
                 heap_size += 3 + n + 1 + n;
             } else {
-                heap_size += DEC_TERM_HAMT_PLACEHOLDER_SIZE(n) + 2*n;
+                heap_size += hashmap_over_estimated_heap_size(n);
             }
 	    break;
 	case STRING_EXT:
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 674dddc86f..66f13461a1 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -352,6 +352,17 @@ int erts_fit_in_bits_int32(Sint32 value)
     return fit_in_bits((Sint64) (Uint32) value, 4);
 }
 
+int erts_fit_in_bits_uint(Uint value)
+{
+#if ERTS_SIZEOF_ETERM == 4
+    return fit_in_bits((Sint64) (Uint32) value, 4);
+#elif ERTS_SIZEOF_ETERM == 8
+    return fit_in_bits(value, 5);
+#else
+# error "No way, Jose"
+#endif
+}
+
 int
 erts_print(int to, void *arg, char *format, ...)
 {
-- 
cgit v1.2.3


From 3fdd4046c4306256233984e289898f24324273f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 10 Mar 2015 09:04:56 +0100
Subject: erts: Add hashmap_iterator_prev to maps

---
 erts/emulator/beam/erl_map.c | 45 ++++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_map.h |  1 +
 2 files changed, 46 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 5eafa69b06..65a31d3680 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1758,6 +1758,51 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
     }
 }
 
+Eterm* hashmap_iterator_prev(ErtsWStack* s) {
+    Eterm node, *ptr, hdr;
+    Uint32 sz,i;
+
+    for (;;) {
+        ASSERT(!WSTACK_ISEMPTY((*s)));
+	node = (Eterm) WSTACK_POP((*s));
+        if (is_non_value(node)) {
+            return NULL;
+        }
+        switch (primary_tag(node)) {
+        case TAG_PRIMARY_LIST:
+            return list_val(node);
+
+        case TAG_PRIMARY_BOXED:
+            ptr = boxed_val(node);
+            hdr = *ptr;
+            ASSERT(is_header(hdr));
+            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_HEAD_ARRAY:
+                ptr++;
+            case HAMT_SUBTAG_NODE_ARRAY:
+                ptr++;
+                i = 0;
+                while(i < 16) { WSTACK_PUSH((*s), (UWord)ptr[i++]); }
+                break;
+            case HAMT_SUBTAG_HEAD_BITMAP:
+                ptr++;
+            case HAMT_SUBTAG_NODE_BITMAP:
+                sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                ASSERT(sz < 17);
+		i = 0;
+                ptr++;
+                while(i < sz) { WSTACK_PUSH((*s), (UWord)ptr[i++]); }
+                break;
+            default:
+                erl_exit(1, "bad header");
+            }
+            break;
+
+        default:
+            erl_exit(1, "bad hamt node");
+	}
+    }
+}
 const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     Eterm *ptr, hdr;
     Eterm th[2];
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 659295184d..db22d461ce 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -102,6 +102,7 @@ int    erts_validate_and_sort_map(map_t* map);
 Uint   hashmap_over_estimated_heap_size(Uint n);
 void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
+Eterm* hashmap_iterator_prev(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys);
 const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
-- 
cgit v1.2.3


From 05169ff8fa0713f3ea969bc27b9d8f58b439863c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 10 Mar 2015 12:57:59 +0100
Subject: erts: Teach hashmaps to match spec compiler

---
 erts/emulator/beam/erl_db_util.c | 222 +++++++++++++++++++++++++++++++++------
 1 file changed, 189 insertions(+), 33 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 748be93fe3..8b5ab16642 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -231,7 +231,8 @@ typedef enum {
     matchConsA, /* Car is below Cdr */
     matchConsB, /* Cdr is below Car (unusual) */
     matchMkTuple,
-    matchMkMap,
+    matchMkFlatMap,
+    matchMkHashMap,
     matchCall0,
     matchCall1,
     matchCall2,
@@ -1424,6 +1425,63 @@ restart:
                     }
                     break;
                 }
+                if (is_hashmap(t)) {
+                    DECLARE_WSTACK(wstack);
+                    Eterm *kv;
+                    num_iters = hashmap_size(t);
+                    if (!structure_checked) {
+                        DMC_PUSH(text, matchMap);
+                        DMC_PUSH(text, num_iters);
+                    }
+                    structure_checked = 0;
+
+                    hashmap_iterator_init(&wstack, t);
+
+                    while ((kv=hashmap_iterator_next(&wstack)) != NULL) {
+                        Eterm key = CAR(kv);
+                        Eterm value = CDR(kv);
+                        if (db_is_variable(key) >= 0) {
+                            if (context.err_info) {
+                                add_dmc_err(context.err_info,
+                                        "Variable found in map key.",
+                                        -1, 0UL, dmcError);
+                            }
+                            DESTROY_WSTACK(wstack);
+                            goto error;
+                        } else if (key == am_Underscore) {
+                            if (context.err_info) {
+                                add_dmc_err(context.err_info,
+                                        "Underscore found in map key.",
+                                        -1, 0UL, dmcError);
+                            }
+                            DESTROY_WSTACK(wstack);
+                            goto error;
+                        }
+                        DMC_PUSH(text, matchKey);
+                        DMC_PUSH(text, dmc_private_copy(&context, key));
+                        {
+                            int old_stack = ++(context.stack_used);
+                            res = dmc_one_term(&context, &heap, &stack, &text,
+                                               value);
+                            ASSERT(res != retFail);
+                            if (res == retRestart) {
+                                DESTROY_WSTACK(wstack);
+                                goto restart;
+                            }
+                            if (old_stack != context.stack_used) {
+                                ASSERT(old_stack + 1 == context.stack_used);
+                                DMC_PUSH(text, matchSwap);
+                            }
+                            if (context.stack_used > context.stack_need) {
+                                context.stack_need = context.stack_used;
+                            }
+                            DMC_PUSH(text, matchPop);
+                            --(context.stack_used);
+                        }
+                    }
+                    DESTROY_WSTACK(wstack);
+                    break;
+                }
 		if (!is_tuple(t)) {
 		    goto simple_term;
 		}
@@ -1946,23 +2004,37 @@ restart:
 	    ++ep;
 	    break;
         case matchMap:
-            if (!is_map_rel(*ep, base)) {
+            if (!is_map_rel(*ep, base) && !is_hashmap_rel(*ep,base)) {
                 FAIL();
             }
             n = *pc++;
-            if (map_get_size(map_val_rel(*ep, base)) < n) {
-                FAIL();
-            }
+            if (is_map_rel(*ep,base)) {
+		if (map_get_size(map_val_rel(*ep, base)) < n) {
+		    FAIL();
+		}
+            } else {
+		ASSERT(is_hashmap_rel(*ep,base));
+		if (hashmap_size_rel(*ep, base) < n) {
+		    FAIL();
+		}
+	    }
             ep = map_val_rel(*ep, base);
             break;
         case matchPushM:
-            if (!is_map_rel(*ep, base)) {
+            if (!is_map_rel(*ep, base) && !is_hashmap_rel(*ep, base)) {
                 FAIL();
             }
             n = *pc++;
-            if (map_get_size(map_val_rel(*ep, base)) < n) {
-                FAIL();
-            }
+            if (is_map_rel(*ep,base)) {
+		if (map_get_size(map_val_rel(*ep, base)) < n) {
+		    FAIL();
+		}
+	    } else {
+		ASSERT(is_hashmap_rel(*ep,base));
+		if (hashmap_size_rel(*ep, base) < n) {
+		    FAIL();
+		}
+	    }
             *sp++ = map_val_rel(*ep++, base);
             break;
         case matchKey:
@@ -2079,7 +2151,7 @@ restart:
 	    }
 	    *esp++ = t;
 	    break;
-        case matchMkMap:
+        case matchMkFlatMap:
             n = *pc++;
             ehp = HAllocX(build_proc, 1 + MAP_HEADER_SIZE + n, HEAP_XTRA);
             t = *ehp++ = *--esp;
@@ -2096,6 +2168,21 @@ restart:
             }
             *esp++ = t;
             break;
+        case matchMkHashMap:
+            n = *pc++;
+            esp -= 2*n;
+            ehp = HAllocX(build_proc, 2*n, HEAP_XTRA);
+            {
+                ErtsHeapFactory factory;
+                Uint ix;
+                factory.p = build_proc;
+                for (ix = 0; ix < 2*n; ix++){
+                    ehp[ix] = esp[ix];
+                }
+                t = erts_hashmap_from_array(&factory, ehp, n, 0);
+            }
+            *esp++ = t;
+            break;
 	case matchCall0:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
 	    t = (*bif)(build_proc, bif_args);
@@ -3366,7 +3453,6 @@ static DMCRet dmc_one_term(DMCContext *context,
     Uint sz, sz2, sz3;
     Uint i, j;
 
-
     switch (c & _TAG_PRIMARY_MASK) {
     case TAG_PRIMARY_IMMED1:
 	if ((n = db_is_variable(c)) >= 0) { /* variable */
@@ -3460,6 +3546,13 @@ static DMCRet dmc_one_term(DMCContext *context,
             DMC_PUSH(*text, n);
             DMC_PUSH(*stack, c);
             break;
+        case (_TAG_HEADER_HASHMAP >> _TAG_PRIMARY_SIZE):
+            n = hashmap_size(c);
+            DMC_PUSH(*text, matchPushM);
+            ++(context->stack_used);
+            DMC_PUSH(*text, n);
+            DMC_PUSH(*stack, c);
+            break;
 	case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
 	{
 	    Eterm* ref_val = internal_ref_val(c);
@@ -3745,30 +3838,87 @@ static DMCRet
 dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
         Eterm t, int *constant)
 {
-    map_t *m = (map_t *)map_val(t);
-    Eterm *values = map_get_values(m);
-    int nelems = map_get_size(m);
+    int nelems;
     int constant_values;
     DMCRet ret;
+    if (is_map(t)) {
+        map_t *m = (map_t *)map_val(t);
+        Eterm *values = map_get_values(m);
 
-    ret = dmc_array(context, heap, text, values, nelems, &constant_values);
-    if (ret != retOk) {
-        return ret;
-    }
-    if (constant_values) {
-        *constant = 1;
+        nelems = map_get_size(m);
+        ret = dmc_array(context, heap, text, values, nelems, &constant_values);
+
+        if (ret != retOk) {
+            return ret;
+        }
+        if (constant_values) {
+            *constant = 1;
+            return retOk;
+        }
+        DMC_PUSH(*text, matchPushC);
+        DMC_PUSH(*text, dmc_private_copy(context, m->keys));
+        if (++context->stack_used > context->stack_need) {
+            context->stack_need = context->stack_used;
+        }
+        DMC_PUSH(*text, matchMkFlatMap);
+        DMC_PUSH(*text, nelems);
+        context->stack_used -= nelems;
+        *constant = 0;
+        return retOk;
+    } else {
+        DECLARE_WSTACK(wstack);
+        Eterm *kv;
+        int c;
+
+        ASSERT(is_hashmap(t));
+
+        hashmap_iterator_init(&wstack, t);
+        constant_values = 1;
+        nelems = hashmap_size(t);
+
+        while ((kv=hashmap_iterator_prev(&wstack)) != NULL) {
+            if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) {
+                DESTROY_WSTACK(wstack);
+                return ret;
+            }
+            if (!c)
+                constant_values = 0;
+        }
+
+        if (constant_values) {
+            *constant = 1;
+            DESTROY_WSTACK(wstack);
+            return retOk;
+        }
+
+        *constant = 0;
+
+        hashmap_iterator_init(&wstack, t);
+
+        while ((kv=hashmap_iterator_prev(&wstack)) != NULL) {
+            /* push key */
+            if ((ret = dmc_expr(context, heap, text, CAR(kv), &c)) != retOk) {
+                DESTROY_WSTACK(wstack);
+                return ret;
+            }
+            if (c) {
+                do_emit_constant(context, text, CAR(kv));
+            }
+            /* push value */
+            if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) {
+                DESTROY_WSTACK(wstack);
+                return ret;
+            }
+            if (c) {
+                do_emit_constant(context, text, CDR(kv));
+            }
+        }
+        DMC_PUSH(*text, matchMkHashMap);
+        DMC_PUSH(*text, nelems);
+        context->stack_used -= nelems;
+        DESTROY_WSTACK(wstack);
         return retOk;
     }
-    DMC_PUSH(*text, matchPushC);
-    DMC_PUSH(*text, dmc_private_copy(context, m->keys));
-    if (++context->stack_used > context->stack_need) {
-        context->stack_need = context->stack_used;
-    }
-    DMC_PUSH(*text, matchMkMap);
-    DMC_PUSH(*text, nelems);
-    context->stack_used -= nelems;
-    *constant = 0;
-    return retOk;
 }
 
 static DMCRet dmc_whole_expression(DMCContext *context,
@@ -4765,7 +4915,7 @@ static DMCRet dmc_expr(DMCContext *context,
 	    return ret;
 	break;
     case TAG_PRIMARY_BOXED:
-        if (is_map(t)) {
+        if (is_map(t) || is_hashmap(t)) {
             return dmc_map(context, heap, text, t, constant);
         }
 	if (!is_tuple(t)) {
@@ -5462,11 +5612,17 @@ void db_match_dis(Binary *bp)
 	    ++t;
 	    erts_printf("MkTuple\t%beu\n", n);
 	    break;
-        case matchMkMap:
+        case matchMkFlatMap:
+            ++t;
+            n = *t;
+            ++t;
+            erts_printf("MkFlatMap\t%beu\n", n);
+            break;
+        case matchMkHashMap:
             ++t;
             n = *t;
             ++t;
-            erts_printf("MkMapA\t%beu\n", n);
+            erts_printf("MkHashMap\t%beu\n", n);
             break;
 	case matchOr:
 	    ++t;
-- 
cgit v1.2.3


From 27e57aa05354b743b735a41716c0e3af18f2843e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Mar 2015 19:03:19 +0100
Subject: erts: Refactor maps naming convention

flatmap: Small map
hashmap: Large map
map:     flatmap or hashmap
---
 erts/emulator/beam/beam_emu.c        |  90 ++++++++--------
 erts/emulator/beam/copy.c            |  10 +-
 erts/emulator/beam/erl_bif_guard.c   |   6 +-
 erts/emulator/beam/erl_bif_op.c      |   2 +-
 erts/emulator/beam/erl_db_util.c     |  50 ++++-----
 erts/emulator/beam/erl_gc.h          |   2 +-
 erts/emulator/beam/erl_map.c         | 196 +++++++++++++++++------------------
 erts/emulator/beam/erl_map.h         |  34 +++---
 erts/emulator/beam/erl_nif.c         |  58 +++++------
 erts/emulator/beam/erl_printf_term.c |   8 +-
 erts/emulator/beam/external.c        |  24 ++---
 erts/emulator/beam/io.c              |   8 +-
 erts/emulator/beam/utils.c           |  36 +++----
 13 files changed, 262 insertions(+), 262 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index d45aa93f73..10ae3ed77c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -699,7 +699,7 @@ void** beam_ops;
         Fail; 								  \
     }
 
-#define IsMap(Src, Fail) if (is_not_map(Src) && is_not_hashmap(Src)) { Fail; }
+#define IsMap(Src, Fail) if (is_not_flatmap(Src) && is_not_hashmap(Src)) { Fail; }
 
 #define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; }
 
@@ -2392,7 +2392,7 @@ void process_main(void)
  }
 
  OpCase(i_has_map_fields_fsI): {
-    map_t* mp;
+    flatmap_t* mp;
     Eterm map;
     Eterm field;
     Eterm *ks;
@@ -2417,17 +2417,17 @@ void process_main(void)
 	goto has_map_fields_ok;
     }
 
-    ASSERT(is_map(map));
+    ASSERT(is_flatmap(map));
 
-    mp = (map_t *)map_val(map);
-    sz = map_get_size(mp);
+    mp = (flatmap_t *)flatmap_val(map);
+    sz = flatmap_get_size(mp);
 
     if (sz == 0) {
 	SET_I((BeamInstr *) Arg(0));
 	goto has_map_fields_fail;
     }
 
-    ks = map_get_keys(mp);
+    ks = flatmap_get_keys(mp);
 
     ASSERT(n>0);
 
@@ -2484,21 +2484,21 @@ do {								\
     n  = (Uint)Arg(2) / 2;
     fs = &Arg(3); /* pattern fields and target registers */
 
-    if (is_map(map)) {
-	map_t *mp;
+    if (is_flatmap(map)) {
+	flatmap_t *mp;
 	Eterm *ks;
 	Eterm *vs;
 
-	mp = (map_t *)map_val(map);
-	sz = map_get_size(mp);
+	mp = (flatmap_t *)flatmap_val(map);
+	sz = flatmap_get_size(mp);
 
 	if (sz == 0) {
 	    SET_I((BeamInstr *) Arg(0));
 	    goto get_map_elements_fail;
 	}
 
-	ks = map_get_keys(mp);
-	vs = map_get_values(mp);
+	ks = flatmap_get_keys(mp);
+	vs = flatmap_get_values(mp);
 
 	while(sz) {
 	    if (EQ((Eterm)*fs,*ks)) {
@@ -6473,15 +6473,15 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
 static int has_not_map_field(Eterm map, Eterm key)
 {
     Uint32 hx;
-    if (is_map(map)) {
-	map_t* mp;
+    if (is_flatmap(map)) {
+	flatmap_t* mp;
 	Eterm* keys;
 	Uint i;
 	Uint n;
 
-	mp   = (map_t *)map_val(map);
-	keys = map_get_keys(mp);
-	n    = map_get_size(mp);
+	mp   = (flatmap_t *)flatmap_val(map);
+	keys = flatmap_get_keys(mp);
+	n    = flatmap_get_size(mp);
 	if (is_immed(key)) {
 	    for (i = 0; i < n; i++) {
 		if (keys[i] == key) {
@@ -6506,16 +6506,16 @@ static Eterm get_map_element(Eterm map, Eterm key)
 {
     Uint32 hx;
     const Eterm *vs;
-    if (is_map(map)) {
-	map_t *mp;
+    if (is_flatmap(map)) {
+	flatmap_t *mp;
 	Eterm *ks;
 	Uint i;
 	Uint n;
 
-	mp = (map_t *)map_val(map);
-	ks = map_get_keys(mp);
-	vs = map_get_values(mp);
-	n  = map_get_size(mp);
+	mp = (flatmap_t *)flatmap_val(map);
+	ks = flatmap_get_keys(mp);
+	vs = flatmap_get_values(mp);
+	n  = flatmap_get_size(mp);
 	if (is_immed(key)) {
 	    for (i = 0; i < n; i++) {
 		if (ks[i] == key) {
@@ -6567,7 +6567,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     Eterm *mhp,*thp;
     Eterm *E;
     BeamInstr *ptr;
-    map_t *mp;
+    flatmap_t *mp;
     ErtsHeapFactory factory;
 
     ptr = &Arg(4);
@@ -6602,7 +6602,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     keys   = make_tuple(thp);
     *thp++ = make_arityval(n/2);
 
-    mp = (map_t *)mhp; mhp += MAP_HEADER_SIZE;
+    mp = (flatmap_t *)mhp; mhp += MAP_HEADER_SIZE;
     mp->thing_word = MAP_HEADER;
     mp->size = n/2;
     mp->keys = keys;
@@ -6612,7 +6612,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
 	GET_TERM(*ptr++, *mhp++);
     }
     p->htop = mhp;
-    return make_map(mp);
+    return make_flatmap(mp);
 }
 
 static Eterm
@@ -6622,7 +6622,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Uint num_old;
     Uint num_updates;
     Uint need;
-    map_t *old_mp, *mp;
+    flatmap_t *old_mp, *mp;
     Eterm res;
     Eterm* hp;
     Eterm* E;
@@ -6635,7 +6635,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     new_p = &Arg(5);
     num_updates = Arg(4) / 2;
 
-    if (is_not_map(map)) {
+    if (is_not_flatmap(map)) {
 	Uint32 hx;
 	Eterm val;
 
@@ -6668,8 +6668,8 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	return res;
     }
 
-    old_mp  = (map_t *) map_val(map);
-    num_old = map_get_size(old_mp);
+    old_mp  = (flatmap_t *) flatmap_val(map);
+    num_old = flatmap_get_size(old_mp);
 
     /*
      * If the old map is empty, create a new map.
@@ -6690,7 +6690,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	reg[live] = map;
 	erts_garbage_collect(p, need, reg, live+1);
 	map      = reg[live];
-	old_mp   = (map_t *)map_val(map);
+	old_mp   = (flatmap_t *)flatmap_val(map);
     }
 
     /*
@@ -6721,14 +6721,14 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     kp = p->htop + 1;		/* Point to first key */
     hp = kp + num_old + num_updates;
 
-    res = make_map(hp);
-    mp = (map_t *)hp;
+    res = make_flatmap(hp);
+    mp = (flatmap_t *)hp;
     hp += MAP_HEADER_SIZE;
     mp->thing_word = MAP_HEADER;
     mp->keys = make_tuple(kp-1);
 
-    old_vals = map_get_values(old_mp);
-    old_keys = map_get_keys(old_mp);
+    old_vals = flatmap_get_values(old_mp);
+    old_keys = flatmap_get_keys(old_mp);
 
     GET_TERM(*new_p, new_key);
     n = num_updates;
@@ -6820,7 +6820,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     /* The expensive case, need to build a hashmap */
     if (n > MAP_SMALL_MAP_LIMIT) {
-	res = erts_hashmap_from_ks_and_vs(p,map_get_keys(mp),map_get_values(mp),n);
+	res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n);
 	if (p->mbuf) {
 	    Uint live = Arg(3);
 	    reg[live] = res;
@@ -6842,7 +6842,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Uint i;
     Uint num_old;
     Uint need;
-    map_t *old_mp, *mp;
+    flatmap_t *old_mp, *mp;
     Eterm res;
     Eterm* hp;
     Eterm* E;
@@ -6855,7 +6855,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     n = Arg(4) / 2;		/* Number of values to be updated */
     ASSERT(n > 0);
 
-    if (is_not_map(map)) {
+    if (is_not_flatmap(map)) {
 	Uint32 hx;
 	Eterm val;
 
@@ -6891,8 +6891,8 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	return res;
     }
 
-    old_mp = (map_t *) map_val(map);
-    num_old = map_get_size(old_mp);
+    old_mp = (flatmap_t *) flatmap_val(map);
+    num_old = flatmap_get_size(old_mp);
 
     /*
      * If the old map is empty, create a new map.
@@ -6912,7 +6912,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	reg[live] = map;
 	erts_garbage_collect(p, need, reg, live+1);
 	map      = reg[live];
-	old_mp   = (map_t *)map_val(map);
+	old_mp   = (flatmap_t *)flatmap_val(map);
     }
 
     /*
@@ -6922,11 +6922,11 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     hp = p->htop;
     E = p->stop;
 
-    old_vals = map_get_values(old_mp);
-    old_keys = map_get_keys(old_mp);
+    old_vals = flatmap_get_values(old_mp);
+    old_keys = flatmap_get_keys(old_mp);
 
-    res = make_map(hp);
-    mp = (map_t *)hp;
+    res = make_flatmap(hp);
+    mp = (flatmap_t *)hp;
     hp += MAP_HEADER_SIZE;
     mp->thing_word = MAP_HEADER;
     mp->size = num_old;
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 5901c00d0a..83214aca19 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -186,10 +186,10 @@ Uint size_object(Eterm obj)
 		case MAP_SUBTAG:
 		    {
 			Uint n;
-			map_t *mp;
-			mp  = (map_t*)map_val_rel(obj,base);
+			flatmap_t *mp;
+			mp  = (flatmap_t*)flatmap_val_rel(obj,base);
 			ptr = (Eterm *)mp;
-			n   = map_get_size(mp) + 1;
+			n   = flatmap_get_size(mp) + 1;
 			sum += n + 2;
 			ptr += 2; /* hdr + size words */
 			while (n--) {
@@ -371,8 +371,8 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		break;
 	    case MAP_SUBTAG:
 		{
-		    i = map_get_size(objp) + 3;
-		    *argp = make_map_rel(htop, dst_base);
+		    i = flatmap_get_size(objp) + 3;
+		    *argp = make_flatmap_rel(htop, dst_base);
 		    while (i--) {
 			*htop++ = *objp++;
 		    }
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index bc0891422b..e7d84ebda1 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -461,9 +461,9 @@ Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live)
     Eterm arg = reg[live];
     Eterm* hp;
     Uint size;
-    if (is_map(arg)) {
-	map_t *mp = (map_t*)map_val(arg);
-	size = map_get_size(mp);
+    if (is_flatmap(arg)) {
+	flatmap_t *mp = (flatmap_t*)flatmap_val(arg);
+	size = flatmap_get_size(mp);
     } else if (is_hashmap(arg)) {
 	size = hashmap_size(arg);
     } else {
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index 11c6c9e556..af9b7bd908 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -324,7 +324,7 @@ BIF_RETTYPE is_record_3(BIF_ALIST_3)
 	
 BIF_RETTYPE is_map_1(BIF_ALIST_1)
 {
-    if (is_map(BIF_ARG_1) || is_hashmap(BIF_ARG_1)) {
+    if (is_flatmap(BIF_ARG_1) || is_hashmap(BIF_ARG_1)) {
 	BIF_RET(am_true);
     }
     BIF_RET(am_false);
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 8b5ab16642..cca3f381a1 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1377,15 +1377,15 @@ restart:
 	for (;;) {
 	    switch (t & _TAG_PRIMARY_MASK) {
 	    case TAG_PRIMARY_BOXED:
-                if (is_map(t)) {
-                    num_iters = map_get_size(map_val(t));
+                if (is_flatmap(t)) {
+                    num_iters = flatmap_get_size(flatmap_val(t));
                     if (!structure_checked) {
                         DMC_PUSH(text, matchMap);
                         DMC_PUSH(text, num_iters);
                     }
                     structure_checked = 0;
                     for (i = 0; i < num_iters; ++i) {
-                        Eterm key = map_get_keys(map_val(t))[i];
+                        Eterm key = flatmap_get_keys(flatmap_val(t))[i];
                         if (db_is_variable(key) >= 0) {
                             if (context.err_info) {
                                 add_dmc_err(context.err_info,
@@ -1405,7 +1405,7 @@ restart:
                         DMC_PUSH(text, dmc_private_copy(&context, key));
                         {
                             int old_stack = ++(context.stack_used);
-                            Eterm value = map_get_values(map_val(t))[i];
+                            Eterm value = flatmap_get_values(flatmap_val(t))[i];
                             res = dmc_one_term(&context, &heap, &stack, &text,
                                                value);
                             ASSERT(res != retFail);
@@ -2004,12 +2004,12 @@ restart:
 	    ++ep;
 	    break;
         case matchMap:
-            if (!is_map_rel(*ep, base) && !is_hashmap_rel(*ep,base)) {
+            if (!is_flatmap_rel(*ep, base) && !is_hashmap_rel(*ep,base)) {
                 FAIL();
             }
             n = *pc++;
-            if (is_map_rel(*ep,base)) {
-		if (map_get_size(map_val_rel(*ep, base)) < n) {
+            if (is_flatmap_rel(*ep,base)) {
+		if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) {
 		    FAIL();
 		}
             } else {
@@ -2018,15 +2018,15 @@ restart:
 		    FAIL();
 		}
 	    }
-            ep = map_val_rel(*ep, base);
+            ep = flatmap_val_rel(*ep, base);
             break;
         case matchPushM:
-            if (!is_map_rel(*ep, base) && !is_hashmap_rel(*ep, base)) {
+            if (!is_flatmap_rel(*ep, base) && !is_hashmap_rel(*ep, base)) {
                 FAIL();
             }
             n = *pc++;
-            if (is_map_rel(*ep,base)) {
-		if (map_get_size(map_val_rel(*ep, base)) < n) {
+            if (is_flatmap_rel(*ep,base)) {
+		if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) {
 		    FAIL();
 		}
 	    } else {
@@ -2035,11 +2035,11 @@ restart:
 		    FAIL();
 		}
 	    }
-            *sp++ = map_val_rel(*ep++, base);
+            *sp++ = flatmap_val_rel(*ep++, base);
             break;
         case matchKey:
             t = (Eterm) *pc++;
-            tp = erts_maps_get_rel(t, make_map_rel(ep, base), base);
+            tp = erts_maps_get_rel(t, make_flatmap_rel(ep, base), base);
             if (!tp) {
                 FAIL();
             }
@@ -2156,12 +2156,12 @@ restart:
             ehp = HAllocX(build_proc, 1 + MAP_HEADER_SIZE + n, HEAP_XTRA);
             t = *ehp++ = *--esp;
             {
-                map_t *m = (map_t *)ehp;
+                flatmap_t *m = (flatmap_t *)ehp;
                 m->thing_word = MAP_HEADER;
                 m->size = n;
                 m->keys = t;
             }
-            t = make_map(ehp);
+            t = make_flatmap(ehp);
             ehp += MAP_HEADER_SIZE;
             while (n--) {
                 *ehp++ = *--esp;
@@ -3373,10 +3373,10 @@ int db_has_variable(Eterm node) {
 		while(arity--) {
 		    ESTACK_PUSH(s,*(++tuple));
 		}
-            } else if (is_map(node)) {
-                Eterm *values = map_get_values(map_val(node));
-                int size = map_get_size(map_val(node));
-                ESTACK_PUSH(s, ((map_t *) map_val(node))->keys);
+            } else if (is_flatmap(node)) {
+                Eterm *values = flatmap_get_values(flatmap_val(node));
+                Uint size = flatmap_get_size(flatmap_val(node));
+                ESTACK_PUSH(s, ((flatmap_t *) flatmap_val(node))->keys);
                 while (size--) {
                     ESTACK_PUSH(s, *(values++));
                 }
@@ -3540,7 +3540,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 	    DMC_PUSH(*stack, c);
 	    break;
         case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):
-            n = map_get_size(map_val(c));
+            n = flatmap_get_size(flatmap_val(c));
             DMC_PUSH(*text, matchPushM);
             ++(context->stack_used);
             DMC_PUSH(*text, n);
@@ -3841,11 +3841,11 @@ dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
     int nelems;
     int constant_values;
     DMCRet ret;
-    if (is_map(t)) {
-        map_t *m = (map_t *)map_val(t);
-        Eterm *values = map_get_values(m);
+    if (is_flatmap(t)) {
+        flatmap_t *m = (flatmap_t *)flatmap_val(t);
+        Eterm *values = flatmap_get_values(m);
 
-        nelems = map_get_size(m);
+        nelems = flatmap_get_size(m);
         ret = dmc_array(context, heap, text, values, nelems, &constant_values);
 
         if (ret != retOk) {
@@ -4915,7 +4915,7 @@ static DMCRet dmc_expr(DMCContext *context,
 	    return ret;
 	break;
     case TAG_PRIMARY_BOXED:
-        if (is_map(t) || is_hashmap(t)) {
+        if (is_flatmap(t) || is_hashmap(t)) {
             return dmc_map(context, heap, text, t, constant);
         }
 	if (!is_tuple(t)) {
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 3fec553684..8afcb060a1 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -55,7 +55,7 @@ do {                                                                    \
     nelts = header_arity(HDR);                                          \
     switch ((HDR) & _HEADER_SUBTAG_MASK) {                              \
     case SUB_BINARY_SUBTAG: nelts++; break;                             \
-    case MAP_SUBTAG: nelts+=map_get_size(PTR) + 1; break;               \
+    case MAP_SUBTAG: nelts+=flatmap_get_size(PTR) + 1; break;               \
     case HASHMAP_SUBTAG: nelts=hashmap_bitcount(MAP_HEADER_VAL(HDR));   \
 	 nelts += is_hashmap_header_head(HDR) ? 1 : 0; break;           \
     case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break;   \
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 65a31d3680..ca43baf1a6 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -78,7 +78,7 @@ typedef struct {
 } hxnode_t;
 
 
-static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB);
+static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
 static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm hashmap_to_list(Process *p, Eterm map);
@@ -101,11 +101,11 @@ static int hxnodecmpkey(hxnode_t* a, hxnode_t* b);
  */
 
 BIF_RETTYPE map_size_1(BIF_ALIST_1) {
-    if (is_map(BIF_ARG_1)) {
+    if (is_flatmap(BIF_ARG_1)) {
 	Eterm *hp;
 	Uint hsz  = 0;
-	map_t *mp = (map_t*)map_val(BIF_ARG_1);
-	Uint n    = map_get_size(mp);
+	flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
+	Uint n    = flatmap_get_size(mp);
 
 	erts_bld_uint(NULL, &hsz, n);
 	hp = HAlloc(BIF_P, hsz);
@@ -128,15 +128,15 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) {
 /* maps:to_list/1 */
 
 BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
-    if (is_map(BIF_ARG_1)) {
+    if (is_flatmap(BIF_ARG_1)) {
 	Uint n;
 	Eterm* hp;
 	Eterm *ks,*vs, res, tup;
-	map_t *mp = (map_t*)map_val(BIF_ARG_1);
+	flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
 
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
-	n   = map_get_size(mp);
+	ks  = flatmap_get_keys(mp);
+	vs  = flatmap_get_values(mp);
+	n   = flatmap_get_size(mp);
 	hp  = HAlloc(BIF_P, (2 + 3) * n);
 	res = NIL;
 
@@ -165,20 +165,20 @@ erts_maps_get(Eterm key, Eterm map)
 #endif
 {
     Uint32 hx;
-    if (is_map(map)) {
+    if (is_flatmap(map)) {
 	Eterm *ks, *vs;
-	map_t *mp;
+	flatmap_t *mp;
 	Uint n, i;
 
-	mp  = (map_t *)map_val_rel(map, map_base);
-	n   = map_get_size(mp);
+	mp  = (flatmap_t *)flatmap_val_rel(map, map_base);
+	n   = flatmap_get_size(mp);
 
 	if (n == 0) {
 	    return NULL;
 	}
 
 	ks  = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1;
-	vs  = map_get_values(mp);
+	vs  = flatmap_get_values(mp);
 
 	if (is_immed(key)) {
 	    for (i = 0; i < n; i++) {
@@ -202,7 +202,7 @@ erts_maps_get(Eterm key, Eterm map)
 }
 
 BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
         Eterm *hp, res;
         const Eterm *value;
 
@@ -226,7 +226,7 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
  */
 
 BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
 	Eterm *hp;
         Eterm error;
         const Eterm *value;
@@ -289,7 +289,7 @@ error:
 static Eterm map_from_validated_list(Process *p, Eterm list, Uint size) {
     Eterm *kv, item = list;
     Eterm *hp, *thp,*vs, *ks, keys, res;
-    map_t *mp;
+    flatmap_t *mp;
     Uint  unused_size = 0;
     Sint  c = 0;
     Sint  idx = 0;
@@ -301,8 +301,8 @@ static Eterm map_from_validated_list(Process *p, Eterm list, Uint size) {
     *hp++ = make_arityval(size);
     ks    = hp;
     hp   += size;
-    mp    = (map_t*)hp;
-    res   = make_map(mp);
+    mp    = (flatmap_t*)hp;
+    res   = make_flatmap(mp);
     hp   += MAP_HEADER_SIZE;
     vs    = hp;
 
@@ -427,7 +427,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     if (hashmap_size(res) <= MAP_SMALL_MAP_LIMIT) {
         DECLARE_WSTACK(wstack);
 	Eterm *kv, *ks, *vs;
-	map_t *mp;
+	flatmap_t *mp;
 	Eterm keys;
         Uint n = hashmap_size(res);
 
@@ -437,7 +437,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
 	*hp++ = make_arityval(n);
 	ks    = hp;
 	hp   += n;
-	mp    = (map_t*)hp;
+	mp    = (flatmap_t*)hp;
 	hp   += MAP_HEADER_SIZE;
 	vs    = hp;
 
@@ -453,10 +453,10 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
 	}
 
 	/* it cannot have multiple keys */
-	erts_validate_and_sort_map(mp);
+	erts_validate_and_sort_flatmap(mp);
 
 	DESTROY_WSTACK(wstack);
-	return make_map(mp);
+	return make_flatmap(mp);
     }
 
     return res;
@@ -877,7 +877,7 @@ static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
 /* maps:is_key/2 */
 
 BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
 	BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -886,19 +886,19 @@ BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
 /* maps:keys/1 */
 
 BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
-    if (is_map(BIF_ARG_1)) {
+    if (is_flatmap(BIF_ARG_1)) {
 	Eterm *hp, *ks, res = NIL;
-	map_t *mp;
+	flatmap_t *mp;
 	Uint n;
 
-	mp  = (map_t*)map_val(BIF_ARG_1);
-	n   = map_get_size(mp);
+	mp  = (flatmap_t*)flatmap_val(BIF_ARG_1);
+	n   = flatmap_get_size(mp);
 
 	if (n == 0)
 	    BIF_RET(res);
 
 	hp  = HAlloc(BIF_P, (2 * n));
-	ks  = map_get_keys(mp);
+	ks  = flatmap_get_keys(mp);
 
 	while(n--) {
 	    res = CONS(hp, ks[n], res); hp += 2;
@@ -913,9 +913,9 @@ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
 /* maps:merge/2 */
 
 BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_1)) {
-	if (is_map(BIF_ARG_2)) {
-	    BIF_RET(map_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
+    if (is_flatmap(BIF_ARG_1)) {
+	if (is_flatmap(BIF_ARG_2)) {
+	    BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
 	} else if (is_hashmap(BIF_ARG_2)) {
 	    /* Will always become a tree */
 	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
@@ -923,7 +923,7 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
     } else if (is_hashmap(BIF_ARG_1)) {
 	if (is_hashmap(BIF_ARG_2)) {
 	    BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
-	} else if (is_map(BIF_ARG_2)) {
+	} else if (is_flatmap(BIF_ARG_2)) {
 	    /* Will always become a tree */
 	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1));
 	}
@@ -931,18 +931,18 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
-static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
+static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     Eterm *hp,*thp;
     Eterm tup;
     Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2;
-    map_t *mp1,*mp2,*mp_new;
+    flatmap_t *mp1,*mp2,*mp_new;
     Uint n,n1,n2,i1,i2,need,unused_size=0;
     int c = 0;
 
-    mp1  = (map_t*)map_val(nodeA);
-    mp2  = (map_t*)map_val(nodeB);
-    n1   = map_get_size(mp1);
-    n2   = map_get_size(mp2);
+    mp1  = (flatmap_t*)flatmap_val(nodeA);
+    mp2  = (flatmap_t*)flatmap_val(nodeB);
+    n1   = flatmap_get_size(mp1);
+    n2   = flatmap_get_size(mp2);
 
     need = MAP_HEADER_SIZE + 1 + 2*(n1 + n2);
 
@@ -950,7 +950,7 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     thp    = hp;
     tup    = make_tuple(thp);
     ks     = hp + 1; hp += 1 + n1 + n2;
-    mp_new = (map_t*)hp; hp += MAP_HEADER_SIZE;
+    mp_new = (flatmap_t*)hp; hp += MAP_HEADER_SIZE;
     vs     = hp; hp += n1 + n2;
 
     mp_new->thing_word = MAP_HEADER;
@@ -958,10 +958,10 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     mp_new->keys = tup;
 
     i1  = 0; i2 = 0;
-    ks1 = map_get_keys(mp1);
-    vs1 = map_get_values(mp1);
-    ks2 = map_get_keys(mp2);
-    vs2 = map_get_values(mp2);
+    ks1 = flatmap_get_keys(mp1);
+    vs1 = flatmap_get_values(mp1);
+    ks2 = flatmap_get_keys(mp2);
+    vs2 = flatmap_get_values(mp2);
 
     while(i1 < n1 && i2 < n2) {
 	c = CMP_TERM(ks1[i1],ks2[i2]);
@@ -1018,8 +1018,8 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 	hxnode_t *hxns;
         ErtsHeapFactory factory;
 
-	ks = map_get_keys(mp_new);
-	vs = map_get_values(mp_new);
+	ks = flatmap_get_keys(mp_new);
+	vs = flatmap_get_values(mp_new);
 
 	hp = HAlloc(p, 2 * n);
 
@@ -1045,12 +1045,12 @@ static Eterm map_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 
     mp_new->size = n;
 
-    return make_map(mp_new);
+    return make_flatmap(mp_new);
 }
 
 static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) {
     Eterm *ks, *vs, *hp, res;
-    map_t *mp;
+    flatmap_t *mp;
     Uint n, i;
     hxnode_t *hxns;
     Uint32 sw, hx;
@@ -1058,14 +1058,14 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
 
     /* convert flat to tree */
 
-    ASSERT(is_map(flat));
+    ASSERT(is_flatmap(flat));
     ASSERT(is_hashmap(tree));
 
-    mp = (map_t*)map_val(flat);
-    n  = map_get_size(mp);
+    mp = (flatmap_t*)flatmap_val(flat);
+    n  = flatmap_get_size(mp);
 
-    ks = map_get_keys(mp);
-    vs = map_get_values(mp);
+    ks = flatmap_get_keys(mp);
+    vs = flatmap_get_values(mp);
 
     hp = HAlloc(p, 2 * n);
 
@@ -1343,24 +1343,24 @@ int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
 BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
     Eterm* hp;
     Eterm tup;
-    map_t *mp;
+    flatmap_t *mp;
 
     hp    = HAlloc(BIF_P, (MAP_HEADER_SIZE + 1));
     tup   = make_tuple(hp);
     *hp++ = make_arityval(0);
 
-    mp    = (map_t*)hp;
+    mp    = (flatmap_t*)hp;
     mp->thing_word = MAP_HEADER;
     mp->size = 0;
     mp->keys = tup;
 
-    BIF_RET(make_map(mp));
+    BIF_RET(make_flatmap(mp));
 }
 
 /* maps:put/3 */
 
 BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
-    if (is_map(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
+    if (is_flatmap(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
 	BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3));
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -1370,23 +1370,23 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
 
 int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
     Uint32 hx;
-    if (is_map(map)) {
+    if (is_flatmap(map)) {
 	Sint n;
 	Uint need;
 	Eterm *hp_start;
 	Eterm *thp, *mhp;
 	Eterm *ks, *vs, tup;
-	map_t *mp = (map_t*)map_val(map);
+	flatmap_t *mp = (flatmap_t*)flatmap_val(map);
 
-	n = map_get_size(mp);
+	n = flatmap_get_size(mp);
 
 	if (n == 0) {
 	    *res = map;
 	    return 1;
 	}
 
-	ks = map_get_keys(mp);
-	vs = map_get_values(mp);
+	ks = flatmap_get_keys(mp);
+	vs = flatmap_get_values(mp);
 
 	/* Assume key exists.
 	 * Release allocated if it didn't.
@@ -1401,7 +1401,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 	tup    = make_tuple(thp);
 	*thp++ = make_arityval(n - 1);
 
-	*res   = make_map(mhp);
+	*res   = make_flatmap(mhp);
 	*mhp++ = MAP_HEADER;
 	*mhp++ = n - 1;
 	*mhp++ = tup;
@@ -1451,7 +1451,7 @@ found_key:
 }
 
 BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
 	Eterm res;
 	if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) {
 	    BIF_RET(res);
@@ -1462,18 +1462,18 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
 
 int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
     Uint32 hx;
-    if (is_map(map)) {
+    if (is_flatmap(map)) {
 	Sint n,i;
 	Eterm* hp,*shp;
 	Eterm *ks,*vs;
-	map_t *mp = (map_t*)map_val(map);
+	flatmap_t *mp = (flatmap_t*)flatmap_val(map);
 
-	if ((n = map_get_size(mp)) == 0) {
+	if ((n = flatmap_get_size(mp)) == 0) {
 	    return 0;
 	}
 
-	ks  = map_get_keys(mp);
-	vs  = map_get_values(mp);
+	ks  = flatmap_get_keys(mp);
+	vs  = flatmap_get_values(mp);
 
 	/* only allocate for values,
 	 * assume key-tuple will be intact
@@ -1511,7 +1511,7 @@ found_key:
 	vs++;
 	if (++i < n)
 	    sys_memcpy(hp, vs, (n - i)*sizeof(Eterm));
-	*res = make_map(shp);
+	*res = make_flatmap(shp);
 	return 1;
     }
 
@@ -1527,21 +1527,21 @@ found_key:
 Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
     Uint32 hx;
     Eterm res;
-    if (is_map(map)) {
+    if (is_flatmap(map)) {
 	Sint n,i;
 	Sint c = 0;
 	Eterm* hp, *shp;
 	Eterm *ks, *vs, tup;
-	map_t *mp = (map_t*)map_val(map);
+	flatmap_t *mp = (flatmap_t*)flatmap_val(map);
 
-	n = map_get_size(mp);
+	n = flatmap_get_size(mp);
 
 	if (n == 0) {
 	    hp    = HAlloc(p, MAP_HEADER_SIZE + 1 + 2);
 	    tup   = make_tuple(hp);
 	    *hp++ = make_arityval(1);
 	    *hp++ = key;
-	    res   = make_map(hp);
+	    res   = make_flatmap(hp);
 	    *hp++ = MAP_HEADER;
 	    *hp++ = 1;
 	    *hp++ = tup;
@@ -1550,8 +1550,8 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 	    return res;
 	}
 
-	ks = map_get_keys(mp);
-	vs = map_get_values(mp);
+	ks = flatmap_get_keys(mp);
+	vs = flatmap_get_values(mp);
 
 	/* only allocate for values,
 	 * assume key-tuple will be intact
@@ -1559,7 +1559,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 
 	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
 	shp = hp; /* save hp, used if optimistic update fails */
-	res = make_map(hp);
+	res = make_flatmap(hp);
 	*hp++ = MAP_HEADER;
 	*hp++ = n;
 	*hp++ = mp->keys;
@@ -1593,8 +1593,8 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 
 	if (n >= MAP_SMALL_MAP_LIMIT) {
 	    HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
-	    ks = map_get_keys(mp);
-	    vs = map_get_values(mp);
+	    ks = flatmap_get_keys(mp);
+	    vs = flatmap_get_values(mp);
 
 	    res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value);
 
@@ -1608,13 +1608,13 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 	*shp++ = make_arityval(n+1);
 
 	hp    = HAlloc(p, 3 + n + 1);
-	res   = make_map(hp);
+	res   = make_flatmap(hp);
 	*hp++ = MAP_HEADER;
 	*hp++ = n + 1;
 	*hp++ = tup;
 
-	ks = map_get_keys(mp);
-	vs = map_get_values(mp);
+	ks = flatmap_get_keys(mp);
+	vs = flatmap_get_values(mp);
 
 	ASSERT(n >= 0);
 
@@ -1653,7 +1653,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 /* maps:update/3 */
 
 BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
-    if (is_map(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
+    if (is_flatmap(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
 	Eterm res;
 	if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) {
 	    BIF_RET(res);
@@ -1666,19 +1666,19 @@ BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
 /* maps:values/1 */
 
 BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
-    if (is_map(BIF_ARG_1)) {
+    if (is_flatmap(BIF_ARG_1)) {
 	Eterm *hp, *vs, res = NIL;
-	map_t *mp;
+	flatmap_t *mp;
 	Uint n;
 
-	mp  = (map_t*)map_val(BIF_ARG_1);
-	n   = map_get_size(mp);
+	mp  = (flatmap_t*)flatmap_val(BIF_ARG_1);
+	n   = flatmap_get_size(mp);
 
 	if (n == 0)
 	    BIF_RET(res);
 
 	hp  = HAlloc(BIF_P, (2 * n));
-	vs  = map_get_values(mp);
+	vs  = flatmap_get_values(mp);
 
 	while(n--) {
 	    res = CONS(hp, vs[n], res); hp += 2;
@@ -2257,7 +2257,7 @@ unroll:
     if (n <= MAP_SMALL_MAP_LIMIT) {
 	DECLARE_WSTACK(wstack);
 	Eterm *kv, *ks, *vs;
-	map_t *mp;
+	flatmap_t *mp;
 	Eterm keys;
 
 	DESTROY_ESTACK(stack);
@@ -2268,7 +2268,7 @@ unroll:
 	*hp++ = make_arityval(n);
 	ks    = hp;
 	hp   += n;
-	mp    = (map_t*)hp;
+	mp    = (flatmap_t*)hp;
 	hp   += MAP_HEADER_SIZE;
 	vs    = hp;
 
@@ -2286,10 +2286,10 @@ unroll:
 	}
 
 	/* it cannot have multiple keys */
-	erts_validate_and_sort_map(mp);
+	erts_validate_and_sort_flatmap(mp);
 
 	DESTROY_WSTACK(wstack);
-	return make_map(mp);
+	return make_flatmap(mp);
     }
 
     hp     = HAlloc(p, size);
@@ -2451,11 +2451,11 @@ not_found:
 }
 
 
-int erts_validate_and_sort_map(map_t* mp)
+int erts_validate_and_sort_flatmap(flatmap_t* mp)
 {
-    Eterm *ks  = map_get_keys(mp);
-    Eterm *vs  = map_get_values(mp);
-    Uint   sz  = map_get_size(mp);
+    Eterm *ks  = flatmap_get_keys(mp);
+    Eterm *vs  = flatmap_get_values(mp);
+    Uint   sz  = flatmap_get_size(mp);
     Uint   ix,jx;
     Eterm  tmp;
     int c;
@@ -2533,8 +2533,8 @@ BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) {
  */
 
 BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
-    if (is_map(BIF_ARG_1)) {
-	map_t *mp = (map_t*)map_val(BIF_ARG_1);
+    if (is_flatmap(BIF_ARG_1)) {
+	flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
 	BIF_RET(mp->keys);
     }
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index db22d461ce..884bf69a34 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -34,11 +34,11 @@ Uint32 hashmap_bitcount(Uint32 x);
 
 /* MAP */
 
-typedef struct map_s {
+typedef struct flatmap_s {
     Eterm thing_word;
     Uint  size;
     Eterm keys;      /* tuple */
-} map_t;
+} flatmap_t;
 /* map node
  *
  * -----------
@@ -66,23 +66,23 @@ typedef struct map_s {
 
 
 /* erl_term.h stuff */
-#define make_map(x)		make_boxed((Eterm*)(x))
-#define make_map_rel(x, BASE)   make_boxed_rel((Eterm*)(x),(BASE))
-#define is_map(x)		(is_boxed((x)) && is_map_header(*boxed_val((x))))
-#define is_map_rel(RTERM,BASE)  is_map(rterm2wterm(RTERM,BASE))
-#define is_not_map(x)           (!is_map((x)))
-#define is_map_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
-#define header_is_map(x)        ((((x) & (_HEADER_SUBTAG_MASK)) == MAP_SUBTAG))
-#define map_val(x)		(_unchecked_boxed_val((x)))
-#define map_val_rel(RTERM, BASE) map_val(rterm2wterm(RTERM, BASE))
-
-#define map_get_values(x)      (((Eterm *)(x)) + 3)
-#define map_get_keys(x)        (((Eterm *)tuple_val(((map_t *)(x))->keys)) + 1)
-#define map_get_size(x)        (((map_t*)(x))->size)
+#define make_flatmap(x)		make_boxed((Eterm*)(x))
+#define make_flatmap_rel(x, BASE)   make_boxed_rel((Eterm*)(x),(BASE))
+#define is_flatmap(x)		(is_boxed((x)) && is_flatmap_header(*boxed_val((x))))
+#define is_flatmap_rel(RTERM,BASE)  is_flatmap(rterm2wterm(RTERM,BASE))
+#define is_not_flatmap(x)           (!is_flatmap((x)))
+#define is_flatmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
+#define header_is_flatmap(x)        ((((x) & (_HEADER_SUBTAG_MASK)) == MAP_SUBTAG))
+#define flatmap_val(x)		(_unchecked_boxed_val((x)))
+#define flatmap_val_rel(RTERM, BASE) flatmap_val(rterm2wterm(RTERM, BASE))
+
+#define flatmap_get_values(x)      (((Eterm *)(x)) + 3)
+#define flatmap_get_keys(x)        (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1)
+#define flatmap_get_size(x)        (((flatmap_t*)(x))->size)
 
 #define MAP_SMALL_MAP_LIMIT    (2) /*SVERK (32) */
 #define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
-#define MAP_HEADER_SIZE        (sizeof(map_t) / sizeof(Eterm))
+#define MAP_HEADER_SIZE        (sizeof(flatmap_t) / sizeof(Eterm))
 
 struct ErtsWStack_;
 struct ErtsEStack_;
@@ -98,7 +98,7 @@ int    erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 Eterm  erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 			      Uint *upsz, struct ErtsEStack_ *sp);
 
-int    erts_validate_and_sort_map(map_t* map);
+int    erts_validate_and_sort_flatmap(flatmap_t* map);
 Uint   hashmap_over_estimated_heap_size(Uint n);
 void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 198acfd128..fd793fd7e4 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1905,15 +1905,15 @@ enif_is_on_dirty_scheduler(ErlNifEnv* env)
 
 int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term)
 {
-    return is_map(term);
+    return is_flatmap(term);
 }
 
 int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)
 {
-    if (is_map(term)) {
-	map_t *mp;
-	mp    = (map_t*)map_val(term);
-	*size = map_get_size(mp);
+    if (is_flatmap(term)) {
+	flatmap_t *mp;
+	mp    = (flatmap_t*)flatmap_val(term);
+	*size = flatmap_get_size(mp);
 	return 1;
     }
     return 0;
@@ -1923,16 +1923,16 @@ ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env)
 {
     Eterm* hp = alloc_heap(env,MAP_HEADER_SIZE+1);
     Eterm tup;
-    map_t *mp;
+    flatmap_t *mp;
 
     tup   = make_tuple(hp);
     *hp++ = make_arityval(0);
-    mp    = (map_t*)hp;
+    mp    = (flatmap_t*)hp;
     mp->thing_word = MAP_HEADER;
     mp->size = 0;
     mp->keys = tup;
 
-    return make_map(mp);
+    return make_flatmap(mp);
 }
 
 int enif_make_map_put(ErlNifEnv* env,
@@ -1941,7 +1941,7 @@ int enif_make_map_put(ErlNifEnv* env,
 		      Eterm value,
 		      Eterm *map_out)
 {
-    if (is_not_map(map_in)) {
+    if (is_not_flatmap(map_in)) {
 	return 0;
     }
     flush_env(env);
@@ -1956,7 +1956,7 @@ int enif_get_map_value(ErlNifEnv* env,
 		       Eterm *value)
 {
     const Eterm *ret;
-    if (is_not_map(map)) {
+    if (is_not_flatmap(map)) {
 	return 0;
     }
     ret = erts_maps_get(key, map);
@@ -1974,7 +1974,7 @@ int enif_make_map_update(ErlNifEnv* env,
 			 Eterm *map_out)
 {
     int res;
-    if (is_not_map(map_in)) {
+    if (is_not_flatmap(map_in)) {
 	return 0;
     }
 
@@ -1990,7 +1990,7 @@ int enif_make_map_remove(ErlNifEnv* env,
 			 Eterm *map_out)
 {
     int res;
-    if (is_not_map(map_in)) {
+    if (is_not_flatmap(map_in)) {
 	return 0;
     }
     flush_env(env);
@@ -2004,13 +2004,13 @@ int enif_map_iterator_create(ErlNifEnv *env,
 			     ErlNifMapIterator *iter,
 			     ErlNifMapIteratorEntry entry)
 {
-    if (is_map(map)) {
-	map_t *mp = (map_t*)map_val(map);
+    if (is_flatmap(map)) {
+	flatmap_t *mp = (flatmap_t*)flatmap_val(map);
 	size_t offset;
 
 	switch (entry) {
 	    case ERL_NIF_MAP_ITERATOR_HEAD: offset = 0; break;
-	    case ERL_NIF_MAP_ITERATOR_TAIL: offset = map_get_size(mp) - 1; break;
+	    case ERL_NIF_MAP_ITERATOR_TAIL: offset = flatmap_get_size(mp) - 1; break;
 	    default: goto error;
 	}
 
@@ -2019,9 +2019,9 @@ int enif_map_iterator_create(ErlNifEnv *env,
 	 */
 
 	iter->map     = map;
-	iter->ks      = ((Eterm *)map_get_keys(mp)) + offset;
-	iter->vs      = ((Eterm *)map_get_values(mp)) + offset;
-	iter->t_limit = map_get_size(mp) + 1;
+	iter->ks      = ((Eterm *)flatmap_get_keys(mp)) + offset;
+	iter->vs      = ((Eterm *)flatmap_get_values(mp)) + offset;
+	iter->t_limit = flatmap_get_size(mp) + 1;
 	iter->idx     = offset + 1;
 
 	return 1;
@@ -2045,22 +2045,22 @@ void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter)
 
 int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_map(iter->map));
-    ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
+    ASSERT(iter && is_flatmap(iter->map));
+    ASSERT(iter->idx >= 0 && (iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1));
     return (iter->t_limit == 1 || iter->idx == iter->t_limit);
 }
 
 int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_map(iter->map));
-    ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1));
+    ASSERT(iter && is_flatmap(iter->map));
+    ASSERT(iter->idx >= 0 && (iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1));
     return (iter->t_limit == 1 || iter->idx == 0);
 }
 
 
 int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_map(iter->map));
+    ASSERT(iter && is_flatmap(iter->map));
     if (iter->idx < iter->t_limit) {
 	iter->idx++;
 	iter->ks++;
@@ -2071,7 +2071,7 @@ int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
 
 int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_map(iter->map));
+    ASSERT(iter && is_flatmap(iter->map));
     if (iter->idx > 0) {
 	iter->idx--;
 	iter->ks--;
@@ -2085,12 +2085,12 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
 			       Eterm *key,
 			       Eterm *value)
 {
-    ASSERT(iter && is_map(iter->map));
+    ASSERT(iter && is_flatmap(iter->map));
     if (iter->idx > 0 && iter->idx < iter->t_limit) {
-	ASSERT(iter->ks >= map_get_keys(map_val(iter->map)) &&
-	       iter->ks  < (map_get_keys(map_val(iter->map)) + map_get_size(map_val(iter->map))));
-	ASSERT(iter->vs >= map_get_values(map_val(iter->map)) &&
-	       iter->vs  < (map_get_values(map_val(iter->map)) + map_get_size(map_val(iter->map))));
+	ASSERT(iter->ks >= flatmap_get_keys(flatmap_val(iter->map)) &&
+	       iter->ks  < (flatmap_get_keys(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map))));
+	ASSERT(iter->vs >= flatmap_get_values(flatmap_val(iter->map)) &&
+	       iter->vs  < (flatmap_get_values(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map))));
 	*key   = *(iter->ks);
 	*value = *(iter->vs);
 	return 1;
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 81fd19693a..8046f54a0c 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -568,10 +568,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    {
 		Uint n;
 		Eterm *ks, *vs;
-		map_t *mp = (map_t *)map_val(wobj);
-		n  = map_get_size(mp);
-		ks = map_get_keys(mp);
-		vs = map_get_values(mp);
+		flatmap_t *mp = (flatmap_t *)flatmap_val(wobj);
+		n  = flatmap_get_size(mp);
+		ks = flatmap_get_keys(mp);
+		vs = flatmap_get_values(mp);
 
 		PRINT_CHAR(res, fn, arg, '#');
 		PRINT_CHAR(res, fn, arg, '{');
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index bd9ad65086..65b4ae5412 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2605,15 +2605,15 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 
 	case MAP_DEF:
 	    {
-		map_t *mp = (map_t*)map_val(obj);
-		Uint size = map_get_size(mp);
+		flatmap_t *mp = (flatmap_t*)flatmap_val(obj);
+		Uint size = flatmap_get_size(mp);
 
 		*ep++ = MAP_EXT;
 		put_int32(size, ep); ep += 4;
 
 		if (size > 0) {
-		    Eterm *kptr = map_get_keys(mp);
-		    Eterm *vptr = map_get_values(mp);
+		    Eterm *kptr = flatmap_get_keys(mp);
+		    Eterm *vptr = flatmap_get_values(mp);
 
 		    WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr,
 				 ENC_MAP_PAIR, size);
@@ -3599,14 +3599,14 @@ dec_term_atom_common:
 		size = get_int32(ep); ep += 4;
 
                 if (size <= MAP_SMALL_MAP_LIMIT) {
-                    map_t *mp;
+                    flatmap_t *mp;
 
                     keys  = make_tuple(hp);
                     *hp++ = make_arityval(size);
                     hp   += size;
                     kptr = hp - 1;
 
-                    mp    = (map_t*)hp;
+                    mp    = (flatmap_t*)hp;
                     hp   += MAP_HEADER_SIZE;
                     hp   += size;
                     vptr = hp - 1;
@@ -3625,7 +3625,7 @@ dec_term_atom_common:
 
                     mp->size       = size;
                     mp->keys       = keys;
-                    *objp          = make_map(mp);
+                    *objp          = make_flatmap(mp);
 
                     for (n = size; n; n--) {
                         *vptr = (Eterm) COMPRESS_POINTER(next);
@@ -3891,7 +3891,7 @@ dec_term_atom_common:
     while (maps_list) {
 	next  = (Eterm *)(EXPAND_POINTER(*maps_list));
 	*maps_list = MAP_HEADER;
-	if (!erts_validate_and_sort_map((map_t*)maps_list))
+	if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list))
 	    goto error;
 	maps_list  = next;
     }
@@ -4120,15 +4120,15 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    break;
 	case MAP_DEF:
 	    {
-		map_t *mp = (map_t*)map_val(obj);
-		Uint size = map_get_size(mp);
+		flatmap_t *mp = (flatmap_t*)flatmap_val(obj);
+		Uint size = flatmap_get_size(mp);
 		Uint i;
 		Eterm *ptr;
 
 		result += 1 + 4; /* tag + 4 bytes size */
 
 		/* push values first */
-		ptr = map_get_values(mp);
+		ptr = flatmap_get_values(mp);
 		i   = size;
 		while(i--) {
 		    if (is_list(*ptr)) {
@@ -4142,7 +4142,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		    ++ptr;
 		}
 
-		ptr = map_get_keys(mp);
+		ptr = flatmap_get_keys(mp);
 		i   = size;
 		while(i--) {
 		    if (is_list(*ptr)) {
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index dc4c6fc350..804d3ddf50 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5595,16 +5595,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    int size = (int)ptr[0];
 	    Eterm* tp = hp;
 	    Eterm* vp;
-	    map_t *mp;
+	    flatmap_t *mp;
 
 	    *tp = make_arityval(size);
 
 	    hp += 1 + size;
-	    mp = (map_t*)hp;
+	    mp = (flatmap_t*)hp;
 	    mp->thing_word = MAP_HEADER;
 	    mp->size = size;
 	    mp->keys = make_tuple(tp);
-	    mess = make_map(mp);
+	    mess = make_flatmap(mp);
 
 	    hp += MAP_HEADER_SIZE + size;   /* advance "heap" pointer */
 
@@ -5615,7 +5615,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		*vp-- = ESTACK_POP(stack);
 		*tp-- = ESTACK_POP(stack);
 	    }
-	    if (!erts_validate_and_sort_map(mp))
+	    if (!erts_validate_and_sort_flatmap(mp))
 		ERTS_DDT_FAIL;
 	    ptr++;
 	    break;
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 66f13461a1..cc97e2f3d1 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1274,11 +1274,11 @@ make_hash2(Eterm term)
 	    break;
 	    case MAP_SUBTAG:
 	    {
-		map_t *mp = (map_t *)map_val(term);
+		flatmap_t *mp = (flatmap_t *)flatmap_val(term);
 		int i;
-		int size  = map_get_size(mp);
-		Eterm *ks = map_get_keys(mp);
-		Eterm *vs = map_get_values(mp);
+		int size  = flatmap_get_size(mp);
+		Eterm *ks = flatmap_get_keys(mp);
+		Eterm *vs = flatmap_get_values(mp);
 		UINT32_HASH(size, HCONST_16);
 		if (size == 0) {
 		    goto hash2_common;
@@ -1669,11 +1669,11 @@ make_internal_hash(Eterm term)
 	    break;
 	    case MAP_SUBTAG:
 	    {
-		map_t *mp = (map_t *)map_val(term);
+		flatmap_t *mp = (flatmap_t *)flatmap_val(term);
 		int i;
-		int size  = map_get_size(mp);
-		Eterm *ks = map_get_keys(mp);
-		Eterm *vs = map_get_values(mp);
+		int size  = flatmap_get_size(mp);
+		Eterm *ks = flatmap_get_keys(mp);
+		Eterm *vs = flatmap_get_values(mp);
 		UINT32_HASH(size, HCONST_16);
 		if (size == 0) {
 		    goto pop_next;
@@ -2574,13 +2574,13 @@ tailrecur_ne:
 		}
 	    case MAP_SUBTAG:
 		{
-		    aa = map_val_rel(a, a_base);
+		    aa = flatmap_val_rel(a, a_base);
 		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
 			goto not_equal;
-		    bb = map_val_rel(b,b_base);
-		    sz = map_get_size((map_t*)aa);
+		    bb = flatmap_val_rel(b,b_base);
+		    sz = flatmap_get_size((flatmap_t*)aa);
 
-		    if (sz != map_get_size((map_t*)bb)) goto not_equal;
+		    if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal;
 		    if (sz == 0) goto pop_next;
 
 		    aa += 2;
@@ -3119,16 +3119,16 @@ tailrecur_ne:
 		++bb;
 		goto term_array;
 	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) :
-		if (!is_map_rel(b,b_base)) {
+		if (!is_flatmap_rel(b,b_base)) {
 		    a_tag = MAP_DEF;
 		    goto mixed_types;
 		}
-		aa = (Eterm *)map_val_rel(a,a_base);
-		bb = (Eterm *)map_val_rel(b,b_base);
+		aa = (Eterm *)flatmap_val_rel(a,a_base);
+		bb = (Eterm *)flatmap_val_rel(b,b_base);
 
-		i = map_get_size((map_t*)aa);
-		if (i != map_get_size((map_t*)bb)) {
-		    RETURN_NEQ((int)(i - map_get_size((map_t*)bb)));
+		i = flatmap_get_size((flatmap_t*)aa);
+		if (i != flatmap_get_size((flatmap_t*)bb)) {
+		    RETURN_NEQ((int)(i - flatmap_get_size((flatmap_t*)bb)));
 		}
 		if (i == 0) {
 		    goto pop_next;
-- 
cgit v1.2.3


From 2fadbefc264d33da526ec61cffc9847f724d8d92 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Mar 2015 19:57:07 +0100
Subject: erts: Reintroduce is_map macro

as shorthand for is_flatmap || is_hashmap
---
 erts/emulator/beam/beam_emu.c    |  2 +-
 erts/emulator/beam/erl_bif_op.c  |  2 +-
 erts/emulator/beam/erl_db_util.c |  6 +++---
 erts/emulator/beam/erl_map.c     | 12 ++++++------
 erts/emulator/beam/erl_term.h    |  3 +++
 5 files changed, 14 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 10ae3ed77c..289b60e1f9 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -699,7 +699,7 @@ void** beam_ops;
         Fail; 								  \
     }
 
-#define IsMap(Src, Fail) if (is_not_flatmap(Src) && is_not_hashmap(Src)) { Fail; }
+#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; }
 
 #define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; }
 
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index af9b7bd908..37dd6457db 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -324,7 +324,7 @@ BIF_RETTYPE is_record_3(BIF_ALIST_3)
 	
 BIF_RETTYPE is_map_1(BIF_ALIST_1)
 {
-    if (is_flatmap(BIF_ARG_1) || is_hashmap(BIF_ARG_1)) {
+    if (is_map(BIF_ARG_1)) {
 	BIF_RET(am_true);
     }
     BIF_RET(am_false);
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index cca3f381a1..efbefb7492 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2004,7 +2004,7 @@ restart:
 	    ++ep;
 	    break;
         case matchMap:
-            if (!is_flatmap_rel(*ep, base) && !is_hashmap_rel(*ep,base)) {
+            if (!is_map_rel(*ep, base)) {
                 FAIL();
             }
             n = *pc++;
@@ -2021,7 +2021,7 @@ restart:
             ep = flatmap_val_rel(*ep, base);
             break;
         case matchPushM:
-            if (!is_flatmap_rel(*ep, base) && !is_hashmap_rel(*ep, base)) {
+            if (!is_map_rel(*ep, base)) {
                 FAIL();
             }
             n = *pc++;
@@ -4915,7 +4915,7 @@ static DMCRet dmc_expr(DMCContext *context,
 	    return ret;
 	break;
     case TAG_PRIMARY_BOXED:
-        if (is_flatmap(t) || is_hashmap(t)) {
+        if (is_map(t)) {
             return dmc_map(context, heap, text, t, constant);
         }
 	if (!is_tuple(t)) {
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ca43baf1a6..16293668ad 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -202,7 +202,7 @@ erts_maps_get(Eterm key, Eterm map)
 }
 
 BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
-    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_map(BIF_ARG_2)) {
         Eterm *hp, res;
         const Eterm *value;
 
@@ -226,7 +226,7 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
  */
 
 BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
-    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_map(BIF_ARG_2)) {
 	Eterm *hp;
         Eterm error;
         const Eterm *value;
@@ -877,7 +877,7 @@ static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
 /* maps:is_key/2 */
 
 BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
-    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_map(BIF_ARG_2)) {
 	BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -1360,7 +1360,7 @@ BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
 /* maps:put/3 */
 
 BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
-    if (is_flatmap(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
+    if (is_map(BIF_ARG_3)) {
 	BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3));
     }
     BIF_ERROR(BIF_P, BADARG);
@@ -1451,7 +1451,7 @@ found_key:
 }
 
 BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
-    if (is_flatmap(BIF_ARG_2) || is_hashmap(BIF_ARG_2)) {
+    if (is_map(BIF_ARG_2)) {
 	Eterm res;
 	if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) {
 	    BIF_RET(res);
@@ -1653,7 +1653,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 /* maps:update/3 */
 
 BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
-    if (is_flatmap(BIF_ARG_3) || is_hashmap(BIF_ARG_3)) {
+    if (is_map(BIF_ARG_3)) {
 	Eterm res;
 	if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) {
 	    BIF_RET(res);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 6d97344868..1625a4ec15 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1019,6 +1019,9 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #define hashmap_val(x)		_unchecked_boxed_val((x))
 #define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE))
 
+#define is_map(x)            (is_flatmap(x) || is_hashmap(x))
+#define is_map_rel(x,BASE)   (is_flatmap_rel(x,BASE) || is_hashmap_rel(x,BASE))
+
 /* number tests */
 
 #define is_integer(x)		(is_small(x) || is_big(x))
-- 
cgit v1.2.3


From f9e568cbad942043592453d0fb7640d8bc02b1ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 11 Mar 2015 10:25:37 +0100
Subject: erts: Add hashmap construction to driver API

---
 erts/emulator/beam/io.c | 78 ++++++++++++++++++++++++++++++++-----------------
 1 file changed, 52 insertions(+), 26 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 804d3ddf50..b64854aac9 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5349,7 +5349,11 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	case ERL_DRV_MAP: { /* int */
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
 	    if ((int) ptr[0] < 0) ERTS_DDT_FAIL;
-	    need += MAP_HEADER_SIZE + 1 + 2*ptr[0];
+            if (ptr[0] > MAP_SMALL_MAP_LIMIT) {
+                need += hashmap_over_estimated_heap_size(ptr[0]);
+            } else {
+                need += MAP_HEADER_SIZE + 1 + 2*ptr[0];
+            }
 	    depth -= 2*ptr[0];
 	    if (depth < 0) ERTS_DDT_FAIL;
 	    ptr++;
@@ -5593,31 +5597,53 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_MAP: { /* int */
 	    int size = (int)ptr[0];
-	    Eterm* tp = hp;
-	    Eterm* vp;
-	    flatmap_t *mp;
-
-	    *tp = make_arityval(size);
-
-	    hp += 1 + size;
-	    mp = (flatmap_t*)hp;
-	    mp->thing_word = MAP_HEADER;
-	    mp->size = size;
-	    mp->keys = make_tuple(tp);
-	    mess = make_flatmap(mp);
-
-	    hp += MAP_HEADER_SIZE + size;   /* advance "heap" pointer */
-
-	    tp += size;    /* point at last key */
-	    vp = hp - 1;   /* point at last value */
-
-	    while(size--) {
-		*vp-- = ESTACK_POP(stack);
-		*tp-- = ESTACK_POP(stack);
-	    }
-	    if (!erts_validate_and_sort_flatmap(mp))
-		ERTS_DDT_FAIL;
-	    ptr++;
+            if (size > MAP_SMALL_MAP_LIMIT) {
+                int ix = 2*size;
+                ErtsHeapFactory factory;
+                Eterm* leafs = hp;
+
+                hp += 2*size;
+                while(ix--) { *--hp = ESTACK_POP(stack); }
+
+                hp += 2*size;
+                factory.p = NULL;
+                factory.hp = hp;
+                /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
+                factory.hp_end = hp + (ERTS_SWORD_MAX / sizeof(Eterm));
+
+                mess = erts_hashmap_from_array(&factory, leafs, size, 1);
+
+                if (is_non_value(mess))
+                    ERTS_DDT_FAIL;
+
+                hp = factory.hp;
+            } else {
+                Eterm* tp = hp;
+                Eterm* vp;
+                flatmap_t *mp;
+
+                *tp = make_arityval(size);
+
+                hp += 1 + size;
+                mp = (flatmap_t*)hp;
+                mp->thing_word = MAP_HEADER;
+                mp->size = size;
+                mp->keys = make_tuple(tp);
+                mess = make_flatmap(mp);
+
+                hp += MAP_HEADER_SIZE + size;   /* advance "heap" pointer */
+
+                tp += size;    /* point at last key */
+                vp = hp - 1;   /* point at last value */
+
+                while(size--) {
+                    *vp-- = ESTACK_POP(stack);
+                    *tp-- = ESTACK_POP(stack);
+                }
+                if (!erts_validate_and_sort_flatmap(mp))
+                    ERTS_DDT_FAIL;
+            }
+            ptr++;
 	    break;
 	}
 
-- 
cgit v1.2.3


From c8f731bfec32a34d49304ea78017b63af053eecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 11 Mar 2015 18:45:12 +0100
Subject: erts, kernel: Fix erts_debug:size/1 for hashmaps

This commit introduces two BIFs:
* erts_internal:map_type/1
* erts_internal:map_hashmap_children/1

erts_internal:map_hashmap_children/1 is only intended for use within
erts_debug:size/1 since the internal hashmap node is not allowed to
leak anywhere.
---
 erts/emulator/beam/bif.tab   |  2 ++
 erts/emulator/beam/erl_map.c | 75 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index b4e821a986..c56a108b34 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -157,6 +157,8 @@ bif erts_internal:request_system_task/3
 bif erts_internal:check_process_code/2
 
 bif erts_internal:map_to_tuple_keys/1
+bif erts_internal:map_type/1
+bif erts_internal:map_hashmap_children/1
 
 # inet_db support
 bif erlang:port_set_data/2
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 16293668ad..0e24f2e1b1 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2540,6 +2540,81 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADARG);
 }
 
+/*
+ * erts_internal:map_type/1
+ *
+ * Used in erts_debug:size/1
+ */
+
+BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
+    DECL_AM(hashmap);
+    DECL_AM(hashmap_node);
+    DECL_AM(flatmap);
+    if (is_flatmap(BIF_ARG_1)) {
+	BIF_RET(AM_flatmap);
+    } else if (is_hashmap(BIF_ARG_1)) {
+        Eterm hdr = *(boxed_val(BIF_ARG_1));
+        ASSERT(is_header(hdr));
+        switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_HEAD_ARRAY:
+            case HAMT_SUBTAG_HEAD_BITMAP:
+                BIF_RET(AM_hashmap);
+            case HAMT_SUBTAG_NODE_ARRAY:
+            case HAMT_SUBTAG_NODE_BITMAP:
+                BIF_RET(AM_hashmap_node);
+            default:
+                erl_exit(1, "bad header");
+        }
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+/*
+ * erts_internal:map_hashmap_children/1
+ *
+ * Used in erts_debug:size/1
+ */
+
+BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) {
+    if (is_hashmap(BIF_ARG_1)) {
+        Eterm node = BIF_ARG_1;
+        Eterm *ptr, hdr, *hp, res = NIL;
+        Uint  sz = 0;
+        ptr = boxed_val(node);
+        hdr = *ptr;
+
+        ASSERT(is_header(hdr));
+
+        switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_NODE_ARRAY:
+                sz   = 16;
+                ptr += 1;
+                break;
+            case HAMT_SUBTAG_NODE_BITMAP:
+                sz   = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                ptr += 1;
+                break;
+            case HAMT_SUBTAG_HEAD_BITMAP:
+                sz   = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                ptr += 2;
+                break;
+            case HAMT_SUBTAG_HEAD_ARRAY:
+                sz   = 16;
+                ptr += 2;
+                break;
+            default:
+                erl_exit(1, "bad header\r\n");
+                break;
+        }
+        ASSERT(sz < 17);
+        hp = HAlloc(BIF_P, 2*sz);
+        while(sz--) { res = CONS(hp, *ptr++, res); hp += 2; }
+        BIF_RET(res);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+
 static Eterm hashmap_info(Process *p, Eterm node) {
     Eterm *hp;
     Eterm res = NIL, info = NIL;
-- 
cgit v1.2.3


From 81551dd13167b9c4458c4fee7ce08e152f9f5273 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Mar 2015 20:39:08 +0100
Subject: erts: Make hashmap iterator more flexible

to allow mixing of 'next' and 'prev' operations.
---
 erts/emulator/beam/erl_db_util.c |   6 +-
 erts/emulator/beam/erl_map.c     | 174 +++++++++++++++++++++++----------------
 erts/emulator/beam/erl_map.h     |   2 +-
 erts/emulator/beam/utils.c       |   4 +-
 4 files changed, 108 insertions(+), 78 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index efbefb7492..e522876d03 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1435,7 +1435,7 @@ restart:
                     }
                     structure_checked = 0;
 
-                    hashmap_iterator_init(&wstack, t);
+                    hashmap_iterator_init(&wstack, t, 0);
 
                     while ((kv=hashmap_iterator_next(&wstack)) != NULL) {
                         Eterm key = CAR(kv);
@@ -3872,7 +3872,7 @@ dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
 
         ASSERT(is_hashmap(t));
 
-        hashmap_iterator_init(&wstack, t);
+        hashmap_iterator_init(&wstack, t, 1);
         constant_values = 1;
         nelems = hashmap_size(t);
 
@@ -3893,7 +3893,7 @@ dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
 
         *constant = 0;
 
-        hashmap_iterator_init(&wstack, t);
+        hashmap_iterator_init(&wstack, t, 1);
 
         while ((kv=hashmap_iterator_prev(&wstack)) != NULL) {
             /* push key */
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 0e24f2e1b1..00ed968295 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -445,7 +445,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
 	mp->size = n;
 	mp->keys = keys;
 
-	hashmap_iterator_init(&wstack, res);
+	hashmap_iterator_init(&wstack, res, 0);
 
 	while ((kv=hashmap_iterator_next(&wstack)) != NULL) {
 	    *ks++ = CAR(kv);
@@ -1697,7 +1697,7 @@ static Eterm hashmap_to_list(Process *p, Eterm node) {
     Eterm res = NIL;
 
     hp  = HAlloc(p, hashmap_size(node) * (2 + 3));
-    hashmap_iterator_init(&stack, node);
+    hashmap_iterator_init(&stack, node, 0);
     while ((kv=hashmap_iterator_next(&stack)) != NULL) {
 	Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv));
 	hp += 3;
@@ -1708,14 +1708,30 @@ static Eterm hashmap_to_list(Process *p, Eterm node) {
     return res;
 }
 
-void hashmap_iterator_init(ErtsWStack* s, Eterm node) {
-    WSTACK_PUSH((*s), (UWord)THE_NON_VALUE);  /* end marker */
-    WSTACK_PUSH((*s), (UWord)node);
+void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) {
+    Eterm hdr = *hashmap_val(node);
+    Uint sz;
+
+    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+    case HAMT_SUBTAG_HEAD_ARRAY:
+	sz = 16;
+	break;
+    case HAMT_SUBTAG_HEAD_BITMAP:
+	sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+	break;
+    default:
+	erl_exit(1, "bad header");
+    }
+
+    WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE,  /* end marker */
+		 (UWord)(!reverse ? 0 : sz+1),
+		 (UWord)node);
 }
 
 Eterm* hashmap_iterator_next(ErtsWStack* s) {
     Eterm node, *ptr, hdr;
     Uint32 sz;
+    Uint idx;
 
     for (;;) {
         ASSERT(!WSTACK_ISEMPTY((*s)));
@@ -1723,44 +1739,50 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
         if (is_non_value(node)) {
             return NULL;
         }
-        switch (primary_tag(node)) {
-        case TAG_PRIMARY_LIST:
-            return list_val(node);
-
-        case TAG_PRIMARY_BOXED:
-            ptr = boxed_val(node);
-            hdr = *ptr;
-            ASSERT(is_header(hdr));
-            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-            case HAMT_SUBTAG_HEAD_ARRAY:
-                ptr++;
-            case HAMT_SUBTAG_NODE_ARRAY:
-                ptr++;
-                sz = 16;
-                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
-                break;
-            case HAMT_SUBTAG_HEAD_BITMAP:
-                ptr++;
-            case HAMT_SUBTAG_NODE_BITMAP:
-                sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-                ASSERT(sz < 17);
-                ptr++;
-                while(sz--) { WSTACK_PUSH((*s), (UWord)ptr[sz]); }
-                break;
-            default:
-                erl_exit(1, "bad header");
-            }
-            break;
+	idx = (Uint) WSTACK_POP((*s));
+        for (;;) {
+	    ASSERT(is_boxed(node));
+	    ptr = boxed_val(node);
+	    hdr = *ptr;
+	    ASSERT(is_header(hdr));
+	    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ptr++;
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		sz = 16;
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		ptr++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+		ASSERT(sz < 17);
+		break;
+	    default:
+		erl_exit(1, "bad header");
+	    }
 
-        default:
-            erl_exit(1, "bad hamt node");
-	}
+	    idx++;
+
+	    if (idx <= sz) {
+		WSTACK_PUSH2((*s), (UWord)idx, (UWord)node);
+
+		if (is_list(ptr[idx])) {
+		    return list_val(ptr[idx]);
+		}
+		ASSERT(is_boxed(ptr[idx]));
+		node = ptr[idx];
+		idx = 0;
+	    }
+	    else
+		break; /* and pop parent node */
+        }
     }
 }
 
 Eterm* hashmap_iterator_prev(ErtsWStack* s) {
     Eterm node, *ptr, hdr;
-    Uint32 sz,i;
+    Uint32 sz;
+    Uint idx;
 
     for (;;) {
         ASSERT(!WSTACK_ISEMPTY((*s)));
@@ -1768,41 +1790,49 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
         if (is_non_value(node)) {
             return NULL;
         }
-        switch (primary_tag(node)) {
-        case TAG_PRIMARY_LIST:
-            return list_val(node);
-
-        case TAG_PRIMARY_BOXED:
-            ptr = boxed_val(node);
-            hdr = *ptr;
-            ASSERT(is_header(hdr));
-            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-            case HAMT_SUBTAG_HEAD_ARRAY:
-                ptr++;
-            case HAMT_SUBTAG_NODE_ARRAY:
-                ptr++;
-                i = 0;
-                while(i < 16) { WSTACK_PUSH((*s), (UWord)ptr[i++]); }
-                break;
-            case HAMT_SUBTAG_HEAD_BITMAP:
-                ptr++;
-            case HAMT_SUBTAG_NODE_BITMAP:
-                sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-                ASSERT(sz < 17);
-		i = 0;
-                ptr++;
-                while(i < sz) { WSTACK_PUSH((*s), (UWord)ptr[i++]); }
-                break;
-            default:
-                erl_exit(1, "bad header");
-            }
-            break;
+	idx = (Uint) WSTACK_POP((*s));
+        for (;;) {
+	    ASSERT(is_boxed(node));
+	    ptr = boxed_val(node);
+	    hdr = *ptr;
+	    ASSERT(is_header(hdr));
+	    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ptr++;
+	    case HAMT_SUBTAG_NODE_ARRAY:
+		sz = 16;
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		ptr++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+		ASSERT(sz < 17);
+		break;
+	    default:
+		erl_exit(1, "bad header");
+	    }
 
-        default:
-            erl_exit(1, "bad hamt node");
-	}
+            if (idx > sz)
+		idx = sz;
+	    else
+		idx--;
+
+	    if (idx >= 1) {
+		WSTACK_PUSH2((*s), (UWord)idx, (UWord)node);
+
+		if (is_list(ptr[idx])) {
+		    return list_val(ptr[idx]);
+		}
+		ASSERT(is_boxed(ptr[idx]));
+		node = ptr[idx];
+		idx = 17;
+	    }
+	    else
+		break; /* and pop parent node */
+        }
     }
 }
+
 const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     Eterm *ptr, hdr;
     Eterm th[2];
@@ -2135,7 +2165,7 @@ static Eterm hashmap_keys(Process* p, Eterm node) {
 
     root = (hashmap_head_t*) boxed_val(node);
     hp  = HAlloc(p, root->size * 2);
-    hashmap_iterator_init(&stack, node);
+    hashmap_iterator_init(&stack, node, 0);
     while ((kv=hashmap_iterator_next(&stack)) != NULL) {
 	res = CONS(hp, CAR(kv), res);
 	hp += 2;
@@ -2152,7 +2182,7 @@ static Eterm hashmap_values(Process* p, Eterm node) {
 
     root = (hashmap_head_t*) boxed_val(node);
     hp  = HAlloc(p, root->size * 2);
-    hashmap_iterator_init(&stack, node);
+    hashmap_iterator_init(&stack, node, 0);
     while ((kv=hashmap_iterator_next(&stack)) != NULL) {
 	res = CONS(hp, CDR(kv), res);
 	hp += 2;
@@ -2276,7 +2306,7 @@ unroll:
 	mp->size = n;
 	mp->keys = keys;
 
-	hashmap_iterator_init(&wstack, map);
+	hashmap_iterator_init(&wstack, map, 0);
 
 	while ((kv=hashmap_iterator_next(&wstack)) != NULL) {
 	    if (EQ(CAR(kv),key))
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 884bf69a34..c6d8b1966c 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -100,7 +100,7 @@ Eterm  erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 
 int    erts_validate_and_sort_flatmap(flatmap_t* map);
 Uint   hashmap_over_estimated_heap_size(Uint n);
-void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node);
+void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 Eterm* hashmap_iterator_prev(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index cc97e2f3d1..d098ee4c72 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3181,8 +3181,8 @@ tailrecur_ne:
                 */
 
                     sp = PSTACK_PUSH(hmap_stack);
-                    hashmap_iterator_init(&stack, a);
-                    hashmap_iterator_init(&b_stack, b);
+                    hashmap_iterator_init(&stack, a, 0);
+                    hashmap_iterator_init(&b_stack, b, 0);
                     sp->ap = hashmap_iterator_next(&stack);
                     sp->bp = hashmap_iterator_next(&b_stack);
                     sp->cmp_res = 0;
-- 
cgit v1.2.3


From 2aeb8cfd42be8ab1b7eee4a7f144122223dd7261 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Mar 2015 21:09:33 +0100
Subject: erts: Fix nif API for hashmaps

---
 erts/emulator/beam/erl_nif.c | 162 +++++++++++++++++++++++++++++++++----------
 erts/emulator/beam/erl_nif.h |  14 +++-
 erts/emulator/beam/global.h  |  16 ++++-
 3 files changed, 150 insertions(+), 42 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index fd793fd7e4..e28365cb1b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1905,7 +1905,7 @@ enif_is_on_dirty_scheduler(ErlNifEnv* env)
 
 int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term)
 {
-    return is_flatmap(term);
+    return is_map(term);
 }
 
 int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)
@@ -1916,6 +1916,10 @@ int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)
 	*size = flatmap_get_size(mp);
 	return 1;
     }
+    else if (is_hashmap(term)) {
+        *size = hashmap_size(term);
+        return 1;
+    }
     return 0;
 }
 
@@ -1941,7 +1945,7 @@ int enif_make_map_put(ErlNifEnv* env,
 		      Eterm value,
 		      Eterm *map_out)
 {
-    if (is_not_flatmap(map_in)) {
+    if (!is_map(map_in)) {
 	return 0;
     }
     flush_env(env);
@@ -1956,7 +1960,7 @@ int enif_get_map_value(ErlNifEnv* env,
 		       Eterm *value)
 {
     const Eterm *ret;
-    if (is_not_flatmap(map)) {
+    if (!is_map(map)) {
 	return 0;
     }
     ret = erts_maps_get(key, map);
@@ -1974,7 +1978,7 @@ int enif_make_map_update(ErlNifEnv* env,
 			 Eterm *map_out)
 {
     int res;
-    if (is_not_flatmap(map_in)) {
+    if (!is_map(map_in)) {
 	return 0;
     }
 
@@ -1990,7 +1994,7 @@ int enif_make_map_remove(ErlNifEnv* env,
 			 Eterm *map_out)
 {
     int res;
-    if (is_not_flatmap(map_in)) {
+    if (!is_map(map_in)) {
 	return 0;
     }
     flush_env(env);
@@ -2019,14 +2023,37 @@ int enif_map_iterator_create(ErlNifEnv *env,
 	 */
 
 	iter->map     = map;
-	iter->ks      = ((Eterm *)flatmap_get_keys(mp)) + offset;
-	iter->vs      = ((Eterm *)flatmap_get_values(mp)) + offset;
-	iter->t_limit = flatmap_get_size(mp) + 1;
+	iter->u.flat.ks = ((Eterm *)flatmap_get_keys(mp)) + offset;
+	iter->u.flat.vs = ((Eterm *)flatmap_get_values(mp)) + offset;
+	iter->size    = flatmap_get_size(mp);
 	iter->idx     = offset + 1;
 
 	return 1;
     }
-
+    else if (is_hashmap(map)) {
+        iter->map = map;
+        iter->size = hashmap_size(map);
+        iter->u.hash.wstack = erts_alloc(ERTS_ALC_T_NIF, sizeof(ErtsDynamicWStack));
+        WSTACK_INIT(iter->u.hash.wstack, ERTS_ALC_T_NIF);
+
+        switch (entry) {
+	    case ERL_NIF_MAP_ITERATOR_HEAD:
+                iter->idx = 1;
+                hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 0);
+                iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws);
+                break;
+	    case ERL_NIF_MAP_ITERATOR_TAIL:
+                iter->idx = hashmap_size(map);
+                hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 1);
+                iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws);
+                break;
+	    default:
+                goto error;
+	}
+        ASSERT(!!iter->u.hash.kv == (iter->idx >= 1 &&
+                                     iter->idx <= iter->size));
+        return 1;
+    }
 error:
 #ifdef DEBUG
     iter->map = THE_NON_VALUE;
@@ -2036,48 +2063,97 @@ error:
 
 void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    /* not used */
+    if (is_hashmap(iter->map)) {
+        WSTACK_DESTROY(iter->u.hash.wstack->ws);
+        erts_free(ERTS_ALC_T_NIF, iter->u.hash.wstack);
+    }
+    else
+        ASSERT(is_flatmap(iter->map));
+
 #ifdef DEBUG
     iter->map = THE_NON_VALUE;
 #endif
-
 }
 
 int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_flatmap(iter->map));
-    ASSERT(iter->idx >= 0 && (iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1));
-    return (iter->t_limit == 1 || iter->idx == iter->t_limit);
+    ASSERT(iter);
+    if (is_flatmap(iter->map)) {
+        ASSERT(iter->idx >= 0);
+        ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1);
+        return (iter->size == 0 || iter->idx > iter->size);
+    }
+    else {
+        ASSERT(is_hashmap(iter->map));
+        return iter->idx > iter->size;
+    }
 }
 
 int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_flatmap(iter->map));
-    ASSERT(iter->idx >= 0 && (iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1));
-    return (iter->t_limit == 1 || iter->idx == 0);
+    ASSERT(iter);
+    if (is_flatmap(iter->map)) {
+        ASSERT(iter->idx >= 0);
+        ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1);
+        return (iter->size == 0 || iter->idx == 0);
+    }
+    else {
+        ASSERT(is_hashmap(iter->map));
+        return iter->idx == 0;
+    }
 }
 
 
 int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_flatmap(iter->map));
-    if (iter->idx < iter->t_limit) {
-	iter->idx++;
-	iter->ks++;
-	iter->vs++;
+    ASSERT(iter);
+    if (is_flatmap(iter->map)) {
+        if (iter->idx <= iter->size) {
+            iter->idx++;
+            iter->u.flat.ks++;
+            iter->u.flat.vs++;
+        }
+        return (iter->idx <= iter->size);
+    }
+    else {
+        ASSERT(is_hashmap(iter->map));
+
+        if (iter->idx <= hashmap_size(iter->map)) {
+            if (iter->idx < 1) {
+                hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 0);
+            }
+            iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws);
+            iter->idx++;
+            ASSERT(!!iter->u.hash.kv == (iter->idx <= iter->size));
+        }
+        return iter->idx <= iter->size;
     }
-    return (iter->idx != iter->t_limit);
 }
 
 int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)
 {
-    ASSERT(iter && is_flatmap(iter->map));
-    if (iter->idx > 0) {
-	iter->idx--;
-	iter->ks--;
-	iter->vs--;
+    ASSERT(iter);
+    if (is_flatmap(iter->map)) {
+        if (iter->idx > 0) {
+            iter->idx--;
+            iter->u.flat.ks--;
+            iter->u.flat.vs--;
+        }
+        return iter->idx > 0;
+    }
+    else {
+        ASSERT(is_hashmap(iter->map));
+
+        if (iter->idx > 0) {
+            if (iter->idx > iter->size) {
+                hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 1);
+            }
+            iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws);
+            iter->idx--;
+            ASSERT(!!iter->u.hash.kv == (iter->idx > 0));
+        }
+        return iter->idx > 0;
     }
-    return (iter->idx > 0);
 }
 
 int enif_map_iterator_get_pair(ErlNifEnv *env,
@@ -2085,15 +2161,25 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
 			       Eterm *key,
 			       Eterm *value)
 {
-    ASSERT(iter && is_flatmap(iter->map));
-    if (iter->idx > 0 && iter->idx < iter->t_limit) {
-	ASSERT(iter->ks >= flatmap_get_keys(flatmap_val(iter->map)) &&
-	       iter->ks  < (flatmap_get_keys(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map))));
-	ASSERT(iter->vs >= flatmap_get_values(flatmap_val(iter->map)) &&
-	       iter->vs  < (flatmap_get_values(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map))));
-	*key   = *(iter->ks);
-	*value = *(iter->vs);
-	return 1;
+    ASSERT(iter);
+    if (is_flatmap(iter->map)) {
+        if (iter->idx > 0 && iter->idx <= iter->size) {
+            ASSERT(iter->u.flat.ks >= flatmap_get_keys(flatmap_val(iter->map)) &&
+                   iter->u.flat.ks  < (flatmap_get_keys(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map))));
+            ASSERT(iter->u.flat.vs >= flatmap_get_values(flatmap_val(iter->map)) &&
+                   iter->u.flat.vs  < (flatmap_get_values(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map))));
+            *key   = *(iter->u.flat.ks);
+            *value = *(iter->u.flat.vs);
+            return 1;
+        }
+    }
+    else {
+        ASSERT(is_hashmap(iter->map));
+        if (iter->idx > 0 && iter->idx <= iter->size) {
+            *key   = CAR(iter->u.hash.kv);
+            *value = CDR(iter->u.hash.kv);
+            return 1;
+        }
     }
     return 0;
 }
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 849024453c..9b2b90c82d 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -201,10 +201,18 @@ typedef enum
 typedef struct /* All fields all internal and may change */
 {
     ERL_NIF_TERM map;
-    ERL_NIF_UINT t_limit;
+    ERL_NIF_UINT size;
     ERL_NIF_UINT idx;
-    ERL_NIF_TERM *ks;
-    ERL_NIF_TERM *vs;
+    union {
+        struct {
+            ERL_NIF_TERM *ks;
+            ERL_NIF_TERM *vs;
+        }flat;
+        struct {
+            struct ErtsDynamicWStack_* wstack;
+            ERL_NIF_TERM* kv;
+        }hash;
+    }u;
     void* __spare__[2]; /* for future additions to be ABI compatible (same struct size) */
 } ErlNifMapIterator;
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 09bcf71a28..ef7a183d08 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -542,6 +542,20 @@ void erl_grow_wstack(ErtsWStack*, Uint need);
     }
 #define DECLARE_WSTACK WSTACK_DECLARE
 
+typedef struct ErtsDynamicWStack_ {
+    UWord default_stack[DEF_WSTACK_SIZE];
+    ErtsWStack ws;
+}ErtsDynamicWStack;
+
+#define WSTACK_INIT(dwsp, ALC_TYPE)                               \
+do {	 	                                                  \
+    (dwsp)->ws.wstart   = (dwsp)->default_stack;                  \
+    (dwsp)->ws.wsp      = (dwsp)->default_stack;                  \
+    (dwsp)->ws.wend     = (dwsp)->default_stack + DEF_WSTACK_SIZE;\
+    (dwsp)->ws.wdefault = (dwsp)->default_stack;                  \
+    (dwsp)->ws.alloc_type = ALC_TYPE;                             \
+} while (0)
+
 #define WSTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
     if (s.wstart != WSTK_DEF_STACK(s)) {				\
@@ -553,7 +567,7 @@ do {									\
 
 #define WSTACK_DESTROY(s)				\
 do {							\
-    if (s.wstart != WSTK_DEF_STACK(s)) {		\
+    if (s.wstart != s.wdefault) {		        \
 	erts_free(s.alloc_type, s.wstart); 		\
     }							\
 } while(0)
-- 
cgit v1.2.3


From 1b13e90161bdb1409b04a273a6b71f02fbb9d5f1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Mar 2015 15:19:35 +0100
Subject: erts: Cleanup comments for make_internal_hash

---
 erts/emulator/beam/utils.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d098ee4c72..23ae885b6d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1570,14 +1570,22 @@ make_hash2(Eterm term)
 }
 
 /* Term hash function for internal use.
- * Is not "portable" in any way betweem different VM instances.
+ *
+ * Limitation #1: Is not "portable" in any way between different VM instances.
+ *
+ * Limitation #2: The hash value is only valid as long as the term exists
+ * somewhere in the VM. Why? Because external pids, ports and refs are hashed
+ * by mixing the node *pointer* value. If a node disappears and later reappears
+ * with a new ErlNode struct, externals from that node will hash different than
+ * before.
  *
  * One IMPORTANT property must hold (for hamt).
  * EVERY BIT of the term that is significant for equality (see EQ)
- * must be used as input for the hash. Two different terms must always have a
- * chance of hashing different when salted: h([Salt|A]) vs h([Salt|B]).
+ * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a
+ * chance of hashing different when salted: hash([Salt|A]) vs hash([Salt|B]).
  *
  * This is why we can not use cached hash values for atoms for example.
+ *
  */
 
 #define CONST_HASH(AConst)                              \
@@ -1854,7 +1862,7 @@ make_internal_hash(Eterm term)
                 ExternalThing* thing = external_thing_ptr(term);
 
                 ASSERT(external_thing_ref_no_of_numbers(thing) == 3);
-                /*SVERK Is it really ok to hash node POINTER? */
+                /* See limitation #2 */
             #ifdef ARCH_64
                 POINTER_HASH(thing->node, HCONST_7);
                 UINT32_HASH(external_thing_ref_numbers(thing)[0], HCONST_7);
@@ -1868,7 +1876,7 @@ make_internal_hash(Eterm term)
             }
             case EXTERNAL_PID_SUBTAG: {
                 ExternalThing* thing = external_thing_ptr(term);
-                /*SVERK Is it really ok to hash node POINTER? */
+                /* See limitation #2 */
             #ifdef ARCH_64
                 POINTER_HASH(thing->node, HCONST_5);
                 UINT32_HASH(thing->data.ui[0], HCONST_5);
@@ -1879,7 +1887,7 @@ make_internal_hash(Eterm term)
             }
 	    case EXTERNAL_PORT_SUBTAG: {
                 ExternalThing* thing = external_thing_ptr(term);
-                /*SVERK Is it really ok to hash node POINTER? */
+                /* See limitation #2 */
             #ifdef ARCH_64
                 POINTER_HASH(thing->node, HCONST_6);
                 UINT32_HASH(thing->data.ui[0], HCONST_6);
-- 
cgit v1.2.3


From a09f11fb0db7f40015f85146691286e1ac735733 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 12 Mar 2015 19:20:36 +0100
Subject: erts: Set Maps small limit to 32 items

Use small limit 3 in debug case
---
 erts/emulator/beam/erl_map.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index c6d8b1966c..ff5e6edd1e 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -80,7 +80,11 @@ typedef struct flatmap_s {
 #define flatmap_get_keys(x)        (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1)
 #define flatmap_get_size(x)        (((flatmap_t*)(x))->size)
 
-#define MAP_SMALL_MAP_LIMIT    (2) /*SVERK (32) */
+#ifdef DEBUG
+#define MAP_SMALL_MAP_LIMIT    (3)
+#else
+#define MAP_SMALL_MAP_LIMIT    (32)
+#endif
 #define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
 #define MAP_HEADER_SIZE        (sizeof(flatmap_t) / sizeof(Eterm))
 
-- 
cgit v1.2.3


From 2dae04b0c73f5fc855d4cfb2e6231bf65e51ebce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 12 Mar 2015 18:39:54 +0100
Subject: Fix beam_load assert

---
 erts/emulator/beam/beam_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index fce710f723..02689e5b19 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3727,7 +3727,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     op->a[j+size] = Fail;
 
 #ifdef DEBUG
-    for (i = 0; i < size; i++) {
+    for (i = 0; i < size - 1; i++) {
 	ASSERT(op->a[i+3].val <= op->a[i+4].val);
     }
 #endif
-- 
cgit v1.2.3


From 2ae925af364d1d1a0e1f4a2693a7030e58a26018 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Mar 2015 20:34:18 +0100
Subject: erts: Fix typo in make_hash2 for 32-bit arch

---
 erts/emulator/beam/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 23ae885b6d..933a8ffa0b 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1188,7 +1188,7 @@ make_hash2(Eterm term)
 #ifdef ARCH_64
 #  define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst)
 #else
-#  define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, Const)
+#  define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst)
 #endif
 
     /* Optimization. Simple cases before declaration of estack. */
-- 
cgit v1.2.3


From b398819d59863db76fc2ce2f0c1584254e38f3bb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Mar 2015 21:03:10 +0100
Subject: erts: Fix windows bug in hashmap_info

undefined symbol 'MAX'
---
 erts/emulator/beam/erl_map.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 00ed968295..b7f07fa6c4 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2670,7 +2670,8 @@ static Eterm hashmap_info(Process *p, Eterm node) {
     do {
 	node = ESTACK_POP(stack);
 	clvl = ESTACK_POP(stack);
-	lvl  = MAX(lvl,clvl);
+	if (lvl < clvl)
+            lvl = clvl;
 	switch(primary_tag(node)) {
 	    case TAG_PRIMARY_LIST:
 		nleaf++;
-- 
cgit v1.2.3


From 59f1846249fbbb76e786d1e73e9be5c72a41fefb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 13 Mar 2015 10:09:33 +0100
Subject: erts: Restrict GCC intrinsics by compiler version

Intrinsics __builtin_clz and __builtin_popcount are only valid on
GCC version 3.4 and above.
---
 erts/emulator/beam/erl_map.c | 3 ++-
 erts/emulator/beam/erl_map.h | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index b7f07fa6c4..964c09e906 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2768,7 +2768,7 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
 
 /* implementation of builtin emulations */
 
-#if !defined(__GNUC__)
+#if !ERTS_AT_LEAST_GCC_VSN__(3, 4, 0)
 /* Count leading zeros emulation */
 Uint32 hashmap_clz(Uint32 x) {
     Uint32 y;
@@ -2780,6 +2780,7 @@ Uint32 hashmap_clz(Uint32 x) {
     y = x >> 1;  if (y != 0) return n - 2;
     return n - x;
 }
+
 const Uint32 SK5 = 0x55555555, SK3 = 0x33333333;
 const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF;
 
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index ff5e6edd1e..d075d87c83 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -24,7 +24,7 @@
 #include "sys.h"
 
 /* instrinsic wrappers */
-#if defined(__GNUC__)
+#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0)
 #define hashmap_clz(x)       ((Uint32) __builtin_clz((unsigned int)(x)))
 #define hashmap_bitcount(x)  ((Uint32) __builtin_popcount((unsigned int) (x)))
 #else
-- 
cgit v1.2.3


From dee91137e9b8c1f32b788ce5b5b34fcb2076d9ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 13 Mar 2015 11:17:37 +0100
Subject: erts: Fix typo in copy_struct for halfword emulator

---
 erts/emulator/beam/copy.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 83214aca19..027b85b079 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -511,7 +511,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
 			i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr));
 			while (i--)  { *htop++ = *objp++; }
-			*argp = make_hashmap_rel(tp, dstbase);
+			*argp = make_hashmap_rel(tp, dst_base);
 			break;
 		    default:
 			erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
-- 
cgit v1.2.3


From 17f4ccfcd91b2ba074f5c6fe224adacfe5952df4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 13 Mar 2015 11:19:31 +0100
Subject: erts: Remove unused variable in crashdump creation

---
 erts/emulator/beam/break.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 4ede2c9d7d..e2fa572546 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -661,7 +661,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 {
 #ifdef ERTS_SMP
     ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
-    int bc;
 #endif
     int fd;
     size_t envsz;
@@ -681,7 +680,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     /* Order all managed threads to block, this has to be done
        first to guarantee that this is the only thread to generate
        crash dump. */
-    bc = erts_thr_progress_fatal_error_block(&tpd_buf);
+    erts_thr_progress_fatal_error_block(&tpd_buf);
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     /*
-- 
cgit v1.2.3


From f0cfce64b6a061ffeafeda254734f0b1f2f452fd Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Sun, 1 Mar 2015 12:33:20 -0500
Subject: Ensure NIF term creation disallows illegal values

Add a check to enif_make_double to see if its double argument is
infinity or NaN, returning a badarg exception if it is. Change the
erl_nif documentation to specify that enif_make_double returns a
badarg exception if its double argument is either infinity or NaN. Add
tests to nif_SUITE for this change.

Add checks to the enif_make* functions for atoms to prevent the
creation of atoms whose name lengths are greater than the allowed
maximum atom length. The enif_make_atom and enif_make_atom_len
functions now return a badarg exception if the input string is too
long. The enif_make_existing_atom and enif_make_existing_atom_len
functions return false if the input string is too long. Change the
erl_nif documentation to reflect the changes to these functions. Add
tests to nif_SUITE for these changes.

Add a field to ErlNifEnv to track that a NIF has raised an exception
via enif_make_badarg. If a NIF calls enif_make_badarg but then ignores
its return value and instead tries to return a non-exception term as
its return value, the runtime still raises a badarg. This is needed to
prevent enif_make_badarg values resulting from calls to
enif_make_double, enif_make_atom, or enif_make_atom_len from being
erroneously stored within other terms and returned from a NIF. Calling
enif_make_badarg but not returning its return value has been
documented as being illegal ever since enif_make_badarg was added, but
the runtime has not enforced it until now. Add tests for regular and
dirty NIFs to ensure that calls to enif_make_badarg result in badarg
exceptions even if a NIF fails to return the result of
enif_make_badarg as its return value. Add documentation to
enif_make_badarg to specify that calling it raises a badarg even if a
NIF ignores its return value.
---
 erts/emulator/beam/beam_emu.c |  2 ++
 erts/emulator/beam/erl_nif.c  | 16 ++++++++++++----
 erts/emulator/beam/global.h   |  1 +
 3 files changed, 15 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8bfb7d2ad2..d352528059 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3523,6 +3523,8 @@ get_map_elements_fail:
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
 		reg[0] = r(0);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+		if (env.exception_thrown)
+		    nif_bif_result = THE_NON_VALUE;
 		erts_post_nif(&env);
 	    }
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index adc3520ebb..ec82ef251e 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -127,6 +127,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
     env->heap_frag = NULL;
     env->fpe_was_unmasked = erts_block_fpe();
     env->tmp_obj_list = NULL;
+    env->exception_thrown = 0;
 }
 
 static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif)
@@ -742,6 +743,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
 
 Eterm enif_make_badarg(ErlNifEnv* env)
 {
+    env->exception_thrown = 1;
     BIF_ERROR(env->proc, BADARG);
 }
 
@@ -964,7 +966,10 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)
 
 ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 {
-    Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
+    Eterm* hp;
+    if (!isfinite(d))
+        return enif_make_badarg(env);
+    hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
     FloatDef f;
     f.fd = d;
     PUT_DOUBLE(f, hp);
@@ -978,6 +983,8 @@ ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name)
 
 ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)
 {
+    if (len > MAX_ATOM_CHARACTERS)
+        return enif_make_badarg(env);
     return erts_atom_put((byte*)name, len, ERTS_ATOM_ENC_LATIN1, 1);
 }
 
@@ -991,6 +998,8 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len,
 				ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)
 {
     ASSERT(encoding == ERL_NIF_LATIN1);
+    if (len > MAX_ATOM_CHARACTERS)
+        return 0;
     return erts_atom_get(name, len, atom, ERTS_ATOM_ENC_LATIN1);
 }
 
@@ -1754,14 +1763,13 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(ep);
     if (ep->fp)
 	fp = NULL;
-    if (is_non_value(result)) {
+    if (is_non_value(result) || env->exception_thrown) {
 	if (proc->freason != TRAP) {
-	    ASSERT(proc->freason == BADARG);
 	    return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv);
 	} else {
 	    if (ep->fp == NULL)
 		restore_nif_mfa(proc, ep, 1);
-	    return result;
+	    return THE_NON_VALUE;
 	}
     }
     else
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 32a2dc43e8..d58aecab9b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -51,6 +51,7 @@ struct enif_environment_t /* ErlNifEnv */
     ErlHeapFragment* heap_frag;
     int fpe_was_unmasked;
     struct enif_tmp_obj_t* tmp_obj_list;
+    int exception_thrown; /* boolean */
 };
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*);
-- 
cgit v1.2.3


From 6f9c07ae2694a2dc1f721fd10af22b8813d15bc1 Mon Sep 17 00:00:00 2001
From: Anthony Ramine <n.oxyde@gmail.com>
Date: Thu, 12 Mar 2015 16:01:33 +0100
Subject: Fix a race condition when calling port_info/1

This variable hold the values returned by erlang:port_info/1 and
shouldn't be static.

Reported-by: Heinz Nikolaus Gies
---
 erts/emulator/beam/io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 012a7d1a4b..a950998c13 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4484,7 +4484,7 @@ make_port_info_term(Eterm **hpp_start,
 	int len;
 	int start;
 	static Eterm item[] = ERTS_PORT_INFO_1_ITEMS;
-	static Eterm value[sizeof(item)/sizeof(item[0])];
+        Eterm value[sizeof(item)/sizeof(item[0])];
 
 	start = 0;
 	len = sizeof(item)/sizeof(item[0]);
-- 
cgit v1.2.3


From 09db67d581e1e5858909aafec7d29226020cd227 Mon Sep 17 00:00:00 2001
From: Dan Gudmundsson <dgud@erlang.org>
Date: Wed, 18 Mar 2015 15:47:47 +0100
Subject: Fix thread name from driver api

if opts was null the name was not set
---
 erts/emulator/beam/erl_drv_thread.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 31b05d22af..240faa823d 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -604,10 +604,12 @@ erl_drv_thread_create(char *name,
     ethr_thr_opts ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
     ethr_thr_opts *use_opts;
 
-    if (!opts)
+    if (!opts && !name)
 	use_opts = NULL;
     else {
-	ethr_opts.suggested_stack_size = opts->suggested_stack_size;
+	if(opts)
+	    ethr_opts.suggested_stack_size = opts->suggested_stack_size;
+
         ethr_opts.name = name;
 	use_opts = &ethr_opts;
     }
-- 
cgit v1.2.3


From 12fc63bcaf68b4a9e89ce91e1235aafb8bcdaee5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Mar 2015 18:00:33 +0100
Subject: erts: Fix map bug in dec_term for 32-bit debug VM

Adding ERTS_SWORD_MAX to a pointer does not work
as a way to disable a bound check.

Remove the hp_end from ErtsHeapFactory as it isn't really used anyway.
---
 erts/emulator/beam/erl_message.c | 1 -
 erts/emulator/beam/erl_message.h | 2 --
 erts/emulator/beam/external.c    | 1 -
 erts/emulator/beam/io.c          | 1 -
 4 files changed, 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index e4cbd8477d..43a03c793e 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1154,7 +1154,6 @@ Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
     } else {
         res = factory->hp;
         factory->hp += need;
-        ASSERT(factory->hp <= factory->hp_end);
     }
     return res;
 }
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index ece75a5ee4..6b8c3cebc7 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -71,8 +71,6 @@ struct erl_heap_fragment {
 typedef struct {
     Process* p;
     Eterm* hp;
-    Eterm* hp_end;
-    /* more to come... */
 } ErtsHeapFactory;
 
 Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 65b4ae5412..9a5ef56c47 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3905,7 +3905,6 @@ dec_term_atom_common:
 	factory.p = NULL;
         factory.hp = hp;
 	/* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
-        factory.hp_end = hp + (ERTS_SWORD_MAX / sizeof(Eterm));
 
         do {
 	    *hamt->objp = erts_hashmap_from_array(&factory,
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b64854aac9..62254ca34d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5609,7 +5609,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
                 factory.p = NULL;
                 factory.hp = hp;
                 /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
-                factory.hp_end = hp + (ERTS_SWORD_MAX / sizeof(Eterm));
 
                 mess = erts_hashmap_from_array(&factory, leafs, size, 1);
 
-- 
cgit v1.2.3


From 401e2544564526039ac87b3ffe79eb8af30b37b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 16 Mar 2015 11:06:44 +0100
Subject: erts: Ensure halfword has correct temp-heap for maps

---
 erts/emulator/beam/erl_map.c | 139 +++++++++++++++++++++++--------------------
 1 file changed, 74 insertions(+), 65 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 964c09e906..4183b532a9 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1835,72 +1835,77 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
 
 const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) {
     Eterm *ptr, hdr;
-    Eterm th[2];
     Uint ix,slot, lvl = 0;
     Uint32 hval,bp;
+    DeclareTmpHeapNoproc(th,2);
+    UseTmpHeapNoproc(2);
 
     for (;;) {
-	switch(primary_tag(node)) {
-	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
-		ptr = list_val(node);
-		if (EQ(CAR(ptr), key)) {
-		    return &(CDR(ptr));
-		}
-		return NULL;
-	    case TAG_PRIMARY_BOXED:
-		ptr = boxed_val(node);
-		hdr = *ptr;
-		ASSERT(is_header(hdr));
-
-		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ix   = hashmap_index(hx);
-			hx   = hashmap_shift_hash(th,hx,lvl,key);
-			node = ptr[ix+1];
-			break;
-		    case HAMT_SUBTAG_HEAD_ARRAY:
-			ix   = hashmap_index(hx);
-			hx   = hashmap_shift_hash(th,hx,lvl,key);
-			node = ptr[ix+2];
-			break;
-		    case HAMT_SUBTAG_NODE_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+1];
-			    break;
-			}
-			/* not occupied */
-			return NULL;
-		    case HAMT_SUBTAG_HEAD_BITMAP:
-			hval = MAP_HEADER_VAL(hdr);
-			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+2];
-			    break;
-			}
-			/* not occupied */
-			return NULL;
-		    default:
-			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
-			break;
-		}
-		break;
-	    default:
-		erl_exit(1, "bad primary tag %p\r\n", node);
-		break;
-	}
+        switch(primary_tag(node)) {
+            case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
+                ptr = list_val(node);
+                UnUseTmpHeapNoproc(2);
+                if (EQ(CAR(ptr), key)) {
+                    return &(CDR(ptr));
+                }
+                return NULL;
+            case TAG_PRIMARY_BOXED:
+                ptr = boxed_val(node);
+                hdr = *ptr;
+                ASSERT(is_header(hdr));
+
+                switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+                    case HAMT_SUBTAG_NODE_ARRAY:
+                        ix   = hashmap_index(hx);
+                        hx   = hashmap_shift_hash(th,hx,lvl,key);
+                        node = ptr[ix+1];
+                        break;
+                    case HAMT_SUBTAG_HEAD_ARRAY:
+                        ix   = hashmap_index(hx);
+                        hx   = hashmap_shift_hash(th,hx,lvl,key);
+                        node = ptr[ix+2];
+                        break;
+                    case HAMT_SUBTAG_NODE_BITMAP:
+                        hval = MAP_HEADER_VAL(hdr);
+                        ix   = hashmap_index(hx);
+                        bp   = 1 << ix;
+                        slot = hashmap_bitcount(hval & (bp - 1));
+
+                        /* occupied */
+                        if (bp & hval) {
+                            hx    = hashmap_shift_hash(th,hx,lvl,key);
+                            node  = ptr[slot+1];
+                            break;
+                        }
+                        /* not occupied */
+                        UnUseTmpHeapNoproc(2);
+                        return NULL;
+                    case HAMT_SUBTAG_HEAD_BITMAP:
+                        hval = MAP_HEADER_VAL(hdr);
+                        ix   = hashmap_index(hx);
+                        bp   = 1 << ix;
+                        slot = hashmap_bitcount(hval & (bp - 1));
+
+                        /* occupied */
+                        if (bp & hval) {
+                            hx    = hashmap_shift_hash(th,hx,lvl,key);
+                            node  = ptr[slot+2];
+                            break;
+                        }
+                        /* not occupied */
+                        UnUseTmpHeapNoproc(2);
+                        return NULL;
+                    default:
+                        erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+                        break;
+                }
+                break;
+            default:
+                erl_exit(1, "bad primary tag %p\r\n", node);
+                break;
+        }
     }
+    UnUseTmpHeapNoproc(2);
     return NULL;
 }
 
@@ -2049,11 +2054,14 @@ unroll:
 
 Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 			     Uint *update_size, ErtsEStack *sp) {
-    Eterm node, fake, *ptr, hdr;
+    Eterm node, *ptr, hdr;
     Eterm res;
     Eterm *nhp = NULL;
     Uint32 ix, cix, bp, hval;
     Uint slot, n;
+    /* Needed for halfword */
+    DeclareTmpHeapNoproc(fake,1);
+    UseTmpHeapNoproc(1);
 
     res = CONS(hp, key, value); hp += 2;
 
@@ -2077,8 +2085,8 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 		break;
 	    case TAG_PRIMARY_HEADER:
 		/* subnodes, fake it */
-		fake = node;
-		node = make_boxed(&fake);
+		*fake = node;
+		node  = make_boxed(fake);
 	    case TAG_PRIMARY_BOXED:
 		ptr = boxed_val(node);
 		hdr = *ptr;
@@ -2154,7 +2162,8 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 
     } while(!ESTACK_ISEMPTY(*sp));
 
-   return res;
+    UnUseTmpHeapNoproc(1);
+    return res;
 }
 
 static Eterm hashmap_keys(Process* p, Eterm node) {
-- 
cgit v1.2.3


From 3c55267c6176a7183c59fc2ca6c1b278df080373 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 16 Mar 2015 16:08:56 +0100
Subject: erts: Do not treat errors as fatal in erl_printf_term

---
 erts/emulator/beam/erl_printf_term.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 8046f54a0c..ac5b139f8d 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -31,7 +31,7 @@
 do {									\
     int res__ = erts_printf_char((FN), (ARG), (C));			\
     if (res__ < 0)							\
-	abort();							\
+	return res__;							\
     (CNT) += res__;							\
 } while (0)
 
@@ -39,7 +39,7 @@ do {									\
 do {									\
     int res__ = erts_printf_string((FN), (ARG), (STR));			\
     if (res__ < 0)							\
-	abort();							\
+	return res__;							\
     (CNT) += res__;							\
 } while (0)
 
@@ -47,7 +47,7 @@ do {									\
 do {									\
     int res__ = erts_printf_buf((FN), (ARG), (char*)(BUF), (LEN));	\
     if (res__ < 0)							\
-	abort();							\
+	return res__;							\
     (CNT) += res__;							\
 } while (0)
 
@@ -55,7 +55,7 @@ do {									\
 do {									\
     int res__ = erts_printf_pointer((FN), (ARG), (void *) (PTR));	\
     if (res__ < 0)							\
-	abort();							\
+	return res__;							\
     (CNT) += res__;							\
 } while (0)
 
@@ -63,7 +63,7 @@ do {									\
 do {									\
     int res__ = erts_printf_uword((FN), (ARG), (C), (P), (W), (I));	\
     if (res__ < 0)							\
-	abort();							\
+	return res__;							\
     (CNT) += res__;							\
 } while (0)
 
@@ -71,7 +71,7 @@ do {									\
 do {									\
     int res__ = erts_printf_sword((FN), (ARG), (C), (P), (W), (I));	\
     if (res__ < 0)							\
-	abort();							\
+	return res__;							\
     (CNT) += res__;							\
 } while (0)
 
@@ -79,7 +79,7 @@ do {									\
 do {									\
     int res__ = erts_printf_double((FN), (ARG), (C), (P), (W), (I));	\
     if (res__ < 0)							\
-	abort();							\
+	return res__;							\
     (CNT) += res__;							\
 } while (0)
 
-- 
cgit v1.2.3


From feffe2deac13ad61477c7ecf63342815c750bfe5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 17 Mar 2015 15:44:07 +0100
Subject: erts: Ensure maps uses _rel functions in halfword

---
 erts/emulator/beam/erl_map.c | 19 +++++++++++++------
 erts/emulator/beam/erl_map.h | 13 ++++++++++---
 2 files changed, 23 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4183b532a9..bd6da0a6f1 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -165,7 +165,7 @@ erts_maps_get(Eterm key, Eterm map)
 #endif
 {
     Uint32 hx;
-    if (is_flatmap(map)) {
+    if (is_flatmap_rel(map, map_base)) {
 	Eterm *ks, *vs;
 	flatmap_t *mp;
 	Uint n, i;
@@ -189,16 +189,16 @@ erts_maps_get(Eterm key, Eterm map)
 	}
 
 	for (i = 0; i < n; i++) {
-	    if (eq_rel(ks[i], NULL, key, map_base)) {
+	    if (eq_rel(ks[i], map_base, key, NULL)) {
 		return &vs[i];
 	    }
 	}
 	return NULL;
     }
-    ASSERT(is_hashmap(map));
+    ASSERT(is_hashmap_rel(map, map_base));
     hx = hashmap_make_hash(key);
 
-    return erts_hashmap_get(hx, key, map);
+    return erts_hashmap_get_rel(hx, key, map, map_base);
 }
 
 BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
@@ -1833,7 +1833,13 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
     }
 }
 
-const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) {
+const Eterm *
+#if HALFWORD_HEAP
+erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base)
+#else
+erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
+#endif
+{
     Eterm *ptr, hdr;
     Uint ix,slot, lvl = 0;
     Uint32 hval,bp;
@@ -1845,7 +1851,8 @@ const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) {
             case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
                 ptr = list_val(node);
                 UnUseTmpHeapNoproc(2);
-                if (EQ(CAR(ptr), key)) {
+
+                if (eq_rel(CAR(ptr), map_base, key, NULL)) {
                     return &(CDR(ptr));
                 }
                 return NULL;
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index d075d87c83..1333a734a8 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -109,7 +109,6 @@ Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 Eterm* hashmap_iterator_prev(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys);
-const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
 #define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \
     erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE);
@@ -117,16 +116,24 @@ const  Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 Eterm  erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n,
 					 Eterm k, Eterm v);
 
-#if HALFWORD_HEAP
 const Eterm *
+#if HALFWORD_HEAP
 erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base);
 #  define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL)
 #else
-const Eterm *
 erts_maps_get(Eterm key, Eterm map);
 #  define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B)
 #endif
 
+const Eterm *
+#if HALFWORD_HEAP
+erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base);
+#  define erts_hashmap_get(Hx, K, M) erts_hashmap_get_rel(Hx, K, M, NULL)
+#else
+erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
+#  define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M)
+#endif
+
 /* hamt nodes v2.0
  *
  * node :: leaf | array | bitmap
-- 
cgit v1.2.3


From a8599e3fbeb4628268f8761cbb1102d24d552133 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 18 Mar 2015 18:34:24 +0100
Subject: erts: Fix bug in ESTACK and WSTACK

The [ew]default field would get uninitialised when the stack was
saved and later restored.

Detected by valgrind.
---
 erts/emulator/beam/global.h | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ef7a183d08..42daa2c9ef 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -426,6 +426,7 @@ do {\
 	memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\
 	(dst)->sp = (dst)->start + _wsz;\
 	(dst)->end = (dst)->start + DEF_ESTACK_SIZE;\
+        (dst)->edefault = NULL;\
 	(dst)->alloc_type = (s).alloc_type;\
     } else\
         *(dst) = (s);\
@@ -593,6 +594,7 @@ do {\
 	memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\
 	(dst)->wsp = (dst)->wstart + _wsz;\
 	(dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\
+        (dst)->wdefault = NULL;\
 	(dst)->alloc_type = s.alloc_type;\
     } else\
         *(dst) = s;\
-- 
cgit v1.2.3


From 65c7116bb38013c3d829f21b00bf094606e46731 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 19 Mar 2015 15:40:29 +0100
Subject: erts: Fix bug in binary_to_term with more than one big map

---
 erts/emulator/beam/external.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9a5ef56c47..458ebd8aa0 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3899,7 +3899,6 @@ dec_term_atom_common:
     /* Iterate through all the hamts and build tree nodes.
      */
     if (hamt_list) {
-        struct dec_term_hamt_placeholder* hamt = hamt_list;
 	ErtsHeapFactory factory;
 
 	factory.p = NULL;
@@ -3907,6 +3906,7 @@ dec_term_atom_common:
 	/* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
 
         do {
+	    struct dec_term_hamt_placeholder* hamt = hamt_list;
 	    *hamt->objp = erts_hashmap_from_array(&factory,
 						  hamt->leafs,
 						  hamt->size,
-- 
cgit v1.2.3


From 6487aac5977cf470bc6a2cd0964da2850ee38717 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 30 Oct 2014 23:57:01 +0100
Subject: Introduce a new time API

The old time API is based on erlang:now/0. The major issue with
erlang:now/0 is that it was intended to be used for so many
unrelated things. This tied these unrelated operations together
and unnecessarily caused performance, scalability as well as
accuracy, and precision issues for operations that do not need
to have such issues. The new API spreads different functionality
over multiple functions in order to improve on this.

The new API consists of a number of new BIFs:
- erlang:convert_time_unit/3
- erlang:monotonic_time/0
- erlang:monotonic_time/1
- erlang:system_time/0
- erlang:system_time/1
- erlang:time_offset/0
- erlang:time_offset/1
- erlang:timestamp/0
- erlang:unique_integer/0
- erlang:unique_integer/1
- os:system_time/0
- os:system_time/1

and a number of extensions of existing BIFs:
- erlang:monitor(time_offset, clock_service)
- erlang:system_flag(time_offset, finalize)
- erlang:system_info(os_monotonic_time_source)
- erlang:system_info(time_offset)
- erlang:system_info(time_warp_mode)
- erlang:system_info(time_correction)
- erlang:system_info(start_time)

See the "Time and Time Correction in Erlang" chapter of the
ERTS User's Guide for more information.
---
 erts/emulator/beam/atom.names         |   13 +-
 erts/emulator/beam/benchmark.h        |    6 +-
 erts/emulator/beam/bif.c              |  277 +++--
 erts/emulator/beam/bif.h              |    1 +
 erts/emulator/beam/bif.tab            |   20 +
 erts/emulator/beam/big.c              |   40 +
 erts/emulator/beam/big.h              |    7 +
 erts/emulator/beam/erl_alloc.types    |    1 +
 erts/emulator/beam/erl_bif_binary.c   |    1 +
 erts/emulator/beam/erl_bif_ddll.c     |    1 +
 erts/emulator/beam/erl_bif_info.c     |  108 +-
 erts/emulator/beam/erl_bif_timer.c    |    1 +
 erts/emulator/beam/erl_bif_trace.c    |    1 +
 erts/emulator/beam/erl_bif_unique.c   |  556 ++++++++++
 erts/emulator/beam/erl_bif_unique.h   |  131 +++
 erts/emulator/beam/erl_gc.c           |    1 +
 erts/emulator/beam/erl_init.c         |  129 ++-
 erts/emulator/beam/erl_lock_check.c   |    7 +-
 erts/emulator/beam/erl_lock_count.c   |    9 +-
 erts/emulator/beam/erl_lock_count.h   |    2 +-
 erts/emulator/beam/erl_monitors.h     |    3 +-
 erts/emulator/beam/erl_nif.c          |    1 +
 erts/emulator/beam/erl_process.c      |   58 +-
 erts/emulator/beam/erl_process.h      |    6 +-
 erts/emulator/beam/erl_thr_progress.c |   24 +-
 erts/emulator/beam/erl_time.h         |  272 ++++-
 erts/emulator/beam/erl_time_sup.c     | 1857 +++++++++++++++++++++++++--------
 erts/emulator/beam/global.h           |    5 -
 erts/emulator/beam/io.c               |    1 +
 erts/emulator/beam/sys.h              |  107 +-
 erts/emulator/beam/time.c             |  477 +++++----
 erts/emulator/beam/utils.c            |    5 +-
 32 files changed, 3201 insertions(+), 927 deletions(-)
 create mode 100644 erts/emulator/beam/erl_bif_unique.c
 create mode 100644 erts/emulator/beam/erl_bif_unique.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 5d06a32941..ced35be265 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -144,9 +144,11 @@ atom catchlevel
 atom cd
 atom cdr
 atom cflags
+atom CHANGE='CHANGE'
 atom characters_to_binary_int
 atom characters_to_list_int
 atom clear
+atom clock_service
 atom close
 atom closed
 atom code
@@ -156,6 +158,7 @@ atom compat_rel
 atom compile
 atom compressed
 atom config_h
+atom convert_time_unit
 atom connect
 atom connected
 atom connection_closed
@@ -236,7 +239,7 @@ atom first
 atom firstline
 atom flags
 atom flush
-atom flush_monitor_message
+atom flush_monitor_messages
 atom force
 atom format_cpu_topology
 atom free
@@ -344,6 +347,8 @@ atom message_queue_len
 atom messages
 atom meta
 atom meta_match_spec
+atom micro_seconds
+atom milli_seconds
 atom min_heap_size
 atom min_bin_vheap_size
 atom minor_version
@@ -354,12 +359,15 @@ atom monitored_by
 atom monitor
 atom monitor_nodes
 atom monitors
+atom monotonic
 atom more
 atom multi_scheduling
 atom multiline
+atom nano_seconds
 atom name
 atom named_table
 atom namelist
+atom native
 atom native_addresses
 atom Neq='=/='
 atom Neqeq='/='
@@ -450,6 +458,7 @@ atom ports
 atom port_count
 atom port_limit
 atom port_op
+atom positive
 atom print
 atom priority
 atom private
@@ -509,6 +518,7 @@ atom schedulers_online
 atom scheme
 atom scientific
 atom scope
+atom seconds
 atom sensitive
 atom sequential_tracer
 atom sequential_trace_token
@@ -554,6 +564,7 @@ atom term_to_binary_trap
 atom this
 atom thread_pool_size
 atom threads
+atom time_offset
 atom timeout
 atom timeout_value
 atom Times='*'
diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h
index 766edaac42..7f267b7201 100644
--- a/erts/emulator/beam/benchmark.h
+++ b/erts/emulator/beam/benchmark.h
@@ -175,10 +175,10 @@ extern BM_TIMER_T start_time;
 
 #else /* !USE_PERFCTR  (Assuming Solaris) */
 
-#define BM_TIMER_T hrtime_t
-#define BM_START_TIMER(t) system_clock = sys_gethrtime()
+#define BM_TIMER_T ErtsMonotonicTime
+#define BM_START_TIMER(t) system_clock = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time())
 #define BM_STOP_TIMER(t) do {                                        \
-    BM_TIMER_T tmp = (sys_gethrtime() - system_clock) - timer_time;  \
+    BM_TIMER_T tmp = (ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) - system_clock) - timer_time;  \
     t##_time += (tmp > 0 ? tmp : 0);                                 \
 } while(0)
 
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 42dd160e38..ec5122292e 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -40,16 +40,21 @@
 #define ERTS_PTAB_WANT_BIF_IMPL__
 #include "erl_ptab.h"
 #include "erl_bits.h"
+#include "erl_bif_unique.h"
 
-static Export* flush_monitor_message_trap = NULL;
+static Export* flush_monitor_messages_trap = NULL;
 static Export* set_cpu_topology_trap = NULL;
 static Export* await_proc_exit_trap = NULL;
 static Export* await_port_send_result_trap = NULL;
 Export* erts_format_cpu_topology_trap = NULL;
+Export *erts_convert_time_unit_trap = NULL;
 
 static Export *await_sched_wall_time_mod_trap;
 static erts_smp_atomic32_t sched_wall_time;
 
+static erts_smp_mtx_t ports_snapshot_mtx;
+erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */
+
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 /*
@@ -391,7 +396,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
     return res;
 }
 
-static int demonitor(Process *c_p, Eterm ref)
+static int demonitor(Process *c_p, Eterm ref, Eterm *multip)
 {
    ErtsMonitor *mon = NULL;  /* The monitor entry to delete */
    Process  *rp;    /* Local target process */
@@ -415,65 +420,73 @@ static int demonitor(Process *c_p, Eterm ref)
        goto done;
    }
 
-   if (mon->type != MON_ORIGIN) {
-       res = ERTS_DEMONITOR_BADARG;
-       goto done;
-   }
-   to = mon->pid;
-
-   if (is_atom(to)) {
-       /* Monitoring a name at node to */
-       ASSERT(is_node_name_atom(to));
-       dep = erts_sysname_to_connected_dist_entry(to);
-       ASSERT(dep != erts_this_dist_entry);
-       if (dep)
-	   deref_de = 1;
-   } else {
-       ASSERT(is_pid(to));
-       dep = pid_dist_entry(to);
-   }
-   if (dep != erts_this_dist_entry) {
-       res = remote_demonitor(c_p, dep, ref, to);
-       /* remote_demonitor() unlocks link lock on c_p */
-       unlock_link = 0;
-   }
-   else { /* Local monitor */
-       if (deref_de) {
-	   deref_de = 0;
-	   erts_deref_dist_entry(dep);
+   switch (mon->type) {
+   case MON_TIME_OFFSET:
+       *multip = am_true;
+       erts_demonitor_time_offset(ref);
+       res = ERTS_DEMONITOR_TRUE;
+       break;
+   case MON_ORIGIN:
+       to = mon->pid;
+       *multip = am_false;
+       if (is_atom(to)) {
+	   /* Monitoring a name at node to */
+	   ASSERT(is_node_name_atom(to));
+	   dep = erts_sysname_to_connected_dist_entry(to);
+	   ASSERT(dep != erts_this_dist_entry);
+	   if (dep)
+	       deref_de = 1;
+       } else {
+	   ASSERT(is_pid(to));
+	   dep = pid_dist_entry(to);
        }
-       dep = NULL;
-       rp = erts_pid2proc_opt(c_p,
-			      ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
-			      to,
-			      ERTS_PROC_LOCK_LINK,
-			      ERTS_P2P_FLG_ALLOW_OTHER_X);
-       mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
+       if (dep != erts_this_dist_entry) {
+	   res = remote_demonitor(c_p, dep, ref, to);
+	   /* remote_demonitor() unlocks link lock on c_p */
+	   unlock_link = 0;
+       }
+       else { /* Local monitor */
+	   if (deref_de) {
+	       deref_de = 0;
+	       erts_deref_dist_entry(dep);
+	   }
+	   dep = NULL;
+	   rp = erts_pid2proc_opt(c_p,
+				  ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
+				  to,
+				  ERTS_PROC_LOCK_LINK,
+				  ERTS_P2P_FLG_ALLOW_OTHER_X);
+	   mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
 #ifndef ERTS_SMP
-       ASSERT(mon);
+	   ASSERT(mon);
 #else
-       if (!mon)
-	   res = ERTS_DEMONITOR_FALSE;
-       else
+	   if (!mon)
+	       res = ERTS_DEMONITOR_FALSE;
+	   else
 #endif
-       {
-	   res = ERTS_DEMONITOR_TRUE;
-	   erts_destroy_monitor(mon);
-       }
-       if (rp) {
-	   ErtsMonitor *rmon;
-	   rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-	   if (rp != c_p)
-	       erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	   if (rmon != NULL)
-	       erts_destroy_monitor(rmon);
-       }
-       else {
-	   ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p);
-       }
+	   {
+	       res = ERTS_DEMONITOR_TRUE;
+	       erts_destroy_monitor(mon);
+	   }
+	   if (rp) {
+	       ErtsMonitor *rmon;
+	       rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
+	       if (rp != c_p)
+		   erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	       if (rmon != NULL)
+		   erts_destroy_monitor(rmon);
+	   }
+	   else {
+	       ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p);
+	   }
 
+       }
+       break;
+   default:
+       res = ERTS_DEMONITOR_BADARG;
+       *multip = am_false;
+       break;
    }
-
  done:
 
    if (unlock_link)
@@ -490,7 +503,8 @@ static int demonitor(Process *c_p, Eterm ref)
 
 BIF_RETTYPE demonitor_1(BIF_ALIST_1)
 {
-    switch (demonitor(BIF_P, BIF_ARG_1)) {
+    Eterm multi;
+    switch (demonitor(BIF_P, BIF_ARG_1, &multi)) {
     case ERTS_DEMONITOR_FALSE:
     case ERTS_DEMONITOR_TRUE:
 	BIF_RET(am_true);
@@ -508,6 +522,7 @@ BIF_RETTYPE demonitor_1(BIF_ALIST_1)
 BIF_RETTYPE demonitor_2(BIF_ALIST_2)
 {
     Eterm res = am_true;
+    Eterm multi = am_false;
     int info = 0;
     int flush = 0;
     Eterm list = BIF_ARG_2;
@@ -530,13 +545,18 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2)
     if (is_not_nil(list))
 	goto badarg;
 
-    switch (demonitor(BIF_P, BIF_ARG_1)) {
+    switch (demonitor(BIF_P, BIF_ARG_1, &multi)) {
     case ERTS_DEMONITOR_FALSE:
 	if (info)
 	    res = am_false;
-	if (flush)
-	    BIF_TRAP2(flush_monitor_message_trap, BIF_P, BIF_ARG_1, res);
+	if (flush) {
+	flush_messages:
+	    BIF_TRAP3(flush_monitor_messages_trap, BIF_P,
+		      BIF_ARG_1, multi, res);
+	}
     case ERTS_DEMONITOR_TRUE:
+	if (multi == am_true && flush)
+	    goto flush_messages;
 	BIF_RET(res);
     case ERTS_DEMONITOR_YIELD_TRUE:
 	ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
@@ -744,7 +764,22 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
     int deref_de = 0;
 
     /* Only process monitors are implemented */
-    if (BIF_ARG_1 != am_process) {
+    switch (BIF_ARG_1) {
+    case am_time_offset: {
+	Eterm ref;
+	if (BIF_ARG_2 != am_clock_service)
+	    goto error;
+	ref = erts_make_ref(BIF_P);
+	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET,
+			 ref, am_clock_service, NIL);
+	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_monitor_time_offset(BIF_P->common.id, ref);
+	BIF_RET(ref);
+    }
+    case am_process:
+	break;
+    default:
 	goto error;
     }
 
@@ -3446,91 +3481,6 @@ BIF_RETTYPE self_0(BIF_ALIST_0)
 
 /**********************************************************************/
 
-/*
-   New representation of refs in R9, see erl_term.h
-
-   In the first data word, only the usual 18 bits are used. Ordinarily,
-   in "long refs" all words are used (in other words, practically never
-   wrap around), but for compatibility with older nodes, "short refs"
-   exist. Short refs come into being by being converted from the old
-   external format for refs (tag REFERENCE_EXT). Short refs are
-   converted back to the old external format.
-
-   When converting a long ref to the external format in the case of
-   preparing for sending to an older node, the ref is truncated by only
-   using the first word (with 18 significant bits), and using the old tag
-   REFERENCE_EXT.
-
-   When comparing refs or different size, only the parts up to the length
-   of the shorter operand are used. This has the desirable effect that a
-   long ref sent to an old node and back will be treated as equal to
-   the original, although some of the bits have been lost.
-
-   The hash value for a ref always considers only the first word, since
-   in the above scenario, the original and the copy should have the same
-   hash value.
-*/
-
-static Uint32 reference0; /* Initialized in erts_init_bif */
-static Uint32 reference1;
-static Uint32 reference2;
-static erts_smp_spinlock_t make_ref_lock;
-static erts_smp_mtx_t ports_snapshot_mtx;
-erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */
-
-void
-erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS])
-{
-    erts_smp_spin_lock(&make_ref_lock);
-
-    reference0++;
-    if (reference0 >= MAX_REFERENCE) {
-	reference0 = 0;
-	reference1++;
-	if (reference1 == 0) {
-	    reference2++;
-	}
-    }
-
-    ref[0] = reference0;
-    ref[1] = reference1;
-    ref[2] = reference2;
-
-    erts_smp_spin_unlock(&make_ref_lock);
-}
-
-Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE])
-{
-    Eterm* hp = buffer;
-    Uint32 ref[ERTS_MAX_REF_NUMBERS];
-
-    erts_make_ref_in_array(ref);
-    write_ref_thing(hp, ref[0], ref[1], ref[2]);
-    return make_internal_ref(hp);
-}
-
-Eterm erts_make_ref(Process *p)
-{
-    Eterm* hp;
-    Uint32 ref[ERTS_MAX_REF_NUMBERS];
-
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
-
-    hp = HAlloc(p, REF_THING_SIZE);
-
-    erts_make_ref_in_array(ref);
-    write_ref_thing(hp, ref[0], ref[1], ref[2]);
-
-    return make_internal_ref(hp);
-}
-
-BIF_RETTYPE make_ref_0(BIF_ALIST_0)
-{
-    return erts_make_ref(BIF_P);
-}
-
-/**********************************************************************/
-
 /* return the time of day */
 
 BIF_RETTYPE time_0(BIF_ALIST_0)
@@ -4508,6 +4458,28 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    break;
 	}
 #endif
+    } else if (BIF_ARG_1 == am_time_offset
+	       && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) {
+	ErtsTimeOffsetState res;
+	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	res = erts_finalize_time_offset();
+        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	switch (res) {
+	case ERTS_TIME_OFFSET_PRELIMINARY: {
+	    DECL_AM(preliminary);
+	    BIF_RET(AM_preliminary);
+	}
+	case ERTS_TIME_OFFSET_FINAL: {
+	    DECL_AM(final);
+	    BIF_RET(AM_final);
+	}
+	case ERTS_TIME_OFFSET_VOLATILE: {
+	    DECL_AM(volatile);
+	    BIF_RET(AM_volatile);
+	}
+	default:
+	    ERTS_INTERNAL_ERROR("Unknown state");
+	}
     } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) {
 	int what;
 	if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2))
@@ -4795,11 +4767,6 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
 
 void erts_init_bif(void)
 {
-    reference0 = 0;
-    reference1 = 0;
-    reference2 = 0;
-
-    erts_smp_spinlock_init(&make_ref_lock, "make_ref");
     erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot");
     erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL);
 
@@ -4816,9 +4783,13 @@ void erts_init_bif(void)
 #endif
 		     , &bif_return_trap);
 
-    flush_monitor_message_trap = erts_export_put(am_erlang,
-						 am_flush_monitor_message,
-						 2);
+    flush_monitor_messages_trap = erts_export_put(am_erts_internal,
+						  am_flush_monitor_messages,
+						  3);
+
+    erts_convert_time_unit_trap = erts_export_put(am_erlang,
+						  am_convert_time_unit,
+						  3);
 
     set_cpu_topology_trap = erts_export_put(am_erlang,
 					    am_set_cpu_topology,
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 72c55ccb55..bd0f8cda2b 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -21,6 +21,7 @@
 #define __BIF_H__
 
 extern Export* erts_format_cpu_topology_trap;
+extern Export *erts_convert_time_unit_trap;
 
 #define BIF_RETTYPE Eterm
 
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index e68b8e6274..db8feb681b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -92,6 +92,8 @@ bif erlang:loaded/0
 bif erlang:localtime/0
 bif erlang:localtime_to_universaltime/2
 bif erlang:make_ref/0
+bif erlang:unique_integer/0
+bif erlang:unique_integer/1
 bif erlang:md5/1
 bif erlang:md5_init/0
 bif erlang:md5_update/2
@@ -104,6 +106,13 @@ ubif erlang:node/1
 ubif erlang:node/0
 bif erlang:nodes/1
 bif erlang:now/0
+bif erlang:monotonic_time/0
+bif erlang:monotonic_time/1
+bif erlang:system_time/0
+bif erlang:system_time/1
+bif erlang:time_offset/0
+bif erlang:time_offset/1
+bif erlang:timestamp/0
 
 bif erlang:open_port/2
 
@@ -158,6 +167,15 @@ bif erts_internal:check_process_code/2
 
 bif erts_internal:map_to_tuple_keys/1
 
+bif erts_internal:time_unit/0
+
+bif erts_internal:get_bif_timer_servers/0
+bif erts_internal:create_bif_timer/0
+bif erts_internal:access_bif_timer/1
+
+bif erts_internal:monitor_process/2
+bif erts_internal:is_system_process/1
+
 # inet_db support
 bif erlang:port_set_data/2
 bif erlang:port_get_data/1
@@ -347,6 +365,8 @@ bif os:getenv/0
 bif os:getenv/1
 bif os:getpid/0
 bif os:timestamp/0
+bif os:system_time/0
+bif os:system_time/1
 
 #
 # Bifs in the erl_ddll module (the module actually does not exist)
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index de7d370938..30e6a2c522 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1577,6 +1577,46 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
     return make_big(hp);
 }
 
+Eterm
+erts_uint64_array_to_big(Uint **hpp, int neg, int len, Uint64 *array)
+{
+    Uint *headerp;
+    int i, pot_digits, digits;
+
+    headerp = *hpp;
+
+    pot_digits = digits = 0;
+    for (i = 0; i < len; i++) {
+#if defined(ARCH_32) || HALFWORD_HEAP
+	Uint low_val = array[i] & ((Uint) 0xffffffff);
+	Uint high_val = (array[i] >> 32) & ((Uint) 0xffffffff);
+	BIG_DIGIT(headerp, pot_digits) = low_val;
+	pot_digits++;
+	if (low_val)
+	    digits = pot_digits;
+	BIG_DIGIT(headerp, pot_digits) = high_val;
+	pot_digits++;
+	if (high_val)
+	    digits = pot_digits;
+#else
+	Uint val = array[i];
+	BIG_DIGIT(headerp, pot_digits) = val;
+	pot_digits++;
+	if (val)
+	    digits = pot_digits;
+#endif
+    }	
+
+    if (neg)
+	*headerp = make_neg_bignum_header(digits);
+    else
+	*headerp = make_pos_bignum_header(digits);
+
+    *hpp = headerp + 1 + digits;
+
+    return make_big(headerp);
+}
+
 /*
 ** Convert a bignum to a double float
 */
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index da31876d75..4e4611de16 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -104,6 +104,9 @@ typedef Uint  dsize_t;	 /* Vector size type */
    : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(Uint64)(X)))
 #define ERTS_UINT64_HEAP_SIZE(X)				\
   (IS_USMALL(0, (X)) ? 0 : ERTS_UINT64_BIG_HEAP_SIZE__((X)))
+#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 2)
+#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 2)
+#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) (2*(LEN)+1)
 
 #else
 
@@ -111,6 +114,9 @@ typedef Uint  dsize_t;	 /* Vector size type */
   (IS_SSMALL((X)) ? 0 : (1 + 1))
 #define ERTS_UINT64_HEAP_SIZE(X)				\
   (IS_USMALL(0, (X)) ? 0 : (1 + 1))
+#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 1)
+#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 1)
+#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) ((LEN)+1)
 
 #endif
 
@@ -156,6 +162,7 @@ int term_to_Uint(Eterm, Uint*);
 int term_to_UWord(Eterm, UWord*);
 int term_to_Sint(Eterm, Sint*);
 #if HAVE_INT64
+Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *);
 int term_to_Uint64(Eterm, Uint64*);
 int term_to_Sint64(Eterm, Sint64*);
 #endif
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 21434eb117..4cd4ad100c 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -269,6 +269,7 @@ type    BUSY_CALLER_TAB	SHORT_LIVED	SYSTEM		busy_caller_table
 type    BUSY_CALLER	SHORT_LIVED	SYSTEM		busy_caller
 type	PROC_SYS_TSK	SHORT_LIVED	PROCESSES	proc_sys_task
 type	PROC_SYS_TSK_QS	SHORT_LIVED	PROCESSES	proc_sys_task_queues
+type    NEW_TIME_OFFSET	SHORT_LIVED	SYSTEM		new_time_offset
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 3bf78adce7..68004a7725 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -36,6 +36,7 @@
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
+#include "erl_bif_unique.h"
 
 
 /*
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 56cd2ba04f..fc4f819f56 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -45,6 +45,7 @@
 #include "big.h"
 #include "dist.h"
 #include "erl_version.h"
+#include "erl_bif_unique.h"
 #include "dtrace-wrapper.h"
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b90362d82c..80d49c7ce2 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -42,6 +42,7 @@
 #include "erl_cpu_topology.h"
 #include "erl_async.h"
 #include "erl_thr_progress.h"
+#include "erl_bif_unique.h"
 #define ERTS_PTAB_WANT_DEBUG_FUNCS__
 #include "erl_ptab.h"
 #ifdef HIPE
@@ -2099,6 +2100,46 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(am_opt);
 #endif
 	BIF_RET(res);
+    } else if (BIF_ARG_1 == am_time_offset) {
+	switch (erts_time_offset_state()) {
+	case ERTS_TIME_OFFSET_PRELIMINARY: {
+	    ERTS_DECL_AM(preliminary);
+	    BIF_RET(AM_preliminary);
+	}
+	case ERTS_TIME_OFFSET_FINAL: {
+	    ERTS_DECL_AM(final);
+	    BIF_RET(AM_final);
+	}
+	case ERTS_TIME_OFFSET_VOLATILE: {
+	    ERTS_DECL_AM(volatile);
+	    BIF_RET(AM_volatile);
+	}
+	default:
+	    ERTS_INTERNAL_ERROR("Invalid time offset state");
+	}
+    } else if (ERTS_IS_ATOM_STR("os_monotonic_time_source", BIF_ARG_1)) {
+	BIF_RET(erts_monotonic_time_source(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("time_correction", BIF_ARG_1)) {
+	BIF_RET(erts_has_time_correction() ? am_true : am_false);
+    } else if (ERTS_IS_ATOM_STR("start_time", BIF_ARG_1)) {
+	BIF_RET(erts_get_monotonic_start_time(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("time_warp_mode", BIF_ARG_1)) {
+	switch (erts_time_warp_mode()) {
+	case ERTS_NO_TIME_WARP_MODE: {
+	    ERTS_DECL_AM(no_time_warp);
+	    BIF_RET(AM_no_time_warp);
+	}
+	case ERTS_SINGLE_TIME_WARP_MODE: {
+	    ERTS_DECL_AM(single_time_warp);
+	    BIF_RET(AM_single_time_warp);
+	}
+	case ERTS_MULTI_TIME_WARP_MODE: {
+	    ERTS_DECL_AM(multi_time_warp);
+	    BIF_RET(AM_multi_time_warp);
+	}
+	default:
+	    ERTS_INTERNAL_ERROR("Invalid time warp mode");
+	}
     } else if (BIF_ARG_1 == am_allocated_areas) {
 	res = erts_allocated_areas(NULL, NULL, BIF_P);
 	BIF_RET(res);
@@ -2700,9 +2741,11 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
         BIF_RET(make_small(erts_db_get_max_tabs()));
     }
     else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) {
-	BIF_RET(erts_disable_tolerant_timeofday
-		? am_disabled
-		: am_enabled);
+	if (erts_has_time_correction()
+	    && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) {
+	    BIF_RET(am_enabled);
+	}
+	BIF_RET(am_disabled);
     }
     else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) {
 	BIF_RET(erts_eager_check_io ? am_true : am_false);
@@ -3400,6 +3443,29 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
         else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) {
             BIF_RET(erts_mmap_debug_info(BIF_P));
         }
+	else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) {
+	    BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P));
+	}
+	else if (ERTS_IS_ATOM_STR("min_unique_monotonic_integer", BIF_ARG_1)) {
+	    Sint64 value = erts_get_min_unique_monotonic_integer();
+	    if (IS_SSMALL(value))
+		BIF_RET(make_small(value));
+	    else {
+		Uint hsz = ERTS_SINT64_HEAP_SIZE(value);
+		Eterm *hp = HAlloc(BIF_P, hsz);
+		BIF_RET(erts_sint64_to_big(value, &hp));
+	    }
+	}
+	else if (ERTS_IS_ATOM_STR("min_unique_integer", BIF_ARG_1)) {
+	    Sint64 value = erts_get_min_unique_integer();
+	    if (IS_SSMALL(value))
+		BIF_RET(make_small(value));
+	    else {
+		Uint hsz = ERTS_SINT64_HEAP_SIZE(value);
+		Eterm *hp = HAlloc(BIF_P, hsz);
+		BIF_RET(erts_sint64_to_big(value, &hp));
+	    }
+	}
     }
     else if (is_tuple(BIF_ARG_1)) {
 	Eterm* tp = tuple_val(BIF_ARG_1);
@@ -3596,6 +3662,38 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    }
 	    break;
 	}
+	case 3: {
+	    if (ERTS_IS_ATOM_STR("check_time_config", tp[1])) {
+		int res, time_correction;
+		ErtsTimeWarpMode time_warp_mode;
+		if (tp[2] == am_true)
+		    time_correction = !0;
+		else if (tp[2] == am_false)
+		    time_correction = 0;
+		else
+		    break;
+		if (ERTS_IS_ATOM_STR("no_time_warp", tp[3]))
+		    time_warp_mode = ERTS_NO_TIME_WARP_MODE;
+		else if (ERTS_IS_ATOM_STR("single_time_warp", tp[3]))
+		    time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE;
+		else if (ERTS_IS_ATOM_STR("multi_time_warp", tp[3]))
+		    time_warp_mode = ERTS_MULTI_TIME_WARP_MODE;
+		else
+		    break;
+		res = erts_check_time_adj_support(time_correction,
+						  time_warp_mode);
+		BIF_RET(res ? am_true : am_false);
+	    }
+	    else if (ERTS_IS_ATOM_STR("make_unique_integer", tp[1])) {
+	      Eterm res = erts_debug_make_unique_integer(BIF_P,
+							 tp[2],
+							 tp[3]);
+	      if (is_non_value(res))
+		  break;
+	      BIF_RET(res);
+	    }
+	    break;
+	}
 	default:
 	    break;
 	}
@@ -3897,6 +3995,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		}
 	    }
 	}
+	else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) {
+	    int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2);
+	    BIF_RET(res ? am_true : am_false);
+	}
     }
 
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 03ac97283c..c9b02b48f5 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -27,6 +27,7 @@
 #include "error.h"
 #include "big.h"
 #include "erl_thr_progress.h"
+#include "erl_bif_unique.h"
 
 /****************************************************************************
 ** BIF Timer support
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 06fbbea123..08796df912 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -38,6 +38,7 @@
 #include "beam_bp.h"
 #include "erl_binary.h"
 #include "erl_thr_progress.h"
+#include "erl_bif_unique.h"
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
new file mode 100644
index 0000000000..57b0bab72f
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -0,0 +1,556 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2014. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "erl_alloc.h"
+#include "export.h"
+#include "bif.h"
+#include "erl_bif_unique.h"
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Reference                                                         *
+\*                                                                   */
+
+static union {
+    erts_atomic64_t count;
+    char align__[ERTS_CACHE_LINE_SIZE];
+} global_reference erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+
+/*
+ * ref[0] indicate thread creating reference as follows:
+ *
+ * - ref[0] == 0 => Non-scheduler thread;
+ * - else; ref[0] <= erts_no_schedulers =>
+ *      ordinary scheduler with id == ref[0];
+ * - else; ref[0] <= erts_no_schedulers
+ *                   + erts_no_dirty_cpu_schedulers =>
+ *      dirty cpu scheduler with id == 'ref[0] - erts_no_schedulers';
+ * - else =>
+ *      dirty io scheduler with id == 'ref[0]
+ *                                     - erts_no_schedulers
+ *                                     - erts_no_dirty_cpu_schedulers'
+ */
+
+#ifdef DEBUG
+static Uint32 max_thr_id;
+#endif
+
+static void
+init_reference(void)
+{
+#ifdef DEBUG
+    max_thr_id = (Uint32) erts_no_schedulers;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    max_thr_id += (Uint32) erts_no_dirty_cpu_schedulers;
+    max_thr_id += (Uint32) erts_no_dirty_io_schedulers;
+#endif
+#endif
+    erts_atomic64_init_nob(&global_reference.count, 0);
+}
+
+static ERTS_INLINE void
+global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+    Uint64 value;
+
+    value = (Uint64) erts_atomic64_inc_read_mb(&global_reference.count);
+    
+    erts_set_ref_numbers(ref, thr_id, value);
+}
+
+static ERTS_INLINE void
+make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    if (esdp)
+	erts_sched_make_ref_in_array(esdp, ref);
+    else
+	global_make_ref_in_array(0, ref);
+}
+
+void
+erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+    make_ref_in_array(ref);
+}
+
+Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE])
+{
+    Eterm* hp = buffer;
+    Uint32 ref[ERTS_MAX_REF_NUMBERS];
+
+    make_ref_in_array(ref);
+    write_ref_thing(hp, ref[0], ref[1], ref[2]);
+    return make_internal_ref(hp);
+}
+
+Eterm erts_make_ref(Process *c_p)
+{
+    Eterm* hp;
+    Uint32 ref[ERTS_MAX_REF_NUMBERS];
+
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+
+    hp = HAlloc(c_p, REF_THING_SIZE);
+
+    make_ref_in_array(ref);
+    write_ref_thing(hp, ref[0], ref[1], ref[2]);
+
+    return make_internal_ref(hp);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Unique Integer                                                    *
+\*                                                                   */
+
+static struct {
+    union {
+	struct {
+	    int left_shift;
+	    int right_shift;
+	    Uint64 mask;
+	    Uint64 val0_max;
+	} o;
+	char align__[ERTS_CACHE_LINE_SIZE];
+    } r;
+    union {
+	erts_atomic64_t val1;
+	char align__[ERTS_CACHE_LINE_SIZE];
+    } w;
+} unique_data erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+static void
+init_unique_integer(void)
+{
+    int bits;
+    unique_data.r.o.val0_max = (Uint64) erts_no_schedulers;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    unique_data.r.o.val0_max += (Uint64) erts_no_dirty_cpu_schedulers;
+    unique_data.r.o.val0_max += (Uint64) erts_no_dirty_io_schedulers;
+#endif
+    bits = erts_fit_in_bits_int64(unique_data.r.o.val0_max);
+    unique_data.r.o.left_shift = bits;
+    unique_data.r.o.right_shift = 64 - bits;
+    unique_data.r.o.mask = (((Uint64) 1) << bits) - 1;
+    erts_atomic64_init_nob(&unique_data.w.val1, -1);
+}
+
+#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2)
+
+static ERTS_INLINE Eterm
+bld_unique_integer_term(Eterm **hpp, Uint *szp,
+			Uint64 val0, Uint64 val1,
+			int positive)
+{
+    Uint hsz;
+    Uint64 unique_val[2];
+
+    unique_val[0] = ((Uint64) val0);
+    unique_val[0] |= ((Uint64) val1) << unique_data.r.o.left_shift;
+    unique_val[1] = ((Uint64) val1) >> unique_data.r.o.right_shift;
+    unique_val[1] &= unique_data.r.o.mask;
+
+    if (positive) {
+	unique_val[0]++;
+	if (unique_val[0] == 0)
+	    unique_val[1]++;
+    }
+    else {
+	ASSERT(MIN_SMALL < 0);
+	if (unique_val[1] == 0
+	    && unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) {
+	    Sint64 s_unique_val = (Sint64) unique_val[0];
+	    s_unique_val += MIN_SMALL;
+	    ASSERT(MIN_SMALL <= s_unique_val && s_unique_val < 0);
+	    if (szp)
+		*szp = 0;
+	    if (!hpp)
+		return THE_NON_VALUE;
+	    return make_small((Sint) s_unique_val);
+	}
+	if (unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) {
+	    ASSERT(unique_val[1] != 0);
+	    unique_val[1] -= 1;
+	}
+	unique_val[0] += MIN_SMALL;
+    }
+
+    if (!unique_val[1]) {
+	if (unique_val[0] <= MAX_SMALL) {
+	    if (szp)
+		*szp = 0;
+	    if (!hpp)
+		return THE_NON_VALUE;
+	    return make_small((Uint) unique_val[0]);
+	}
+
+	if (szp)
+	    *szp = ERTS_UINT64_HEAP_SIZE(unique_val[0]);
+	if (!hpp)
+	    return THE_NON_VALUE;
+	return erts_uint64_to_big(unique_val[0], hpp);
+    }
+    else {
+	Eterm tmp, *tmp_hp, res;
+	DeclareTmpHeapNoproc(local_heap, 2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE);
+
+	UseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE);
+
+	tmp_hp = local_heap;
+
+	tmp = erts_uint64_array_to_big(&tmp_hp, 0, 2, unique_val);
+	ASSERT(is_big(tmp));
+
+	hsz = big_arity(tmp) + 1;
+
+	ASSERT(hsz <= ERTS_MAX_UNIQUE_INT_HEAP_SIZE);
+
+	if (szp)
+	    *szp = hsz;
+
+	if (!hpp)
+	    res = THE_NON_VALUE;
+	else {
+	    int hix;
+	    Eterm *hp = *hpp;
+	    tmp_hp = big_val(tmp);
+	    for (hix = 0; hix < hsz; hix++)
+		hp[hix] = tmp_hp[hix];
+
+	    *hpp = hp + hsz;
+	    res = make_big(hp);
+	}
+
+	UnUseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE);
+
+	return res;
+    }
+}
+
+static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive)
+{
+    ErtsSchedulerData *esdp;
+    Uint64 thr_id, unique;
+    Uint hsz;
+    Eterm *hp;
+
+    esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    thr_id = (Uint64) esdp->thr_id;
+    unique = esdp->unique++;
+    bld_unique_integer_term(NULL, &hsz, thr_id, unique, positive);
+    hp = hsz ? HAlloc(c_p, hsz) : NULL;
+    return bld_unique_integer_term(&hp, NULL, thr_id, unique, positive);
+}
+
+Uint
+erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES])
+{
+    Uint sz;
+    bld_unique_integer_term(NULL, &sz, val[0], val[1], 0);
+    return sz;
+}
+
+Eterm
+erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES])
+{
+    return bld_unique_integer_term(hpp, NULL, val[0], val[1], 0);
+}
+
+void
+erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES])
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    if (esdp) {
+	val[0] = (Uint64) esdp->thr_id;
+	val[1] = esdp->unique++;
+    }
+    else {
+	val[0] = (Uint64) 0;
+	val[1] = (Uint64) erts_atomic64_inc_read_nob(&unique_data.w.val1);
+    }
+}
+
+
+Sint64
+erts_get_min_unique_integer(void)
+{
+    return (Sint64) MIN_SMALL;
+}
+
+/* --- Debug --- */
+
+Eterm
+erts_debug_make_unique_integer(Process *c_p, Eterm etval0, Eterm etval1)
+{
+    Uint64 val0, val1;
+    Uint hsz;
+    Eterm res, *hp, *end_hp;
+
+    if (!term_to_Uint64(etval0, &val0))
+	return THE_NON_VALUE;
+
+    if (!term_to_Uint64(etval1, &val1))
+	return THE_NON_VALUE;
+
+    bld_unique_integer_term(NULL, &hsz, val0, val1, 0);
+
+    hp = HAlloc(c_p, hsz);
+    end_hp = hp + hsz;
+
+    res = bld_unique_integer_term(&hp, NULL, val0, val1, 0);
+    if (hp != end_hp)
+	ERTS_INTERNAL_ERROR("Heap allocation error");
+
+    return res;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Strict Monotonic Counter                                          *
+\*                                                                   */
+
+static struct {
+    union {
+	erts_atomic64_t value;
+	char align__[ERTS_CACHE_LINE_SIZE];
+    } w;
+} raw_unique_monotonic_integer erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+#if defined(ARCH_32) || HALFWORD_HEAP
+#  define ERTS_UNIQUE_MONOTONIC_OFFSET ERTS_SINT64_MIN
+#else
+#  define ERTS_UNIQUE_MONOTONIC_OFFSET MIN_SMALL
+#endif
+
+static void
+init_unique_monotonic_integer(void)
+{
+    erts_atomic64_init_nob(&raw_unique_monotonic_integer.w.value,
+			   (erts_aint64_t) -1);
+}
+
+static ERTS_INLINE Uint64
+get_raw_unique_monotonic_integer(void)
+{
+    return (Uint64) erts_atomic64_inc_read_mb(&raw_unique_monotonic_integer.w.value);
+}
+
+static ERTS_INLINE Uint
+get_unique_monotonic_integer_heap_size(Uint64 raw, int positive)
+{
+    if (positive) {
+	Uint64 value = raw+1;
+	return ERTS_UINT64_HEAP_SIZE(value);
+    }
+    else {
+	Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET;
+	if (IS_SSMALL(value))
+	    return 0;
+#if defined(ARCH_32) || HALFWORD_HEAP
+	return ERTS_SINT64_HEAP_SIZE(value);
+#else
+	return ERTS_UINT64_HEAP_SIZE((Uint64) value);
+#endif
+    }
+}
+
+static ERTS_INLINE Eterm
+make_unique_monotonic_integer_value(Eterm *hp, Uint hsz, Uint64 raw, int positive)
+{
+    Eterm res;
+#ifdef DEBUG
+    Eterm *end_hp = hp + hsz;
+#endif
+
+    if (positive) {
+	Uint64 value = raw+1;
+	res = hsz ? erts_uint64_to_big(value, &hp) : make_small(value);
+    }
+    else {
+	Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET;
+	if (hsz == 0)
+	    res = make_small(value);
+	else {
+#if defined(ARCH_32) || HALFWORD_HEAP
+	    res = erts_sint64_to_big(value, &hp);
+#else 
+	    res = erts_uint64_to_big((Uint64) value, &hp);
+#endif
+	}
+    }
+
+    ASSERT(end_hp == hp);
+
+    return res;
+}
+
+static ERTS_INLINE Eterm
+unique_monotonic_integer_bif(Process *c_p, int positive)
+{
+    Uint64 raw;
+    Uint hsz;
+    Eterm *hp;
+
+    raw = get_raw_unique_monotonic_integer();
+    hsz = get_unique_monotonic_integer_heap_size(raw, positive);
+    hp = hsz ? HAlloc(c_p, hsz) : NULL;
+    return make_unique_monotonic_integer_value(hp, hsz, raw, positive);
+}
+
+Sint64
+erts_raw_get_unique_monotonic_integer(void)
+{
+    return get_raw_unique_monotonic_integer();
+}
+
+Uint
+erts_raw_unique_monotonic_integer_heap_size(Sint64 raw)
+{
+    return get_unique_monotonic_integer_heap_size(raw, 0);
+}
+
+Eterm
+erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw)
+{
+    Uint hsz = get_unique_monotonic_integer_heap_size(raw, 0);
+    Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, 0);
+    *hpp += hsz;
+    return res;
+}
+
+Sint64
+erts_get_min_unique_monotonic_integer(void)
+{
+    return ERTS_UNIQUE_MONOTONIC_OFFSET;
+}
+
+/* --- Debug --- */
+
+int
+erts_debug_set_unique_monotonic_integer_state(Eterm et_value)
+{
+    Sint64 value;
+
+    if (!term_to_Sint64(et_value, &value)) {
+	Uint64 uvalue;
+	if (!term_to_Uint64(et_value, &uvalue))
+	    return 0;
+	value = (Sint64) uvalue;
+    }
+
+    erts_atomic64_set_mb(&raw_unique_monotonic_integer.w.value,
+			 (erts_aint64_t) value);
+    return 1;
+}
+
+Eterm
+erts_debug_get_unique_monotonic_integer_state(Process *c_p)
+{
+    Uint64 value;
+    Eterm hsz, *hp;
+
+    value = (Uint64) erts_atomic64_read_mb(&raw_unique_monotonic_integer.w.value);
+
+    if (IS_USMALL(0, value))
+	return make_small(value);
+    hsz = ERTS_UINT64_HEAP_SIZE(value);
+    hp = HAlloc(c_p, hsz);
+    return erts_uint64_to_big(value, &hp);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Initilazation                                                     *
+\*                                                                   */
+
+void
+erts_bif_unique_init(void)
+{
+    init_reference();
+    init_unique_monotonic_integer();
+    init_unique_integer();
+}
+
+void
+erts_sched_bif_unique_init(ErtsSchedulerData *esdp)
+{
+    esdp->unique = (Uint64) 0;
+    esdp->ref = (Uint64) 0;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * The BIFs                                                          *
+\*                                                                   */
+
+
+BIF_RETTYPE make_ref_0(BIF_ALIST_0)
+{
+    BIF_RETTYPE res;
+    Eterm* hp;
+
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
+
+    hp = HAlloc(BIF_P, REF_THING_SIZE);
+
+    res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp);
+
+    BIF_RET(res);
+}
+
+BIF_RETTYPE unique_integer_0(BIF_ALIST_0)
+{
+    BIF_RET(unique_integer_bif(BIF_P, 0));
+}
+
+BIF_RETTYPE unique_integer_1(BIF_ALIST_1)
+{
+    Eterm modlist = BIF_ARG_1;
+    int monotonic = 0;
+    int positive = 0;
+    BIF_RETTYPE res;
+
+    while (is_list(modlist)) {
+	Eterm *consp = list_val(modlist);
+	switch (CAR(consp)) {
+	case am_monotonic:
+	    monotonic = 1;
+	    break;
+	case am_positive:
+	    positive = 1;
+	    break;
+	default:
+	    BIF_ERROR(BIF_P, BADARG);
+	}
+	modlist = CDR(consp);
+    }
+
+    if (is_not_nil(modlist))
+	BIF_ERROR(BIF_P, BADARG);
+
+    if (monotonic)
+	res = unique_monotonic_integer_bif(BIF_P, positive);
+    else
+	res = unique_integer_bif(BIF_P, positive);
+
+    BIF_RET(res);
+}
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
new file mode 100644
index 0000000000..cd001172a1
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -0,0 +1,131 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2014. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef ERTS_BIF_UNIQUE_H__
+#define ERTS_BIF_UNIQUE_H__
+
+#include "erl_process.h"
+#include "big.h"
+
+void erts_bif_unique_init(void);
+void erts_sched_bif_unique_init(ErtsSchedulerData *esdp);
+
+/* reference */
+Eterm erts_make_ref(Process *);
+Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]);
+void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+
+/* strict monotonic counter */
+
+#define ERTS_MAX_UNIQUE_MONOTONIC_INTEGER_HEAP_SIZE ERTS_MAX_UINT64_HEAP_SIZE
+
+/*
+ * Note that a raw value is an intermediate value that 
+ * not necessarily correspond to the end result.
+ */
+Sint64 erts_raw_get_unique_monotonic_integer(void);
+Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw);
+Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw);
+
+Sint64 erts_get_min_unique_monotonic_integer(void);
+
+int erts_debug_set_unique_monotonic_integer_state(Eterm et_value);
+Eterm erts_debug_get_unique_monotonic_integer_state(Process *c_p);
+
+/* unique integer */
+#define ERTS_UNIQUE_INT_RAW_VALUES 2
+#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2)
+
+Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]);
+Eterm erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]);
+void erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]);
+Sint64 erts_get_min_unique_integer(void);
+
+Eterm erts_debug_make_unique_integer(Process *c_p,
+				     Eterm etval0,
+				     Eterm etval1);
+
+
+ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS],
+					  Uint32 thr_id, Uint64 value);
+ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+ERTS_GLB_INLINE void erts_sched_make_ref_in_array(ErtsSchedulerData *esdp,
+						  Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+ERTS_GLB_INLINE Eterm erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp,
+						    Eterm buffer[REF_THING_SIZE]);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 value)
+{
+    /*
+     * We cannot use thread id in the first 18-bit word since
+     * the hash/phash/phash2 BIFs only hash on this word. If
+     * we did, we would get really poor hash values. Instead
+     * we have to shuffle the bits a bit.
+     */
+    ASSERT(thr_id == (thr_id & ((Uint32) 0x3ffff)));
+    ref[0] = (Uint32) (value & ((Uint64) 0x3ffff));
+    ref[1] = (((Uint32) (value & ((Uint64) 0xfffc0000)))
+	      | (thr_id & ((Uint32) 0x3ffff)));
+    ref[2] = (Uint32) ((value >> 32) & ((Uint64) 0xffffffff));
+}
+
+ERTS_GLB_INLINE Uint32
+erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+    return ref[1] & ((Uint32) 0x3ffff);
+}
+
+ERTS_GLB_INLINE Uint64
+erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+    return (((((Uint64) ref[2]) & ((Uint64) 0xffffffff)) << 32)
+	    | (((Uint64) ref[1]) & ((Uint64) 0xfffc0000))
+	    | (((Uint64) ref[0]) & ((Uint64) 0x3ffff)));
+}
+
+ERTS_GLB_INLINE void
+erts_sched_make_ref_in_array(ErtsSchedulerData *esdp,
+			     Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+    Uint64 value;
+
+    ASSERT(esdp);
+    value = esdp->ref++;
+    erts_set_ref_numbers(ref, (Uint32) esdp->thr_id, value);
+}
+
+ERTS_GLB_INLINE Eterm
+erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp,
+			      Eterm buffer[REF_THING_SIZE])
+{
+    Eterm* hp = buffer;
+    Uint32 ref[ERTS_MAX_REF_NUMBERS];
+
+    erts_sched_make_ref_in_array(esdp, ref);
+    write_ref_thing(hp, ref[0], ref[1], ref[2]);
+    return make_internal_ref(hp);
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /*  ERTS_BIF_UNIQUE_H__ */
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 0db42d4325..fea9b16e90 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -37,6 +37,7 @@
 #include "hipe_mode_switch.h"
 #endif
 #include "dtrace-wrapper.h"
+#include "erl_bif_unique.h"
 
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 61f8385efc..0e128c9b99 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -45,6 +45,7 @@
 #include "erl_thr_queue.h"
 #include "erl_async.h"
 #include "erl_ptab.h"
+#include "erl_bif_unique.h"
 
 #ifdef HIPE
 #include "hipe_mode_switch.h"	/* for hipe_mode_switch_init() */
@@ -134,7 +135,9 @@ static void erl_init(int ncpu,
 		     int legacy_proc_tab,
 		     int port_tab_sz,
 		     int port_tab_sz_ignore_files,
-		     int legacy_port_tab);
+		     int legacy_port_tab,
+		     int time_correction,
+		     ErtsTimeWarpMode time_warp_mode);
 
 static erts_atomic_t exiting;
 
@@ -188,10 +191,6 @@ static int no_dirty_io_schedulers;
 Uint32 verbose;             /* See erl_debug.h for information about verbose */
 #endif
 
-int erts_disable_tolerant_timeofday; /* Time correction can be disabled it is
-				      * not and/or it is too slow.
-				      */
-
 int erts_atom_table_size = ATOM_LIMIT;	/* Maximum number of atoms */
 
 int erts_modified_timing_level;
@@ -269,6 +268,19 @@ this_rel_num(void)
     return this_rel;
 }
 
+static ERTS_INLINE void
+set_default_time_adj(int *time_correction_p, ErtsTimeWarpMode *time_warp_mode_p)
+{
+    *time_correction_p = 1;
+    *time_warp_mode_p = ERTS_NO_TIME_WARP_MODE;
+    if (!erts_check_time_adj_support(*time_correction_p,
+				     *time_warp_mode_p)) {
+	*time_correction_p = 0;
+	ASSERT(erts_check_time_adj_support(*time_correction_p,
+					   *time_warp_mode_p));
+    }
+}
+
 /*
  * Common error printout function, all error messages
  * that don't go to the error logger go through here.
@@ -284,13 +296,22 @@ static int early_init(int *argc, char **argv);
 void
 erts_short_init(void)
 {
-    int ncpu = early_init(NULL, NULL);
+    
+    int ncpu;
+    int time_correction;
+    ErtsTimeWarpMode time_warp_mode;
+
+    set_default_time_adj(&time_correction,
+			 &time_warp_mode);
+    ncpu = early_init(NULL, NULL);
     erl_init(ncpu,
 	     ERTS_DEFAULT_MAX_PROCESSES,
 	     0,
 	     ERTS_DEFAULT_MAX_PORTS,
 	     0,
-	     0);
+	     0,
+	     time_correction,
+	     time_warp_mode);
     erts_initialized = 1;
 }
 
@@ -300,12 +321,15 @@ erl_init(int ncpu,
 	 int legacy_proc_tab,
 	 int port_tab_sz,
 	 int port_tab_sz_ignore_files,
-	 int legacy_port_tab)
+	 int legacy_port_tab,
+	 int time_correction,
+	 ErtsTimeWarpMode time_warp_mode)
 {
     init_benchmarking();
 
+    erts_bif_unique_init();
     erts_init_monitors();
-    erts_init_time();
+    erts_init_time(time_correction, time_warp_mode);
     erts_init_sys_common_misc();
     erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab);
     erts_init_scheduling(no_schedulers,
@@ -509,9 +533,9 @@ void erts_usage(void)
 
     /*    erts_fprintf(stderr, "-b func    set the boot function (default boot)\n"); */
 
-    erts_fprintf(stderr, "-c             disable continuous date/time correction with\n");
-    erts_fprintf(stderr, "               respect to uptime\n");
-
+    erts_fprintf(stderr, "-c bool        enable or disable time correction\n");
+    erts_fprintf(stderr, "-C mode        set time warp mode; valid modes are:\n");
+    erts_fprintf(stderr, "               no_time_warp|single_time_warp|multi_time_warp\n");
     erts_fprintf(stderr, "-d             don't write a crash dump for internally detected errors\n");
     erts_fprintf(stderr, "               (halt(String) will still produce a crash dump)\n");
     erts_fprintf(stderr, "-fn[u|a|l]     Control how filenames are interpreted\n");
@@ -681,7 +705,6 @@ early_init(int *argc, char **argv) /*
 
     erts_sched_compact_load = 1;
     erts_printf_eterm_func = erts_printf_term;
-    erts_disable_tolerant_timeofday = 0;
     display_items = 200;
     erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE;
     erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS;
@@ -1144,6 +1167,7 @@ early_init(int *argc, char **argv) /*
 
     /* Creates threads on Windows that depend on the arguments, so has to be after erl_sys_args */
     erl_sys_init();
+    erts_early_init_time_sup();
 
     erts_ets_realloc_always_moves = 0;
     erts_ets_always_compress = 0;
@@ -1187,7 +1211,11 @@ erl_start(int argc, char **argv)
     int port_tab_sz_ignore_files = 0;
     int legacy_proc_tab = 0;
     int legacy_port_tab = 0;
+    int time_correction;
+    ErtsTimeWarpMode time_warp_mode;
 
+    set_default_time_adj(&time_correction,
+			 &time_warp_mode);
 
     envbufsz = sizeof(envbuf);
     if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0)
@@ -1896,15 +1924,56 @@ erl_start(int argc, char **argv)
 	    }
 	    break;
 	}
-	case 'c':
-	    if (argv[i][2] == 0) { /* -c: documented option */
-		erts_disable_tolerant_timeofday = 1;
+	case 'C':
+	    arg = get_arg(argv[i]+2, argv[i+1], &i);
+	    if (sys_strcmp(arg, "no_time_warp") == 0)
+		time_warp_mode = ERTS_NO_TIME_WARP_MODE;
+	    else if (sys_strcmp(arg, "single_time_warp") == 0)
+		time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE;
+	    else if (sys_strcmp(arg, "multi_time_warp") == 0)
+		time_warp_mode = ERTS_MULTI_TIME_WARP_MODE;
+	    else {
+		erts_fprintf(stderr,
+			     "Invalid time warp mode: %s\n", arg);
+		erts_usage();
 	    }
+	    break;
+	case 'c':
+	    if (sys_strcmp(argv[i]+2, "false") == 0)
+		goto time_correction_false;
+	    else if (sys_strcmp(argv[i]+2, "true") == 0)
+		goto time_correction_true;
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
 	    else if (argv[i][2] == 'i') { /* -ci: undcoumented option*/
 		count_instructions = 1;
 	    }
 #endif
+	    else if (argv[i][2] == '\0') {
+		if (i + 1 >= argc)
+		    goto time_correction_false;
+		else {
+		    if (sys_strcmp(argv[i+1], "false") == 0) {
+			(void) get_arg(argv[i]+2, argv[i+1], &i);
+			goto time_correction_false;
+		    }
+		    else if (sys_strcmp(argv[i+1], "true") == 0) {
+			(void) get_arg(argv[i]+2, argv[i+1], &i);
+		    time_correction_true:
+			time_correction = 1;
+			break;
+		    }
+		    else {
+		    time_correction_false:
+			time_correction = 0;
+			break;
+		    }
+		}
+	    }
+	    else {
+		arg = get_arg(argv[i]+2, argv[i+1], &i);
+		erts_fprintf(stderr, "Invalid time correnction value: %s\n", arg);
+		erts_usage();
+	    }
 	    break;
 	case 'W':
 	    arg = get_arg(argv[i]+2, argv[i+1], &i);
@@ -1950,6 +2019,30 @@ erl_start(int argc, char **argv)
 	i++;
     }
 
+    if (!erts_check_time_adj_support(time_correction, time_warp_mode)) {
+	char *time_correction_str = time_correction ? "Enabled" : "Disabled";
+	char *time_warp_str = "undefined";
+	switch (time_warp_mode) {
+	case ERTS_NO_TIME_WARP_MODE:
+	    time_warp_str = "no";
+	    break;
+	case ERTS_SINGLE_TIME_WARP_MODE:
+	    time_warp_str = "single";
+	    break;
+	case ERTS_MULTI_TIME_WARP_MODE:
+	    time_warp_str = "multi";
+	    break;
+	default:
+	    time_warp_str = "undefined";
+	    break;
+	}
+	erts_fprintf(stderr, "%s time correction with %s time warp mode "
+		     "is not supported on this platform\n",
+		     time_correction_str,
+		     time_warp_str);
+	erts_usage();
+    }
+
 /* Output format on windows for sprintf defaults to three exponents.
  * We use two-exponent to mimic normal sprintf behaviour.
  */
@@ -1983,7 +2076,9 @@ erl_start(int argc, char **argv)
 	     legacy_proc_tab,
 	     port_tab_sz,
 	     port_tab_sz_ignore_files,
-	     legacy_port_tab);
+	     legacy_port_tab,
+	     time_correction,
+	     time_warp_mode);
 
     load_preloaded();
     erts_end_staging_code_ix();
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index b105ece6f1..261460d054 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -140,7 +140,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"async_enq_mtx",			NULL			},
 #ifdef ERTS_SMP
     {	"atom_tab",				NULL			},
-    {	"make_ref",				NULL			},
     {	"misc_op_list_pre_alloc_lock",		"address"		},
     {	"message_pre_alloc_lock",		"address"		},
     {	"ptimer_pre_alloc_lock",		"address",		},
@@ -168,6 +167,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"timer_wheel",				NULL			},
     {	"system_block",				NULL			},
     {	"timeofday",				NULL			},
+    {	"get_time",				NULL			},
+    {	"get_corrected_time",			NULL			},
     {	"breakpoints",				NULL			},
     {	"pollsets_lock",			NULL			},
     {	"pix_lock",				"address"		},
@@ -184,10 +185,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {   "efile_drv dtrace mutex",               NULL                    },
 #endif
     {	"mtrace_buf",				NULL			},
-#ifdef __WIN32__
 #ifdef ERTS_SMP
-    {   "sys_gethrtime",                        NULL                    },
-#endif
+    {	"os_monotonic_time",			NULL			},
 #endif
     {	"erts_alloc_hard_debug",		NULL			},
     {	"hard_dbg_mseg",		        NULL	                },
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index cf6996ea06..ddeb56a6be 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -104,11 +104,10 @@ static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) {
 }
 
 static void lcnt_time(erts_lcnt_time_t *time) {
-#if 0 || defined(HAVE_GETHRTIME)
-    SysHrTime hr_time;
-    hr_time  = sys_gethrtime();
-    time->s  = (unsigned long)(hr_time / 1000000000LL);
-    time->ns = (unsigned long)(hr_time - 1000000000LL*time->s);
+#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
+    ErtsMonotonicTime mtime = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time());
+    time->s  = (unsigned long) (mtime / 1000000000LL);
+    time->ns = (unsigned long) (mtime - 1000000000LL*time->s);
 #else
   SysTimeval tv;
   sys_gettimeofday(&tv);
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index ffbb93da1b..09fadd7e9e 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -76,7 +76,7 @@
 
 /* histogram */
 #define ERTS_LCNT_HISTOGRAM_MAX_NS    (((unsigned long)1LL << 28) - 1)
-#if 0 || defined(HAVE_GETHRTIME)
+#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
 #define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30)
 #define ERTS_LCNT_HISTOGRAM_RSHIFT    (0)
 #else
diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h
index fb11dbbd22..9972890db7 100644
--- a/erts/emulator/beam/erl_monitors.h
+++ b/erts/emulator/beam/erl_monitors.h
@@ -82,6 +82,7 @@
 /* Type tags for monitors */
 #define MON_ORIGIN 1
 #define MON_TARGET 3
+#define MON_TIME_OFFSET 7
 
 /* Type tags for links */
 #define LINK_PID 1   /* ...Or port */
@@ -103,7 +104,7 @@ typedef struct erts_monitor_or_link {
 typedef struct erts_monitor {
     struct erts_monitor *left, *right; 
     Sint16 balance;
-    Uint16 type;  /* MON_ORIGIN | MON_TARGET */
+    Uint16 type;  /* MON_ORIGIN | MON_TARGET | MON_TIME_OFFSET */
     Eterm ref;
     Eterm pid;    /* In case of distributed named monitor, this is the
 		     nodename atom in MON_ORIGIN process, otherwise a pid or
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index adc3520ebb..47d0af16bc 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -36,6 +36,7 @@
 #include "erl_thr_progress.h"
 #include "dtrace-wrapper.h"
 #include "erl_process.h"
+#include "erl_bif_unique.h"
 #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP))
 #define HAVE_USE_DTRACE 1
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ea63d20dfa..81bc3d2429 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -43,6 +43,7 @@
 #include "erl_async.h"
 #include "dtrace-wrapper.h"
 #include "erl_ptab.h"
+#include "erl_bif_unique.h"
 
 
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
@@ -702,8 +703,8 @@ init_sched_wall_time(ErtsSchedWallTime *swtp)
 static ERTS_INLINE Uint64
 sched_wall_time_ts(void)
 {
-#ifdef HAVE_GETHRTIME
-    return (Uint64) sys_gethrtime();
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    return (Uint64) erts_os_monotonic_time();
 #else
     Uint64 res;
     SysTimeval tv;
@@ -2843,7 +2844,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     else
 #endif
     {
-	erts_aint_t dt;
 
 	erts_smp_atomic32_set_relb(&function_calls, 0);
 	*fcalls = 0;
@@ -2868,6 +2868,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    goto sys_aux_work;
 
 	while (spincount-- > 0) {
+	    ErtsMonotonicTime current_time;
 
 	sys_poll_aux_work:
 
@@ -2877,8 +2878,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
 	    erl_sys_schedule(1); /* Might give us something to do */
 
-	    dt = erts_do_time_read_and_reset();
-	    if (dt) erts_bump_timer(dt);
+	    current_time = erts_get_monotonic_time();
+	    if (current_time >= erts_next_timeout_time())
+		erts_bump_timers(current_time);
 
 	sys_aux_work:
 #ifndef ERTS_SMP
@@ -2993,8 +2995,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	erl_sys_schedule(0);
 
-	dt = erts_do_time_read_and_reset();
-	if (dt) erts_bump_timer(dt);
+	{
+	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
+	    if (current_time >= erts_next_timeout_time())
+		erts_bump_timers(current_time);
+	}
 
 #ifndef ERTS_SMP
 	if (rq->len == 0 && !rq->misc.start)
@@ -5276,6 +5281,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->run_queue = runq;
     esdp->run_queue->scheduler = esdp;
 
+    esdp->thr_id = (Uint32) num;
+    erts_sched_bif_unique_init(esdp);
+
     if (daww_ptr) {
 	init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr);
 #ifdef ERTS_SMP
@@ -7717,6 +7725,8 @@ sched_dirty_cpu_thread_func(void *vesdp)
     callbacks.wait = NULL;
     callbacks.finalize_wait = NULL;
 
+    esdp->thr_id += erts_no_schedulers;
+
     erts_thr_progress_register_unmanaged_thread(&callbacks);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -7778,6 +7788,8 @@ sched_dirty_io_thread_func(void *vesdp)
     callbacks.wait = NULL;
     callbacks.finalize_wait = NULL;
 
+    esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers;
+
     erts_thr_progress_register_unmanaged_thread(&callbacks);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -8897,7 +8909,6 @@ Process *schedule(Process *p, int calls)
 {
     Process *proxy_p = NULL;
     ErtsRunQueue *rq;
-    erts_aint_t dt;
     ErtsSchedulerData *esdp;
     int context_reds;
     int fcalls;
@@ -9027,11 +9038,13 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-	dt = erts_do_time_read_and_reset();
-	if (dt) {
-	    erts_smp_runq_unlock(rq);
-	    erts_bump_timer(dt);
-	    erts_smp_runq_lock(rq);
+	{
+	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
+	    if (current_time >= erts_next_timeout_time()) {
+		erts_smp_runq_unlock(rq);
+		erts_bump_timers(current_time);
+		erts_smp_runq_lock(rq);
+	    }
 	}
 	BM_STOP_TIMER(system);
 
@@ -9177,6 +9190,7 @@ Process *schedule(Process *p, int calls)
 	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
 		 (fcalls > input_reductions &&
 		  prepare_for_sys_schedule(esdp, !0))) {
+	    ErtsMonotonicTime current_time;
 	    /*
 	     * Schedule system-level activities.
 	     */
@@ -9189,8 +9203,10 @@ Process *schedule(Process *p, int calls)
 #endif
 	    erts_smp_runq_unlock(rq);
 	    erl_sys_schedule(1);
-	    dt = erts_do_time_read_and_reset();
-	    if (dt) erts_bump_timer(dt);
+
+	    current_time = erts_get_monotonic_time();
+	    if (current_time >= erts_next_timeout_time())
+		erts_bump_timers(current_time);
 
 #ifdef ERTS_SMP
 	    erts_smp_runq_lock(rq);
@@ -11501,7 +11517,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
     ErtsMonitor *rmon;
     Process *rp;
 
-    if (mon->type == MON_ORIGIN) {
+    switch (mon->type) {
+    case MON_ORIGIN:
 	/* We are monitoring someone else, we need to demonitor that one.. */
 	if (is_atom(mon->pid)) { /* remote by name */
 	    ASSERT(is_node_name_atom(mon->pid));
@@ -11564,7 +11581,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		}
 	    }
 	}
-    } else { /* type == MON_TARGET */
+	break;
+    case MON_TARGET:
 	ASSERT(mon->type == MON_TARGET);
 	ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid));
 	if (is_internal_port(mon->pid)) {
@@ -11623,6 +11641,12 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		}
 	    }
 	}
+	break;
+    case MON_TIME_OFFSET:
+	erts_demonitor_time_offset(mon->ref);
+	break;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid monitor type");
     }
  done:
     /* As the monitors are previously removed from the process, 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index f50b217d4a..6ef56b1974 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -341,7 +341,7 @@ typedef struct {
 } ErtsRunQueueInfo;
 
 
-#ifdef HAVE_GETHRTIME
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
 #  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
 #  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
 #endif
@@ -588,6 +588,10 @@ struct ErtsSchedulerData_ {
     ErtsAuxWorkData aux_work_data;
     ErtsAtomCacheMap atom_cache_map;
 
+    Uint32 thr_id;
+    Uint64 unique;
+    Uint64 ref;
+
     ErtsSchedAllocData alloc_data;
 
     Uint64 reductions;
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 664c479eb6..f111846041 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1342,18 +1342,16 @@ erts_thr_progress_fatal_error_block(SWord timeout,
     ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
     erts_aint32_t bc;
     SWord time_left = timeout;
-    SysTimeval to;
+    ErtsMonotonicTime timeout_time;
 
     /*
      * Counting poll intervals may give us a too long timeout
-     * if cpu is busy. If we got tolerant time of day we use it
-     * to prevent this.
+     * if cpu is busy. We use timeout time to try to prevent
+     * this. In case we havn't got time correction this may
+     * however fail too...
      */
-    if (!erts_disable_tolerant_timeofday) {
-	erts_get_timeval(&to);
-	to.tv_sec += timeout / 1000;
-	to.tv_sec += timeout % 1000;
-    }
+    timeout_time = erts_get_monotonic_time();
+    timeout_time += ERTS_MSEC_TO_MONOTONIC((ErtsMonotonicTime) timeout);
 
     if (!tpd) {
 	/*
@@ -1378,14 +1376,8 @@ erts_thr_progress_fatal_error_block(SWord timeout,
 	    break; /* Succefully blocked all managed threads */
 	if (time_left <= 0)
 	    break; /* Timeout */
-	if (!erts_disable_tolerant_timeofday) {
-	    SysTimeval now;
-	    erts_get_timeval(&now);
-	    if (now.tv_sec > to.tv_sec)
-		break; /* Timeout */
-	    if (now.tv_sec == to.tv_sec && now.tv_usec >= to.tv_usec)
-		break; /* Timeout */
-	}
+	if (timeout_time <= erts_get_monotonic_time())
+	    break; /* Timeout */
     }
 }
 
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 4bbdcaa3e3..e461594e9c 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -20,11 +20,12 @@
 #ifndef ERL_TIME_H__
 #define ERL_TIME_H__
 
-#define ERTS_SHORT_TIME_T_MAX ERTS_AINT32_T_MAX
-#define ERTS_SHORT_TIME_T_MIN ERTS_AINT32_T_MIN
-typedef erts_aint32_t erts_short_time_t;
+#if defined(DEBUG) || 0
+#define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B)
+#else
+#define ERTS_TIME_ASSERT(B) ((void) 1)
+#endif
 
-extern erts_smp_atomic32_t do_time;	/* set at clock interrupt */
 extern SysTimeval erts_first_emu_time;
 
 /*
@@ -34,7 +35,7 @@ typedef struct erl_timer {
     struct erl_timer* next;	/* next entry tiw slot or chain */
     struct erl_timer* prev;	/* prev entry tiw slot or chain */
     Uint slot;			/* slot in timer wheel */
-    Uint count;			/* number of loops remaining */
+    ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
     int    active;		/* 1=activated, 0=deactivated */
     /* called when timeout */
     void (*timeout)(void*);
@@ -70,36 +71,36 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
 void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer);
 #endif
 
+void erts_monitor_time_offset(Eterm id, Eterm ref);
+int erts_demonitor_time_offset(Eterm ref);
+
+void erts_early_init_time_sup(void);
+void erts_late_init_time_sup(void);
+
 /* timer-wheel api */
 
-void erts_init_time(void);
+void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode);
 void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
 void erts_cancel_timer(ErlTimer*);
-void erts_bump_timer(erts_short_time_t);
+void erts_bump_timers(ErtsMonotonicTime);
 Uint erts_timer_wheel_memory_size(void);
 Uint erts_time_left(ErlTimer *);
-erts_short_time_t erts_next_time(void);
 
 #ifdef DEBUG
 void erts_p_slpq(void);
 #endif
 
-ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void);
-ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t);
+ErtsMonotonicTime erts_check_next_timeout_time(ErtsMonotonicTime);
 
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+extern erts_atomic64_t erts_next_timeout__;
 
-ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void)
-{
-    erts_short_time_t time = erts_smp_atomic32_xchg_acqb(&do_time, 0);
-    if (time < 0)
-	erl_exit(ERTS_ABORT_EXIT, "Internal time management error\n");
-    return time;
-}
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(void);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed)
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(void)
 {
-    erts_smp_atomic32_add_relb(&do_time, elapsed);
+    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&erts_next_timeout__);
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
@@ -121,25 +122,228 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
 typedef UWord erts_approx_time_t;
 erts_approx_time_t erts_get_approx_time(void);
 
-void erts_get_timeval(SysTimeval *tv);
-erts_time_t erts_get_time(void);
+int erts_has_time_correction(void);
+int erts_check_time_adj_support(int time_correction,
+				ErtsTimeWarpMode time_warp_mode);
+
+ErtsTimeWarpMode erts_time_warp_mode(void);
+
+typedef enum {
+    ERTS_TIME_OFFSET_PRELIMINARY,
+    ERTS_TIME_OFFSET_FINAL,
+    ERTS_TIME_OFFSET_VOLATILE
+} ErtsTimeOffsetState;
+
+ErtsTimeOffsetState erts_time_offset_state(void); 
+ErtsTimeOffsetState erts_finalize_time_offset(void); 
+struct process;
+Eterm erts_get_monotonic_start_time(struct process *c_p);
+Eterm erts_monotonic_time_source(struct process*c_p);
+
+#ifdef SYS_CLOCK_RESOLUTION
+#define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000))
+#else
+#define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution)
+#endif
+
+struct erts_time_sup_read_only__ {
+    ErtsMonotonicTime monotonic_time_unit;
+#ifndef SYS_CLOCK_RESOLUTION
+    ErtsMonotonicTime clktck_resolution;
+#endif
+};
+
+typedef struct {
+    union {
+	struct erts_time_sup_read_only__ o;
+	char align__[(((sizeof(struct erts_time_sup_read_only__) - 1)
+		       / ASSUMED_CACHE_LINE_SIZE) + 1)
+		     * ASSUMED_CACHE_LINE_SIZE];
+    } r;
+} ErtsTimeSupData;
+
+extern ErtsTimeSupData erts_time_sup__;
 
-ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
+ERTS_GLB_INLINE Uint64
+erts_time_unit_conversion(Uint64 value,
+			  Uint32 from_time_unit,
+			  Uint32 to_time_unit);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE int
-erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p)
+ERTS_GLB_INLINE Uint64
+erts_time_unit_conversion(Uint64 value,
+			  Uint32 from_time_unit,
+			  Uint32 to_time_unit)
 {
-    if (t1p->tv_sec == t2p->tv_sec) {
-	if (t1p->tv_usec < t2p->tv_usec)
-	    return -1;
-	else if (t1p->tv_usec > t2p->tv_usec)
-	    return 1;
-	return 0;
-    }
-    return t1p->tv_sec < t2p->tv_sec ? -1 : 1;
+    Uint64 high, low, result;
+    if (value <= ~((Uint64) 0)/to_time_unit)
+	return (value*to_time_unit)/from_time_unit;
+
+    low = value & ((Uint64) 0xffffffff);
+    high = (value >> 32) & ((Uint64) 0xffffffff);
+
+    low *= to_time_unit;
+    high *= to_time_unit;
+
+    high += (low >> 32) & ((Uint64) 0xffffffff);
+    low &= ((Uint64) 0xffffffff);
+
+    result = high % from_time_unit;
+    high /= from_time_unit;
+    high <<= 32;
+
+    result <<= 32;
+    result += low;
+    result /= from_time_unit;
+    result += high;
+
+    return result;
 }
 
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+
+/*
+ * If the monotonic time unit is a compile time constant,
+ * it is assumed (and need) to be a power of 10.
+ */
+
+#define ERTS_MONOTONIC_TIME_UNIT \
+    ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT)
+
+#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000
+/* Nano-second time unit */
+
+#define ERTS_MONOTONIC_TO_SEC__(NSEC) ((NSEC) / (1000*1000*1000))
+#define ERTS_MONOTONIC_TO_MSEC__(NSEC) ((NSEC) / (1000*1000))
+#define ERTS_MONOTONIC_TO_USEC__(NSEC) ((NSEC) / 1000)
+#define ERTS_MONOTONIC_TO_NSEC__(NSEC) (NSEC)
+
+#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000*1000))
+#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*(1000*1000))
+#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))*1000)
+#define ERTS_NSEC_TO_MONOTONIC__(NSEC) ((ErtsMonotonicTime) (NSEC))
+
+#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000
+/* Micro-second time unit */
+
+#define ERTS_MONOTONIC_TO_SEC__(USEC) ((USEC) / (1000*1000))
+#define ERTS_MONOTONIC_TO_MSEC__(USEC) ((USEC) / 1000)
+#define ERTS_MONOTONIC_TO_USEC__(USEC) (USEC)
+#define ERTS_MONOTONIC_TO_NSEC__(USEC) ((USEC)*1000)
+
+#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000))
+#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*1000)
+#define ERTS_USEC_TO_MONOTONIC__(USEC) ((ErtsMonotonicTime) (USEC))
+#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/1000)
+
+#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000
+/* Milli-second time unit */
+
+#define ERTS_MONOTONIC_TO_SEC__(MSEC) ((USEC)/(1000))
+#define ERTS_MONOTONIC_TO_MSEC__(MSEC) (MSEC)
+#define ERTS_MONOTONIC_TO_USEC__(MSEC) ((MSEC)*1000)
+#define ERTS_MONOTONIC_TO_NSEC__(MSEC) ((MSEC)*(1000*1000))
+
+#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*1000)
+#define ERTS_MSEC_TO_MONOTONIC__(MSEC) ((ErtsMonotonicTime) (MSEC))
+#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))/1000)
+#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/(1000*1000))
+
+#else
+#error Missing implementation for monotonic time unit
+#endif
+
+#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \
+    ((MON) / (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION))
+#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \
+    ((TCKS) * (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION))
+
+#else /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
+
+#define ERTS_MONOTONIC_TIME_UNIT (erts_time_sup__.r.o.monotonic_time_unit)
+
+#define ERTS_CONV_FROM_MON_UNIT___(M, TO)				\
+    ((ErtsMonotonicTime)						\
+     erts_time_unit_conversion((Uint64) (M),				\
+			       (Uint32) ERTS_MONOTONIC_TIME_UNIT,	\
+			       (Uint32) (TO)))
+
+#define ERTS_CONV_TO_MON_UNIT___(M, FROM)				\
+    ((ErtsMonotonicTime)						\
+     erts_time_unit_conversion((Uint64) (M),				\
+			       (Uint32) (FROM),				\
+			       (Uint32) ERTS_MONOTONIC_TIME_UNIT))	\
+
+#define ERTS_MONOTONIC_TO_SEC__(M) \
+    ERTS_CONV_FROM_MON_UNIT___((M), 1)
+#define ERTS_MONOTONIC_TO_MSEC__(M) \
+    ERTS_CONV_FROM_MON_UNIT___((M), 1000)
+#define ERTS_MONOTONIC_TO_USEC__(M) \
+    ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000)
+#define ERTS_MONOTONIC_TO_NSEC__(M) \
+    ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000*1000)
+
+#define ERTS_SEC_TO_MONOTONIC__(SEC) \
+    ERTS_CONV_TO_MON_UNIT___((SEC), 1)
+#define ERTS_MSEC_TO_MONOTONIC__(MSEC) \
+    ERTS_CONV_TO_MON_UNIT___((MSEC), 1000)
+#define ERTS_USEC_TO_MONOTONIC__(USEC) \
+    ERTS_CONV_TO_MON_UNIT___((USEC), 1000*1000)
+#define ERTS_NSEC_TO_MONOTONIC__(NSEC) \
+    ERTS_CONV_TO_MON_UNIT___((NSEC), 1000*1000*1000)
+
+#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \
+    ERTS_CONV_FROM_MON_UNIT___((MON), ERTS_CLKTCK_RESOLUTION)
+#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \
+    ERTS_CONV_TO_MON_UNIT___((TCKS), ERTS_CLKTCK_RESOLUTION)
+
+#endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
+
+#define ERTS_MSEC_TO_CLKTCKS__(MON) \
+    ((MON) * (ERTS_CLKTCK_RESOLUTION/1000))
+#define ERTS_CLKTCKS_TO_MSEC__(TCKS) \
+    ((TCKS) / (ERTS_CLKTCK_RESOLUTION/1000))
+
+#define ERTS_MONOTONIC_TO_SEC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_MONOTONIC_TO_SEC__((X)))
+#define ERTS_MONOTONIC_TO_MSEC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_MONOTONIC_TO_MSEC__((X)))
+#define ERTS_MONOTONIC_TO_USEC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_MONOTONIC_TO_USEC__((X)))
+#define ERTS_MONOTONIC_TO_NSEC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_MONOTONIC_TO_NSEC__((X)))
+#define ERTS_SEC_TO_MONOTONIC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_SEC_TO_MONOTONIC__((X)))
+#define ERTS_MSEC_TO_MONOTONIC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_MSEC_TO_MONOTONIC__((X)))
+#define ERTS_USEC_TO_MONOTONIC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_USEC_TO_MONOTONIC__((X)))
+#define ERTS_NSEC_TO_MONOTONIC(X)	\
+    (ERTS_TIME_ASSERT((X) >= 0),	\
+     ERTS_NSEC_TO_MONOTONIC__((X)))
+
+#define ERTS_MONOTONIC_TO_CLKTCKS(X) \
+    (ERTS_TIME_ASSERT((X) >= 0),		\
+     ERTS_MONOTONIC_TO_CLKTCKS__((X)))
+#define ERTS_CLKTCKS_TO_MONOTONIC(X) \
+    (ERTS_TIME_ASSERT((X) >= 0),		\
+     ERTS_CLKTCKS_TO_MONOTONIC__((X)))
+
+#define ERTS_MSEC_TO_CLKTCKS(X) \
+    (ERTS_TIME_ASSERT((X) >= 0),		\
+     ERTS_MSEC_TO_CLKTCKS__((X)))
+#define ERTS_CLKTCKS_TO_MSEC(X) \
+    (ERTS_TIME_ASSERT((X) >= 0),		\
+     ERTS_CLKTCKS_TO_MSEC__((X)))
+
 #endif /* ERL_TIME_H__ */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 3272a5326d..1534fb8058 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -18,60 +18,8 @@
  */
 
 /*
-** Support routines for the timer wheel
-**
-** This code contains two strategies for dealing with 
-** date/time changes in the system. 
-** If the system has some kind of high resolution timer (HAVE_GETHRTIME),
-** the high resolution timer is used to correct the time-of-day and the
-** timeouts, the base source is the hrtimer, but at certain intervals the 
-** OS time-of-day is checked and if it is not within certain bounds, the 
-** delivered time gets slowly adjusted for each call until
-** it corresponds to the system time (built-in adjtime...). 
-** The call gethrtime() is detected by autoconf on Unix, but other 
-** platforms may define it in erl_*_sys.h and implement 
-** their own high resolution timer. The high resolution timer
-** strategy is (probably) best on all systems where the timer have 
-** a resolution higher or equal to gettimeofday (or what's implemented
-** is sys_gettimeofday()). The actual resolution is the interesting thing,
-** not the unit's thats used (i.e. on VxWorks, nanoseconds can be
-** retrieved in terms of units, but the actual resolution is the same as 
-** for the clock ticks).
-** If the systems best timer routine is kernel ticks returned from 
-** sys_times(), and the actual resolution of sys_gettimeofday() is
-** better (like most unixes that does not have any realtime extensions), 
-** another strategy is used. The tolerant gettimeofday() corrects 
-** the value with respect to uptime (sys_times() return value) and checks 
-** for correction both when delivering timeticks and delivering nowtime.
-** this strategy is slower, but accurate on systems without better timer 
-** routines. The kernel tick resolution is not enough to implement
-** a gethrtime routine. On Linux and other non solaris unix-boxes the second 
-** strategy is used, on all other platforms we use the first.
-** 
-** The following is expected (from sys.[ch] and erl_*_sys.h):
-**
-** 64 bit integers. So it is, and so it will be.
-**
-** sys_init_time(), will return the clock resolution in MS and
-** that's about it. More could be added of course
-** If the clock-rate is constant (i.e. 1 ms) one can define 
-** SYS_CLOCK_RESOLUTION (to 1),
-** which makes erts_deliver_time/erts_time_remaining a bit faster.
-**
-** if HAVE_GETHRTIME is defined:
-**    sys_gethrtime() will return a SysHrTime (long long) representing 
-**    nanoseconds, sys_init_hrtime() will do any initialization.
-** else
-**    a long (64bit) integer type called Sint64 should be defined.
-**
-** sys_times() will return clock_ticks since start and 
-**    fill in a SysTimes structure (struct tms). Instead of CLK_TCK, 
-**    SYS_CLK_TCK is used to determine the resolution of kernel ticks.
-**
-** sys_gettimeofday() will take a SysTimeval (a struct timeval) as parameter
-**    and fill it in as gettimeofday(X,NULL).
-**
-*/
+ * Support routines for the time
+ */
 
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
@@ -80,384 +28,1028 @@
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
-
+ 
 static erts_smp_mtx_t erts_timeofday_mtx;
-
-static SysTimeval inittv; /* Used everywhere, the initial time-of-day */
+static erts_smp_mtx_t erts_get_time_mtx;
 
 static SysTimes t_start; /* Used in elapsed_time_both */
-static SysTimeval gtv; /* Used in wall_clock_elapsed_time_both */
-static SysTimeval then; /* Used in get_now */
-static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */
-SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */
+static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */
+static ErtsMonotonicTime previous_now; /* Used in get_now */
+
+static ErtsMonitor *time_offset_monitors = NULL;
+static Uint no_time_offset_monitors = 0;
 
-union {
-    erts_smp_atomic_t time;
-    char align[ERTS_CACHE_LINE_SIZE];
-} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+#ifdef DEBUG
+static int time_sup_initialized = 0;
+#endif
+
+#define ERTS_MONOTONIC_TIME_KILO \
+    ((ErtsMonotonicTime) 1000)
+#define ERTS_MONOTONIC_TIME_MEGA \
+    (ERTS_MONOTONIC_TIME_KILO*ERTS_MONOTONIC_TIME_KILO)
+#define ERTS_MONOTONIC_TIME_GIGA \
+    (ERTS_MONOTONIC_TIME_MEGA*ERTS_MONOTONIC_TIME_KILO)
+#define ERTS_MONOTONIC_TIME_TERA \
+    (ERTS_MONOTONIC_TIME_GIGA*ERTS_MONOTONIC_TIME_KILO)
 
 static void
-init_approx_time(void)
+schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset);
+
+/*
+ * NOTE! ERTS_MONOTONIC_TIME_START *need* to be a multiple
+ *       of ERTS_MONOTONIC_TIME_UNIT.
+ */
+
+#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+
+#ifdef ARCH_32
+/*
+ * Want to use a big-num of arity 2 as long as possible (584 years
+ * in the nano-second time unit case).
+ */
+#define ERTS_MONOTONIC_TIME_START		\
+    (((((((ErtsMonotonicTime) 1) << 32)-1)	\
+       / ERTS_MONOTONIC_TIME_UNIT)		\
+      * ERTS_MONOTONIC_TIME_UNIT)		\
+     + ERTS_MONOTONIC_TIME_UNIT)
+
+#else /* ARCH_64 */
+
+#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 1000*1000
+
+/*
+ * Using micro second time unit or lower. Start at zero since
+ * time will remain an immediate for a very long time anyway
+ * (18279 years in the micro second case)...
+ */
+#define ERTS_MONOTONIC_TIME_START ((ErtsMonotonicTime) 0)
+
+#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */
+
+/*
+ * Want to use an immediate as long as possible (36 years in the
+ * nano-second time unit case).
+*/
+#define ERTS_MONOTONIC_TIME_START 		\
+    ((((ErtsMonotonicTime) MIN_SMALL)		\
+      / ERTS_MONOTONIC_TIME_UNIT)		\
+     * ERTS_MONOTONIC_TIME_UNIT)
+
+#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */
+
+#endif /* ARCH_64 */
+
+#define ERTS_MONOTONIC_OFFSET_NATIVE ERTS_MONOTONIC_TIME_START
+#define ERTS_MONOTONIC_OFFSET_NSEC ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_TIME_START)
+#define ERTS_MONOTONIC_OFFSET_USEC ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_TIME_START)
+#define ERTS_MONOTONIC_OFFSET_MSEC ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_TIME_START)
+#define ERTS_MONOTONIC_OFFSET_SEC ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_TIME_START)
+
+#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
+
+/*
+ * Initialized in erts_init_time_sup()...
+ */
+
+#define ERTS_MONOTONIC_TIME_START (time_sup.r.o.start)
+#define ERTS_MONOTONIC_OFFSET_NATIVE (time_sup.r.o.start_offset.native)
+#define ERTS_MONOTONIC_OFFSET_NSEC (time_sup.r.o.start_offset.nsec)
+#define ERTS_MONOTONIC_OFFSET_USEC (time_sup.r.o.start_offset.usec)
+#define ERTS_MONOTONIC_OFFSET_MSEC (time_sup.r.o.start_offset.msec)
+#define ERTS_MONOTONIC_OFFSET_SEC (time_sup.r.o.start_offset.sec)
+
+#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
+
+#define ERTS_MONOTONIC_TO_SYS_TIME_VAL(TVP, MT)				\
+    do {								\
+	ErtsMonotonicTime sec__, usec__;				\
+	sec__ = ERTS_MONOTONIC_TO_SEC((MT));				\
+	usec__ = ERTS_MONOTONIC_TO_USEC((MT)) - sec__*1000000;		\
+	ASSERT(usec__ < 1000000);					\
+	(TVP)->tv_sec = sec__;						\
+	(TVP)->tv_usec = usec__;					\
+    } while (0)
+
+#define ERTS_MAX_SYSTEM_TIME_DIFF ERTS_MSEC_TO_MONOTONIC(10)
+#define ERTS_SYSTEM_TIME_DIFF_EXCEED_LIMIT(ESYSTIME, OSSYSTIME)		\
+    (((Uint64) (ESYSTIME)) - (((Uint64) (OSSYSTIME))			\
+			      - ERTS_MAX_SYSTEM_TIME_DIFF)		\
+     > 2*ERTS_MAX_SYSTEM_TIME_DIFF)
+
+#define ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF (ERTS_MAX_SYSTEM_TIME_DIFF/2)
+#define ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(500)
+
+struct time_sup_read_only__ {
+    ErtsMonotonicTime (*get_time)(void);
+    int correction;
+    ErtsTimeWarpMode warp_mode;
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    ErtsMonotonicTime moffset;
+    int os_monotonic_disable;
+    char *os_monotonic_func;
+    char *os_monotonic_clock_id;
+    int os_monotonic_locked;
+    Uint64 os_monotonic_resolution;
+#endif
+#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+    ErtsMonotonicTime start;
+    struct {
+	ErtsMonotonicTime native;
+	ErtsMonotonicTime nsec;
+	ErtsMonotonicTime usec;
+	ErtsMonotonicTime msec;
+	ErtsMonotonicTime sec;
+    } start_offset;
+#endif
+};
+
+typedef struct {
+#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+    ErtsMonotonicTime drift; /* Correction for os monotonic drift */
+#endif
+    ErtsMonotonicTime error; /* Correction for error between system times */
+} ErtsMonotonicCorrection;
+
+typedef struct {
+    ErtsMonotonicTime erl_mtime;
+    ErtsMonotonicTime os_mtime;
+    ErtsMonotonicCorrection correction;
+} ErtsMonotonicCorrectionInstance;
+
+#define ERTS_DRIFT_INTERVALS 5
+typedef struct {
+    struct {
+	struct {
+	    ErtsMonotonicTime sys;
+	    ErtsMonotonicTime mon;
+	} diff;
+	struct {
+	    ErtsMonotonicTime sys;
+	    ErtsMonotonicTime mon;
+	} time;
+    } intervals[ERTS_DRIFT_INTERVALS];
+    struct {
+	ErtsMonotonicTime sys;
+	ErtsMonotonicTime mon;
+    } acc;
+    int ix;
+    int dirty_counter;
+} ErtsMonotonicDriftData;
+
+typedef struct {
+    ErtsMonotonicCorrectionInstance prev;
+    ErtsMonotonicCorrectionInstance curr;    
+#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+    ErtsMonotonicDriftData drift;
+#endif
+    ErtsMonotonicTime last_check;
+    int short_check_interval;
+} ErtsMonotonicCorrectionData;
+
+struct time_sup_infrequently_changed__ {
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    struct {
+	erts_smp_rwmtx_t rwmtx;
+	ErlTimer timer;
+	ErtsMonotonicCorrectionData cdata;
+    } parmon;
+    ErtsMonotonicTime minit;
+#endif
+    int finalized_offset;
+    SysTimeval inittv; /* Used everywhere, the initial time-of-day */
+    ErtsMonotonicTime not_corrected_moffset;
+    erts_atomic64_t offset;
+};
+
+struct time_sup_frequently_changed__ {
+    ErtsMonotonicTime last_not_corrected_time;
+};
+
+static struct {
+    union {
+	struct time_sup_read_only__ o;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_read_only__))];
+    } r;
+    union {
+	struct time_sup_infrequently_changed__ c;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_infrequently_changed__))];
+    } inf;
+    union {
+	struct time_sup_frequently_changed__ c;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_frequently_changed__))];
+    } f;
+} time_sup erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+ErtsTimeSupData erts_time_sup__ erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+/*
+ * erts_get_approx_time() returns an *approximate* time
+ * in seconds. NOTE that this time may jump backwards!!!
+ */
+erts_approx_time_t
+erts_get_approx_time(void)
 {
-    erts_smp_atomic_init_nob(&approx.time, 0);
+    SysTimeval tv;
+    sys_gettimeofday(&tv);
+
+    return (erts_approx_time_t) tv.tv_sec;
 }
 
-static ERTS_INLINE erts_approx_time_t
-get_approx_time(void)
+static ERTS_INLINE void
+init_time_offset(ErtsMonotonicTime offset)
 {
-    return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time);
+    erts_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset);
 }
 
 static ERTS_INLINE void
-update_approx_time(SysTimeval *tv)
+set_time_offset(ErtsMonotonicTime offset)
 {
-    erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec;
-    erts_approx_time_t old_secs = get_approx_time();
-    if (old_secs != new_secs)
-	erts_smp_atomic_set_nob(&approx.time, new_secs);
+    erts_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset);
 }
 
-/*
- * erts_get_approx_time() returns an *approximate* time
- * in seconds. NOTE that this time may jump backwards!!!
- */
-erts_approx_time_t
-erts_get_approx_time(void)
+static ERTS_INLINE ErtsMonotonicTime
+get_time_offset(void)
 {
-    return get_approx_time();
+    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&time_sup.inf.c.offset);
 }
 
-#ifdef HAVE_GETHRTIME
 
-int erts_disable_tolerant_timeofday;
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+
+/*
+ * Time correction adjustments made due to
+ * error between Erlang system time and OS
+ * system time:
+ * - Large adjustment ~1%
+ * - Small adjustment ~0.05%
+ */
+#define ERTS_TCORR_ERR_UNIT 2048
+#define ERTS_TCORR_ERR_LARGE_ADJ 20
+#define ERTS_TCORR_ERR_SMALL_ADJ 1
 
-static SysHrTime hr_init_time, hr_last_correction_check, 
-    hr_correction, hr_last_time;
+#define ERTS_INIT_SHORT_INTERVAL_COUNTER 10
+#define ERTS_LONG_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(60)
+#define ERTS_SHORT_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(15)
 
-static void init_tolerant_timeofday(void)
+#define ERTS_TIME_DRIFT_MAX_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(100)
+#define ERTS_TIME_DRIFT_MIN_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(5)
+
+static ERTS_INLINE ErtsMonotonicTime
+calc_corrected_erl_mtime(ErtsMonotonicTime os_mtime,
+			 ErtsMonotonicCorrectionInstance *cip,
+			 ErtsMonotonicTime *os_mdiff_p)
 {
-    /* Should be in sys.c */
-#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_CONF)
-    if (sysconf(_SC_NPROCESSORS_CONF) > 1) {
-	char b[1024];
-	int maj,min,build;
-	os_flavor(b,1024);
-	os_version(&maj,&min,&build);
-	if (!strcmp(b,"sunos") && maj <= 5 && min <= 7) {
-	    erts_disable_tolerant_timeofday = 1;
-	}
-    }
+    ErtsMonotonicTime erl_mtime, diff = os_mtime - cip->os_mtime;
+    ERTS_TIME_ASSERT(diff >= 0);
+#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+    diff += (cip->correction.drift*diff)/ERTS_MONOTONIC_TIME_UNIT;
 #endif
-    hr_init_time = sys_gethrtime();
-    hr_last_correction_check = hr_last_time = hr_init_time;
-    hr_correction = 0;
+    erl_mtime = cip->erl_mtime;
+    erl_mtime += diff;
+    erl_mtime += cip->correction.error*(diff/ERTS_TCORR_ERR_UNIT);
+    if (os_mdiff_p)
+	*os_mdiff_p = diff;
+    return erl_mtime;
 }
 
-static void get_tolerant_timeofday(SysTimeval *tv)
+static ErtsMonotonicTime get_corrected_time(void)
 {
-    SysHrTime diff_time, curr;
+    ErtsMonotonicTime os_mtime;
+    ErtsMonotonicCorrectionData cdata;
+    ErtsMonotonicCorrectionInstance *cip;
+
+    erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
+
+    os_mtime = erts_os_monotonic_time();
+
+    cdata = time_sup.inf.c.parmon.cdata;
 
-    if (erts_disable_tolerant_timeofday) {
-	sys_gettimeofday(tv);
-	return;
+    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
+
+    if (os_mtime >= cdata.curr.os_mtime)
+	cip = &cdata.curr;
+    else {
+	if (os_mtime < cdata.prev.os_mtime)
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "OS monotonic time stepped backwards\n");
+	cip = &cdata.prev;
     }
-    *tv = inittv;
-    diff_time = ((curr = sys_gethrtime()) + hr_correction - hr_init_time) / 1000; 
 
-    if (curr < hr_init_time) {
-	erl_exit(1,"Unexpected behaviour from operating system high "
-		 "resolution timer");
+    return calc_corrected_erl_mtime(os_mtime, cip, NULL);
+}
+
+static void
+check_time_correction(void *unused)
+{
+    ErtsMonotonicCorrectionData cdata;
+    ErtsMonotonicCorrection new_correction;
+    ErtsMonotonicCorrectionInstance *cip;
+    ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime, erl_stime, time_offset;
+    Uint timeout;
+    SysTimeval tod;
+    int set_new_correction, begin_short_intervals = 0;
+
+    erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
+
+    ASSERT(time_sup.inf.c.finalized_offset);
+
+    os_mtime = erts_os_monotonic_time();
+    sys_gettimeofday(&tod);
+
+    cdata = time_sup.inf.c.parmon.cdata;
+
+    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
+
+    if (os_mtime < cdata.curr.os_mtime)
+	erl_exit(ERTS_ABORT_EXIT,
+		 "OS monotonic time stepped backwards\n");
+    cip = &cdata.curr;
+
+    erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, &mdiff);
+    time_offset = get_time_offset();
+    erl_stime = erl_mtime + time_offset;
+
+    os_stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
+    os_stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
+
+    sdiff = erl_stime - os_stime;
+
+    new_correction = cip->correction;
+    
+    if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE
+	&& (sdiff < -2*ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF
+	    || 2*ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF < sdiff)) {
+	/* System time diff exeeded limits; change time offset... */
+	time_offset -= sdiff;
+	sdiff = 0;
+	set_time_offset(time_offset);
+	schedule_send_time_offset_changed_notifications(time_offset);
+	begin_short_intervals = 1;
+	if (cdata.curr.correction.error == 0)
+	    set_new_correction = 0;
+	else {
+	    set_new_correction = 1;
+	    new_correction.error = 0;
+	}
+    }
+    else if (cdata.curr.correction.error == 0) {
+	if (sdiff < -ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	    set_new_correction = 1;
+	    if (sdiff < -ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
+	    else
+		new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ;
+	}
+	else if (sdiff > ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	    set_new_correction = 1;
+	    if (sdiff > ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
+	    else
+		new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ;
+	}
+	else {
+	    set_new_correction = 0;
+	}
+    }
+    else if (cdata.curr.correction.error > 0) {
+	if (sdiff < 0) {
+	    if (cdata.curr.correction.error == ERTS_TCORR_ERR_LARGE_ADJ
+		|| -ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF <= sdiff)
+		set_new_correction = 0;
+	    else {
+		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
+		set_new_correction = 1;
+	    }
+	}
+	else if (sdiff > ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	    set_new_correction = 1;
+	    if (sdiff > ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
+	    else
+		new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ;
+	}
+	else {
+	    set_new_correction = 1;
+	    new_correction.error = 0;
+	}
+    }
+    else /* if (cdata.curr.correction.error < 0) */ { 
+	if (0 < sdiff) {
+	    if (cdata.curr.correction.error == -ERTS_TCORR_ERR_LARGE_ADJ
+		|| sdiff <= ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+		set_new_correction = 0;
+	    else {
+		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
+		set_new_correction = 1;
+	    }
+	    set_new_correction = 0;
+	}
+	else if (sdiff < -ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	    set_new_correction = 1;
+	    if (sdiff < -ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
+	    else
+		new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ;
+	}
+	else {
+	    set_new_correction = 1;
+	    new_correction.error = 0;
+	}
     }
 
-    if ((curr - hr_last_correction_check) / 1000 > 1000000) {
-	/* Check the correction need */
-	SysHrTime tv_diff, diffdiff;
-	SysTimeval tmp;
-	int done = 0;
-
-	sys_gettimeofday(&tmp);
-	tv_diff = ((SysHrTime) tmp.tv_sec) * 1000000 + tmp.tv_usec;
-	tv_diff -= ((SysHrTime) inittv.tv_sec) * 1000000 + inittv.tv_usec;
-	diffdiff = diff_time - tv_diff;
-	if (diffdiff > 10000) {
-	    SysHrTime corr = (curr - hr_last_time) / 100;
-	    if (corr / 1000 >= diffdiff) {
-		++done;
-		hr_correction -= ((SysHrTime)diffdiff) * 1000;
-	    } else {
-		hr_correction -= corr;
+#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+    {
+	ErtsMonotonicDriftData *ddp = &time_sup.inf.c.parmon.cdata.drift;
+	int ix = ddp->ix;
+	ErtsMonotonicTime mtime_diff, old_os_mtime;
+
+	old_os_mtime = ddp->intervals[ix].time.mon;
+	mtime_diff = os_mtime - old_os_mtime;
+
+	if (mtime_diff >= ERTS_SEC_TO_MONOTONIC(10)) {
+	    ErtsMonotonicTime drift_adj, drift_adj_diff, old_os_stime,
+		stime_diff, mtime_acc, stime_acc, avg_drift_adj;
+
+	    old_os_stime = ddp->intervals[ix].time.sys;
+
+	    mtime_acc = ddp->acc.mon;
+	    stime_acc = ddp->acc.sys;
+
+	    avg_drift_adj = ((stime_acc - mtime_acc)*ERTS_MONOTONIC_TIME_UNIT) / mtime_acc;
+
+	    mtime_diff = os_mtime - old_os_mtime;
+	    stime_diff = os_stime - old_os_stime;
+	    drift_adj = ((stime_diff - mtime_diff)*ERTS_MONOTONIC_TIME_UNIT) / mtime_diff;
+	    
+	    ix++;
+	    if (ix >= ERTS_DRIFT_INTERVALS)
+		ix = 0;
+	    mtime_acc -= ddp->intervals[ix].diff.mon;
+	    mtime_acc += mtime_diff;
+	    stime_acc -= ddp->intervals[ix].diff.sys;
+	    stime_acc += stime_diff;
+
+	    ddp->intervals[ix].diff.mon = mtime_diff;
+	    ddp->intervals[ix].diff.sys = stime_diff;
+	    ddp->intervals[ix].time.mon = os_mtime;
+	    ddp->intervals[ix].time.sys = os_stime;
+
+	    ddp->ix = ix;
+	    ddp->acc.mon = mtime_acc;
+	    ddp->acc.sys = stime_acc;
+
+	    /*
+	     * If calculated drift adjustment is if off by more than 20% from the
+	     * average drift we interpret this as a discontinous leap in system
+	     * time and ignore it. If it actually is a change in drift we will
+	     * later detect this when the average drift change.
+	     */
+	    drift_adj_diff = avg_drift_adj - drift_adj;
+	    if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF
+		|| ERTS_TIME_DRIFT_MAX_ADJ_DIFF < drift_adj_diff) {
+		ddp->dirty_counter = ERTS_DRIFT_INTERVALS;
+		begin_short_intervals = 1;
 	    }
-	    diff_time = (curr + hr_correction - hr_init_time) / 1000; 
-	} else if (diffdiff < -10000) {
-	    SysHrTime corr = (curr - hr_last_time) / 100;
-	    if (corr / 1000 >= -diffdiff) {
-		++done;
-		hr_correction -= ((SysHrTime)diffdiff) * 1000;
-	    } else {
-		hr_correction += corr;
+	    else {
+		if (ddp->dirty_counter <= 0) {
+		    drift_adj = ((stime_acc - mtime_acc)*ERTS_MONOTONIC_TIME_UNIT) / mtime_acc;
+		}
+		if (ddp->dirty_counter >= 0) {
+		    if (ddp->dirty_counter == 0) {
+			/* Force set new drift correction... */
+			set_new_correction = 1;
+		    }
+		    ddp->dirty_counter--;
+		}
+		drift_adj_diff = drift_adj - new_correction.drift;
+		if (drift_adj_diff) {
+		    if (drift_adj_diff > ERTS_TIME_DRIFT_MAX_ADJ_DIFF)
+			drift_adj_diff = ERTS_TIME_DRIFT_MAX_ADJ_DIFF;
+		    else if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF)
+			drift_adj_diff = -ERTS_TIME_DRIFT_MAX_ADJ_DIFF;
+		    new_correction.drift += drift_adj_diff;
+
+		    if (drift_adj_diff < -ERTS_TIME_DRIFT_MIN_ADJ_DIFF
+			|| ERTS_TIME_DRIFT_MIN_ADJ_DIFF < drift_adj_diff) {
+			set_new_correction = 1;
+		    }
+		}
 	    }
-	    diff_time = (curr + hr_correction - hr_init_time) / 1000; 
-	} else {
-	    ++done;
 	}
-	if (done) {
-	    hr_last_correction_check = curr;
+    }
+#endif
+
+    begin_short_intervals |= set_new_correction;
+    
+    if (begin_short_intervals) {
+	time_sup.inf.c.parmon.cdata.short_check_interval
+	    = ERTS_INIT_SHORT_INTERVAL_COUNTER;
+    }
+    else if ((os_mtime - time_sup.inf.c.parmon.cdata.last_check
+	      >= ERTS_SHORT_TIME_CORRECTION_CHECK - ERTS_MONOTONIC_TIME_UNIT)
+	     && time_sup.inf.c.parmon.cdata.short_check_interval > 0) {
+	time_sup.inf.c.parmon.cdata.short_check_interval--;
+    }
+    time_sup.inf.c.parmon.cdata.last_check = os_mtime;
+
+    if (new_correction.error == 0)
+	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK);
+    else {
+	ErtsMonotonicTime ecorr = new_correction.error;
+	if (sdiff < 0)
+	    sdiff = -1*sdiff;
+	if (ecorr < 0)
+	    ecorr = -1*ecorr;
+	if (sdiff > ecorr*(ERTS_LONG_TIME_CORRECTION_CHECK/ERTS_TCORR_ERR_UNIT))
+	    timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK);
+	else {
+	    timeout = ERTS_MONOTONIC_TO_MSEC((ERTS_TCORR_ERR_UNIT*sdiff)/ecorr);
+	    if (timeout < 10)
+		timeout = 10;
 	}
     }
-    tv->tv_sec += (int) (diff_time / ((SysHrTime) 1000000));
-    tv->tv_usec += (int) (diff_time % ((SysHrTime) 1000000));
-    if (tv->tv_usec >= 1000000) {
-	tv->tv_usec -= 1000000;
-	tv->tv_sec += 1;
+
+    if (timeout > ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK)
+	&& time_sup.inf.c.parmon.cdata.short_check_interval) {
+	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK);
     }
-    hr_last_time = curr;
-}
+    
+    if (set_new_correction) {
+	erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx);
 
-#define correction (hr_correction/1000000)
+	os_mtime = erts_os_monotonic_time();
 
-#else /* !HAVE_GETHRTIME */
-#if !defined(CORRECT_USING_TIMES) 
-#define init_tolerant_timeofday() 
-#define get_tolerant_timeofday(tvp) sys_gettimeofday(tvp)
-#else
+	/* Save previous correction instance */
+	time_sup.inf.c.parmon.cdata.prev = *cip;
 
-typedef Sint64 Milli;
-
-static clock_t init_ct;
-static Sint64 ct_wrap;
-static Milli init_tv_m;
-static Milli correction_supress; 
-static Milli last_ct_diff;
-static Milli last_cc; 
-static clock_t last_ct;
-
-/* sys_times() might need to be wrapped and the values shifted (right)
-   a bit to cope with newer linux (2.5.*) kernels, this has to be taken care 
-   of dynamically to start with, a special version that uses
-   the times() return value as a high resolution timer can be made
-   to fully utilize the faster ticks, like on windows, but for now, we'll
-   settle with this silly workaround */
-#ifdef ERTS_WRAP_SYS_TIMES 
-#define KERNEL_TICKS() (sys_times_wrap() &  \
-			((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) 
-#else
-SysTimes dummy_tms;
+	/*
+	 * Current correction instance begin when
+	 * OS monotonic time has increased one unit.
+	 */
+	os_mtime++;
 
-#define KERNEL_TICKS() (sys_times(&dummy_tms) &  \
-			((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) 
+	/*
+	 * Erlang monotonic time corresponding to
+	 * next OS monotonic time using previous
+	 * correction.
+	 */
+	erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, NULL);
 
-#endif
+	/*
+	 * Save new current correction instance.
+	 */
+	time_sup.inf.c.parmon.cdata.curr.erl_mtime = erl_mtime;
+	time_sup.inf.c.parmon.cdata.curr.os_mtime = os_mtime;
+	time_sup.inf.c.parmon.cdata.curr.correction = new_correction;
 
-static void init_tolerant_timeofday(void)
-{
-    last_ct = init_ct = KERNEL_TICKS();
-    last_cc = 0;
-    init_tv_m = (((Milli) inittv.tv_sec) * 1000) + 
-	(inittv.tv_usec / 1000);
-    ct_wrap = 0;
-    correction_supress = 0;
+	erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx);
+    }
+
+    erts_set_timer(&time_sup.inf.c.parmon.timer,
+		   check_time_correction,
+		   NULL,
+		   NULL,
+		   timeout);
 }
 
+#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
 
-static void get_tolerant_timeofday(SysTimeval *tvp)
+static void
+init_check_time_correction(void *unused)
 {
-    clock_t current_ct;
-    SysTimeval current_tv;
-    Milli ct_diff;
-    Milli tv_diff;
-    Milli current_correction;
-    Milli act_correction;	/* long shown to be too small */
-    Milli max_adjust;
-
-    if (erts_disable_tolerant_timeofday) {
-	sys_gettimeofday(tvp);
-	return;
+    ErtsMonotonicDriftData *ddp;
+    ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff, stime_diff;
+    int ix;
+    SysTimeval tod;
+
+    ddp = &time_sup.inf.c.parmon.cdata.drift;
+    ix = ddp->ix;
+    old_mtime = ddp->intervals[0].time.mon;
+    old_stime = ddp->intervals[0].time.sys;
+
+    mtime = erts_os_monotonic_time();
+    sys_gettimeofday(&tod);
+
+    stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
+    stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
+
+    mtime_diff = mtime - old_mtime;
+    stime_diff = stime - old_stime;
+    if (100*stime_diff  < 80*mtime_diff || 120*mtime_diff < 100*stime_diff ) {
+	/* Had a system time leap... pretend no drift... */
+	stime_diff = mtime_diff;
+    }
+    
+    /*
+     * We use old time values in order to trigger
+     * a drift adjustment, and repeat this interval
+     * in all slots...
+     */
+    for (ix = 0; ix < ERTS_DRIFT_INTERVALS; ix++) {
+	ddp->intervals[ix].diff.mon = mtime_diff;
+	ddp->intervals[ix].diff.sys = stime_diff;
+	ddp->intervals[ix].time.mon = old_mtime;
+	ddp->intervals[ix].time.sys = old_stime;
     }
 
-#ifdef ERTS_WRAP_SYS_TIMES 
-#define TICK_MS (1000 / SYS_CLK_TCK_WRAP)
+    ddp->acc.sys = stime_diff*ERTS_DRIFT_INTERVALS;
+    ddp->acc.mon = mtime_diff*ERTS_DRIFT_INTERVALS;
+    ddp->ix = 0;
+    ddp->dirty_counter = ERTS_DRIFT_INTERVALS;
+
+    check_time_correction(NULL);
+}
+
+#endif
+
+static ErtsMonotonicTime
+finalize_corrected_time_offset(SysTimeval *todp)
+{
+    ErtsMonotonicTime os_mtime;
+    ErtsMonotonicCorrectionData cdata;
+    ErtsMonotonicCorrectionInstance *cip;
+
+    erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
+
+    os_mtime = erts_os_monotonic_time();
+    sys_gettimeofday(todp);
+
+    cdata = time_sup.inf.c.parmon.cdata;
+
+    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
+
+    if (os_mtime < cdata.curr.os_mtime)
+	erl_exit(ERTS_ABORT_EXIT,
+		 "OS monotonic time stepped backwards\n");
+    cip = &cdata.curr;
+
+    return calc_corrected_erl_mtime(os_mtime, cip, NULL);
+}
+
+static void
+late_init_time_correction(void)
+{
+    if (time_sup.inf.c.finalized_offset) {
+	erts_set_timer(&time_sup.inf.c.parmon.timer,
+#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+		       init_check_time_correction,
 #else
-#define TICK_MS (1000 / SYS_CLK_TCK)
+		       check_time_correction,
 #endif
-    current_ct = KERNEL_TICKS();
-    sys_gettimeofday(&current_tv);
-
-    /* I dont know if uptime can move some units backwards
-       on some systems, but I allow for small backward 
-       jumps to avoid such problems if they exist...*/
-    if (last_ct > 100 && current_ct < (last_ct - 100)) {
-	ct_wrap += ((Sint64) 1) << ((sizeof(clock_t) * 8) - 1);
+		       NULL,
+		       NULL,
+		       ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK));
     }
-    last_ct = current_ct;
-    ct_diff = ((ct_wrap + current_ct) - init_ct) * TICK_MS;
+}
 
-    /*
-     * We will adjust the time in milliseconds and we allow for 1% 
-     * adjustments, but if this function is called more often then every 100 
-     * millisecond (which is obviously possible), we will never adjust, so 
-     * we accumulate small times by setting last_ct_diff iff max_adjust > 0
-     */
-    if ((max_adjust = (ct_diff - last_ct_diff)/100) > 0)
-	last_ct_diff = ct_diff;
+#endif /* ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT */
 
-    tv_diff = ((((Milli) current_tv.tv_sec) * 1000) + 
-	       (current_tv.tv_usec / 1000)) - init_tv_m;
+static ErtsMonotonicTime get_not_corrected_time(void)
+{
+    SysTimeval tmp_tv;
+    ErtsMonotonicTime stime, mtime;
 
-    current_correction = ((ct_diff - tv_diff) / TICK_MS) * TICK_MS; /* trunc */
+    erts_smp_mtx_lock(&erts_get_time_mtx);
 
-    /* 
-     * We allow the current_correction value to wobble a little, as it
-     * suffers from the low resolution of the kernel ticks. 
-     * if it hasn't changed more than one tick in either direction, 
-     * we will keep the old value.
-     */
-    if ((last_cc > current_correction + TICK_MS) ||
-	(last_cc < current_correction - TICK_MS)) {
-	last_cc = current_correction;
-    } else {
-	current_correction = last_cc;
-    }
-    
-    /*
-     * As time goes, we try to get the actual correction to 0, 
-     * that is, make erlangs time correspond to the systems dito.
-     * The act correction is what we seem to need (current_correction)
-     * minus the correction suppression. The correction supression
-     * will change slowly (max 1% of elapsed time) but in millisecond steps.
-     */
-    act_correction = current_correction - correction_supress;
-    if (max_adjust > 0) {
-	/*
-	 * Here we slowly adjust erlangs time to correspond with the 
-	 * system time by changing the correction_supress variable.
-	 * It can change max_adjust milliseconds which is 1% of elapsed time
-	 */
-	if (act_correction > 0) {
-	    if (current_correction - correction_supress > max_adjust) {
-		correction_supress += max_adjust;
-	    } else {
-		correction_supress = current_correction;
-	    }
-	    act_correction = current_correction - correction_supress;
-	} else if (act_correction < 0) {
-	    if (correction_supress - current_correction > max_adjust) {
-		correction_supress -= max_adjust;
-	    } else {
-		correction_supress = current_correction;
+    sys_gettimeofday(&tmp_tv);
+
+    stime = ERTS_SEC_TO_MONOTONIC(tmp_tv.tv_sec);
+    stime += ERTS_USEC_TO_MONOTONIC(tmp_tv.tv_usec);
+
+    mtime = stime - time_sup.inf.c.not_corrected_moffset;
+
+    if (mtime >= time_sup.f.c.last_not_corrected_time)
+	time_sup.f.c.last_not_corrected_time = mtime;
+    else {
+	mtime = time_sup.f.c.last_not_corrected_time;
+
+	if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE) {
+	    ErtsMonotonicTime new_offset = stime - mtime;
+	    new_offset = ERTS_MONOTONIC_TO_USEC(new_offset);
+	    new_offset = ERTS_USEC_TO_MONOTONIC(new_offset);
+	    if (time_sup.inf.c.not_corrected_moffset != new_offset) {
+		time_sup.inf.c.not_corrected_moffset = new_offset;
+		set_time_offset(new_offset);
+		schedule_send_time_offset_changed_notifications(new_offset);
 	    }
-	    act_correction = current_correction - correction_supress;
 	}
+
     }
-    /*
-     * The actual correction will correct the timeval so that system 
-     * time warps gets smothed down.
-     */
-    current_tv.tv_sec += act_correction / 1000;
-    current_tv.tv_usec += (act_correction % 1000) * 1000;
-
-    if (current_tv.tv_usec >= 1000000) {
-	++current_tv.tv_sec ;
-	current_tv.tv_usec -= 1000000;
-    } else if (current_tv.tv_usec < 0) {
-	--current_tv.tv_sec;
-	current_tv.tv_usec += 1000000;
-    }
-    *tvp = current_tv;
-#undef TICK_MS
+
+    ASSERT(stime == mtime + time_sup.inf.c.not_corrected_moffset);
+
+    erts_smp_mtx_unlock(&erts_get_time_mtx);
+
+    return mtime;
 }
 
-#endif /* CORRECT_USING_TIMES */
-#endif /* !HAVE_GETHRTIME */
+int erts_check_time_adj_support(int time_correction,
+				ErtsTimeWarpMode time_warp_mode)
+{
+    if (!time_correction)
+	return 1;
 
-/*
-** Why this? Well, most platforms have a constant clock resolution of 1,
-** we dont want the deliver_time/time_remaining routines to waste 
-** time dividing and multiplying by/with a variable that's always one.
-** so the return value of sys_init_time is ignored on those platforms.
-*/
- 
-#ifndef SYS_CLOCK_RESOLUTION
-static int clock_resolution;
-#define CLOCK_RESOLUTION clock_resolution
+    /* User wants time correction */
+
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    return !time_sup.r.o.os_monotonic_disable;
 #else
-#define CLOCK_RESOLUTION SYS_CLOCK_RESOLUTION
+    return 0;
 #endif
+}
 
-/*
-** The clock resolution should really be the resolution of the 
-** time function in use, which on most platforms 
-** is 1. On VxWorks the resolution should be
-** the number of ticks per second (or 1, which would work nicely to).
-**
-** Setting lower resolutions is mostly interesting when timers are used
-** instead of something like select.
-*/
-
-static SysTimeval last_delivered; 
-
-static void init_erts_deliver_time(const SysTimeval *inittv)
+int
+erts_has_time_correction(void)
 {
-    /* We set the initial values for deliver_time here */
-    last_delivered = *inittv;
-    last_delivered.tv_usec = 1000 * (last_delivered.tv_usec / 1000); 
-                                                   /* ms resolution */
+    return time_sup.r.o.correction;
 }
 
-static void do_erts_deliver_time(const SysTimeval *current)
+void
+erts_early_init_time_sup(void)
 {
-    SysTimeval cur_time;
-    erts_time_t elapsed;
-    
-    /* calculate and deliver appropriate number of ticks */
-    cur_time = *current;
-    cur_time.tv_usec = 1000 * (cur_time.tv_usec / 1000); /* ms resolution */
-    elapsed = (1000 * (cur_time.tv_sec - last_delivered.tv_sec) +
-	       (cur_time.tv_usec - last_delivered.tv_usec) / 1000) / 
-	CLOCK_RESOLUTION;
+    ErtsSysInitTimeResult sys_init_time_res
+	= ERTS_SYS_INIT_TIME_RESULT_INITER;
 
-    /* Sometimes the time jump backwards,
-       resulting in a negative elapsed time. We compensate for
-       this by simply pretend as if the time stood still. :) */
+    sys_init_time(&sys_init_time_res);
 
-    if (elapsed > 0) {
+    erts_time_sup__.r.o.monotonic_time_unit
+	= sys_init_time_res.os_monotonic_time_unit;
 
-	ASSERT(elapsed < ((erts_time_t) ERTS_SHORT_TIME_T_MAX));
+#ifndef SYS_CLOCK_RESOLUTION
+    erts_time_sup__.r.o.clktck_resolution
+	= sys_init_time_res.sys_clock_resolution;
+    erts_time_sup__.r.o.clktck_resolution *= 1000;
+#endif
 
-	erts_do_time_add((erts_short_time_t) elapsed);
-	last_delivered = cur_time;
-    }
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    time_sup.r.o.os_monotonic_disable
+	= !sys_init_time_res.have_os_monotonic;
+    time_sup.r.o.os_monotonic_func
+	= sys_init_time_res.os_monotonic_info.func;
+    time_sup.r.o.os_monotonic_clock_id
+	= sys_init_time_res.os_monotonic_info.clock_id;
+    time_sup.r.o.os_monotonic_locked
+	= sys_init_time_res.os_monotonic_info.locked_use;
+    time_sup.r.o.os_monotonic_resolution
+	= sys_init_time_res.os_monotonic_info.resolution;
+#endif
 }
 
 int 
-erts_init_time_sup(void)
+erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 {
+#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+    ErtsMonotonicTime abs_start;
+#endif
+
+    ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX);
+
     erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday");
+    erts_smp_mtx_init(&erts_get_time_mtx, "get_time");
+
+    time_sup.r.o.correction = time_correction;
+    time_sup.r.o.warp_mode = time_warp_mode;
+ 
+    if (time_warp_mode == ERTS_SINGLE_TIME_WARP_MODE)
+	time_sup.inf.c.finalized_offset = 0;
+    else
+	time_sup.inf.c.finalized_offset = ~0;
+
+#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+
+#ifdef ARCH_32
+    time_sup.r.o.start = ((((ErtsMonotonicTime) 1) << 32)-1);
+    time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
+    time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
+    time_sup.r.o.start += ERTS_MONOTONIC_TIME_UNIT;
+    abs_start = time_sup.r.o.start;
+#else /* ARCH_64 */
+    if (ERTS_MONOTONIC_TIME_UNIT <= 1000*1000)
+	abs_start = time_sup.r.o.start = 0;
+    else {
+	time_sup.r.o.start = ((ErtsMonotonicTime) MIN_SMALL);
+	time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
+	time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
+	abs_start = -1*time_sup.r.o.start;
+    }
+#endif
 
-    init_approx_time();
+    time_sup.r.o.start_offset.native = time_sup.r.o.start;
+    time_sup.r.o.start_offset.nsec = (ErtsMonotonicTime)
+	erts_time_unit_conversion((Uint64) abs_start,
+				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
+				  (Uint32) 1000*1000*1000);
+    time_sup.r.o.start_offset.usec = (ErtsMonotonicTime)
+	erts_time_unit_conversion((Uint64) abs_start,
+				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
+				  (Uint32) 1000*1000);
+    time_sup.r.o.start_offset.msec = (ErtsMonotonicTime)
+	erts_time_unit_conversion((Uint64) abs_start,
+				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
+				  (Uint32) 1000);
+    time_sup.r.o.start_offset.sec = (ErtsMonotonicTime)
+	erts_time_unit_conversion((Uint64) abs_start,
+				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
+				  (Uint32) 1);
+    if (time_sup.r.o.start < 0) {
+	time_sup.r.o.start_offset.nsec *= -1;
+	time_sup.r.o.start_offset.usec *= -1;
+	time_sup.r.o.start_offset.msec *= -1;
+	time_sup.r.o.start_offset.sec *= -1;
+    }
 
-    last_emu_time.tv_sec = 0;
-    last_emu_time.tv_usec = 0;
+#endif
 
-#ifndef SYS_CLOCK_RESOLUTION
-    clock_resolution = sys_init_time();
+    if (ERTS_MONOTONIC_TIME_UNIT < ERTS_CLKTCK_RESOLUTION)
+	ERTS_INTERNAL_ERROR("Too small monotonic time time unit");
+
+#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    time_sup.r.o.correction = 0;
 #else
-    (void) sys_init_time();
-#endif
-    sys_gettimeofday(&inittv);
+    if (time_sup.r.o.os_monotonic_disable)
+	time_sup.r.o.correction = 0;
+
+    if (time_sup.r.o.correction) {
+	ErtsMonotonicCorrectionData *cdatap;
+	erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+	ErtsMonotonicTime offset;
+	time_sup.inf.c.minit = erts_os_monotonic_time();
+	sys_gettimeofday(&time_sup.inf.c.inittv);
+	time_sup.r.o.moffset = -1*time_sup.inf.c.minit;
+	offset = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
+	offset += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	init_time_offset(offset);
+
+	rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+	rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+	erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx,
+				&rwmtx_opts, "get_corrected_time");
+
+	cdatap = &time_sup.inf.c.parmon.cdata;
     
-#ifdef HAVE_GETHRTIME
-    sys_init_hrtime();
+#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+	cdatap->drift.intervals[0].time.sys
+	    = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
+	cdatap->drift.intervals[0].time.sys
+	    += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit;
+	cdatap->curr.correction.drift = 0;
+#endif
+	cdatap->curr.correction.error = 0;
+	cdatap->curr.erl_mtime = 0;
+	cdatap->curr.os_mtime = time_sup.inf.c.minit;
+	cdatap->last_check = time_sup.inf.c.minit;
+	cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER;
+	cdatap->prev = cdatap->curr;
+
+	time_sup.r.o.get_time = get_corrected_time;
+    }
+    else
 #endif
-    init_tolerant_timeofday();
+    {
+	ErtsMonotonicTime stime, offset;
+	time_sup.r.o.get_time = get_not_corrected_time;
+	stime = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
+	stime += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	offset = stime;
+	time_sup.inf.c.not_corrected_moffset = offset;
+	init_time_offset(offset);
+	time_sup.f.c.last_not_corrected_time = 0;
+    }
+
+    prev_wall_clock_elapsed = 0;
 
-    init_erts_deliver_time(&inittv);
-    gtv = inittv;
-    then.tv_sec = then.tv_usec = 0;
+    previous_now = ERTS_MONOTONIC_TO_USEC(get_time_offset());
 
-    erts_deliver_time();
+#ifdef DEBUG
+    time_sup_initialized = 1;
+#endif
 
-    return CLOCK_RESOLUTION;
+    return ERTS_CLKTCK_RESOLUTION/1000;
 }    
+
+void
+erts_late_init_time_sup(void)
+{
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    /* Timer wheel must be initialized */
+    if (time_sup.r.o.get_time == get_corrected_time)
+	late_init_time_correction();
+#endif
+}
+
+ErtsTimeWarpMode erts_time_warp_mode(void)
+{
+    return time_sup.r.o.warp_mode;
+}
+
+ErtsTimeOffsetState erts_time_offset_state(void)
+{
+    switch (time_sup.r.o.warp_mode) {
+    case ERTS_NO_TIME_WARP_MODE:
+	return ERTS_TIME_OFFSET_FINAL;
+    case ERTS_SINGLE_TIME_WARP_MODE:
+	if (time_sup.inf.c.finalized_offset)
+	    return ERTS_TIME_OFFSET_FINAL;
+	return ERTS_TIME_OFFSET_PRELIMINARY;
+    case ERTS_MULTI_TIME_WARP_MODE:
+	return ERTS_TIME_OFFSET_VOLATILE;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid time warp mode");
+	return ERTS_TIME_OFFSET_VOLATILE;
+    }
+}
+
+/*
+ * erts_finalize_time_offset() will only change time offset
+ * the first time it is called when the emulator has been
+ * started in "single time warp" mode. Returns previous
+ * state:
+ * * ERTS_TIME_OFFSET_PRELIMINARY - Finalization performed
+ * * ERTS_TIME_OFFSET_FINAL - Already finialized; nothing changed
+ * * ERTS_TIME_OFFSET_VOLATILE - Not supported, either in
+ * * no correction mode (or multi time warp mode; not yet implemented).
+ */
+
+ErtsTimeOffsetState
+erts_finalize_time_offset(void)
+{
+    switch (time_sup.r.o.warp_mode) {
+    case ERTS_NO_TIME_WARP_MODE:
+	return ERTS_TIME_OFFSET_FINAL;
+    case ERTS_MULTI_TIME_WARP_MODE:
+	return ERTS_TIME_OFFSET_VOLATILE;
+    case ERTS_SINGLE_TIME_WARP_MODE: {
+	ErtsTimeOffsetState res = ERTS_TIME_OFFSET_FINAL;
+
+	erts_smp_mtx_lock(&erts_get_time_mtx);
+
+	if (!time_sup.inf.c.finalized_offset) {
+	    ErtsMonotonicTime mtime, new_offset;
+	    SysTimeval tv;
+
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+	    if (!time_sup.r.o.correction)
+#endif
+	    {
+		ErtsMonotonicTime stime;
+		sys_gettimeofday(&tv);
+
+		stime = ERTS_SEC_TO_MONOTONIC(tv.tv_sec);
+		stime += ERTS_USEC_TO_MONOTONIC(tv.tv_usec);
+
+		mtime = stime - time_sup.inf.c.not_corrected_moffset;
+
+		if (mtime >= time_sup.f.c.last_not_corrected_time) {
+		    time_sup.f.c.last_not_corrected_time = mtime;
+		    new_offset = time_sup.inf.c.not_corrected_moffset;
+		}
+		else {
+		    mtime = time_sup.f.c.last_not_corrected_time;
+
+		    ASSERT(time_sup.inf.c.not_corrected_moffset != stime - mtime);
+		    new_offset = stime - mtime;
+		    time_sup.inf.c.not_corrected_moffset = new_offset;
+		}
+
+	    }
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+	    else {
+		mtime = finalize_corrected_time_offset(&tv);
+		new_offset = ERTS_SEC_TO_MONOTONIC(tv.tv_sec);
+		new_offset += ERTS_USEC_TO_MONOTONIC(tv.tv_usec);
+		new_offset -= mtime;
+
+	    }
+#endif
+	    new_offset = ERTS_MONOTONIC_TO_USEC(new_offset);
+	    new_offset = ERTS_USEC_TO_MONOTONIC(new_offset);
+
+	    set_time_offset(new_offset);
+	    schedule_send_time_offset_changed_notifications(new_offset);
+
+	    time_sup.inf.c.finalized_offset = ~0;
+	    res = ERTS_TIME_OFFSET_PRELIMINARY;
+	}
+
+	erts_smp_mtx_unlock(&erts_get_time_mtx);
+
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+	if (res == ERTS_TIME_OFFSET_PRELIMINARY
+	    && time_sup.r.o.get_time == get_corrected_time) {
+	    late_init_time_correction();
+	}
+#endif
+
+	return res;
+    }
+    default:
+	ERTS_INTERNAL_ERROR("Invalid time warp mode");
+	return ERTS_TIME_OFFSET_VOLATILE;
+    }
+}
+
 /* info functions */
 
 void 
@@ -498,23 +1090,16 @@ elapsed_time_both(UWord *ms_user, UWord *ms_sys,
 void 
 wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff)
 {
-    UWord prev_total;
-    SysTimeval tv;
+    ErtsMonotonicTime now, elapsed;
 
     erts_smp_mtx_lock(&erts_timeofday_mtx);
 
-    get_tolerant_timeofday(&tv);
+    now = time_sup.r.o.get_time();
 
-    *ms_total = 1000 * (tv.tv_sec - inittv.tv_sec) +
-	(tv.tv_usec - inittv.tv_usec) / 1000;
-
-    prev_total = 1000 * (gtv.tv_sec - inittv.tv_sec) +
-	(gtv.tv_usec - inittv.tv_usec) / 1000;
-    *ms_diff = *ms_total - prev_total;
-    gtv = tv;
-
-    /* must sync the machine's idea of time here */
-    do_erts_deliver_time(&tv);
+    elapsed = ERTS_MONOTONIC_TO_MSEC(now);
+    *ms_total = (UWord) elapsed;
+    *ms_diff = (UWord) (elapsed - prev_wall_clock_elapsed);
+    prev_wall_clock_elapsed = elapsed;
 
     erts_smp_mtx_unlock(&erts_timeofday_mtx);
 }
@@ -890,38 +1475,41 @@ univ_to_local(Sint *year, Sint *month, Sint *day,
     return 0;
 }
 
-
 /* get a timestamp */
 void
 get_now(Uint* megasec, Uint* sec, Uint* microsec)
 {
-    SysTimeval now;
+    ErtsMonotonicTime now_megasec, now_sec, now, mtime, time_offset;
     
+    mtime = time_sup.r.o.get_time();
+    time_offset = get_time_offset();
+    now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset);
+
     erts_smp_mtx_lock(&erts_timeofday_mtx);
-    
-    get_tolerant_timeofday(&now);
-    do_erts_deliver_time(&now);
-
-    /* Make sure time is later than last */
-    if (then.tv_sec > now.tv_sec ||
-	(then.tv_sec == now.tv_sec && then.tv_usec >= now.tv_usec)) {
-	now = then;
-	now.tv_usec++;
-    }
-    /* Check for carry from above + general reasonability */
-    if (now.tv_usec >= 1000000) {
-	now.tv_usec = 0;
-	now.tv_sec++;
-    }
-    then = now;
+
+    /* Make sure now time is later than last time */
+    if (now <= previous_now)
+	now = previous_now + 1;
+
+    previous_now = now;
     
     erts_smp_mtx_unlock(&erts_timeofday_mtx);
-    
-    *megasec = (Uint) (now.tv_sec / 1000000);
-    *sec = (Uint) (now.tv_sec % 1000000);
-    *microsec = (Uint) (now.tv_usec);
 
-    update_approx_time(&now);
+    now_megasec = now / ERTS_MONOTONIC_TIME_TERA;
+    now_sec = now / ERTS_MONOTONIC_TIME_MEGA;
+    *megasec = (Uint) now_megasec;
+    *sec = (Uint) (now_sec - now_megasec*ERTS_MONOTONIC_TIME_MEGA);
+    *microsec = (Uint) (now - now_sec*ERTS_MONOTONIC_TIME_MEGA);
+
+    ASSERT(((ErtsMonotonicTime) *megasec)*ERTS_MONOTONIC_TIME_TERA
+	   + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA
+	   + ((ErtsMonotonicTime) *microsec) == now);
+}
+
+ErtsMonotonicTime
+erts_get_monotonic_time(void)
+{
+    return time_sup.r.o.get_time();
 }
 
 void
@@ -934,102 +1522,465 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
     *megasec = (Uint) (now.tv_sec / 1000000);
     *sec = (Uint) (now.tv_sec % 1000000);
     *microsec = (Uint) (now.tv_usec);
-
-    update_approx_time(&now);
 }
 
+#ifdef HAVE_ERTS_NOW_CPU
+void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) {
+  SysCpuTime t;
+  SysTimespec tp;
 
-/* deliver elapsed *ticks* to the machine - takes a pointer
-   to a struct timeval representing current time (to save
-   a gettimeofday() where possible) or NULL */
+  sys_get_proc_cputime(t, tp);
+  *microsec = (Uint)(tp.tv_nsec / 1000);
+  t = (tp.tv_sec / 1000000);
+  *megasec = (Uint)(t % 1000000);
+  *sec = (Uint)(tp.tv_sec % 1000000);
+}
+#endif
 
-void erts_deliver_time(void) {
-    SysTimeval now;
-    
-    erts_smp_mtx_lock(&erts_timeofday_mtx);
-    
-    get_tolerant_timeofday(&now);
-    do_erts_deliver_time(&now);
-    
-    erts_smp_mtx_unlock(&erts_timeofday_mtx);
+#include "big.h"
 
-    update_approx_time(&now);
+void
+erts_monitor_time_offset(Eterm id, Eterm ref)
+{
+    erts_smp_mtx_lock(&erts_get_time_mtx);
+    erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL);
+    no_time_offset_monitors++;
+    erts_smp_mtx_unlock(&erts_get_time_mtx);
+}
+
+int
+erts_demonitor_time_offset(Eterm ref)
+{
+    int res;
+    ErtsMonitor *mon;
+    ASSERT(is_internal_ref(ref));
+    erts_smp_mtx_lock(&erts_get_time_mtx);
+    mon = erts_remove_monitor(&time_offset_monitors, ref);
+    if (!mon)
+	res = 0;
+    else {
+	ASSERT(no_time_offset_monitors > 0);
+	no_time_offset_monitors--;
+	res = 1;
+    }
+    erts_smp_mtx_unlock(&erts_get_time_mtx);
+    if (res)
+	erts_destroy_monitor(mon);
+    return res;
 }
 
-/* get *real* time (not ticks) remaining until next timeout - if there
-   isn't one, give a "long" time, that is guaranteed
-   to not cause overflow when we report elapsed time later on */
+typedef struct {
+    Eterm pid;
+    Eterm ref;
+    Eterm heap[REF_THING_SIZE];
+} ErtsTimeOffsetMonitorInfo;
+
+typedef struct {
+    Uint ix;
+    ErtsTimeOffsetMonitorInfo *to_mon_info;
+} ErtsTimeOffsetMonitorContext;
 
-void erts_time_remaining(SysTimeval *rem_time)
+static void
+save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt)
 {
-    erts_time_t ticks;
-    SysTimeval cur_time;
-    erts_time_t elapsed;
-
-    /* erts_next_time() returns no of ticks to next timeout or -1 if none */
-
-    ticks = (erts_time_t) erts_next_time();
-    if (ticks == (erts_time_t) -1) {
-	/* timer queue empty */
-	/* this will cause at most 100000000 ticks */
-	rem_time->tv_sec = 100000;
-	rem_time->tv_usec = 0;
-    } else {
-	/* next timeout after ticks ticks */
-	ticks *= CLOCK_RESOLUTION;
-	
-	erts_smp_mtx_lock(&erts_timeofday_mtx);
-	
-	get_tolerant_timeofday(&cur_time);
-	cur_time.tv_usec = 1000 * 
-	    (cur_time.tv_usec / 1000);/* ms resolution*/
-	elapsed = 1000 * (cur_time.tv_sec - last_delivered.tv_sec) +
-	    (cur_time.tv_usec - last_delivered.tv_usec) / 1000;
-	
-	erts_smp_mtx_unlock(&erts_timeofday_mtx);
+    ErtsTimeOffsetMonitorContext *cntxt;
+    Eterm *from_hp, *to_hp;
+    Uint mix;
+    int hix;
+
+    cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt;
+    mix = (cntxt->ix)++;
+    cntxt->to_mon_info[mix].pid = mon->pid;
+    to_hp = &cntxt->to_mon_info[mix].heap[0];
+
+    ASSERT(is_internal_ref(mon->ref));
+    from_hp = internal_ref_val(mon->ref);
+    ASSERT(thing_arityval(*from_hp) + 1 == REF_THING_SIZE);
+
+    for (hix = 0; hix < REF_THING_SIZE; hix++)
+	to_hp[hix] = from_hp[hix];
+
+    cntxt->to_mon_info[mix].ref
+	= make_internal_ref(&cntxt->to_mon_info[mix].heap[0]);
+
+}
+
+static void
+send_time_offset_changed_notifications(void *new_offsetp)
+{
+    ErtsMonotonicTime new_offset;
+    ErtsTimeOffsetMonitorInfo *to_mon_info;
+    Uint no_monitors;
+    char *tmp = NULL;
+
+#ifdef ARCH_64
+    new_offset = (ErtsMonotonicTime) new_offsetp;
+#else
+    new_offset = *((ErtsMonotonicTime *) new_offsetp);
+    erts_free(ERTS_ALC_T_NEW_TIME_OFFSET, new_offsetp);
+#endif
+    new_offset -= ERTS_MONOTONIC_OFFSET_NATIVE;
+
+    erts_smp_mtx_lock(&erts_get_time_mtx);
+
+    no_monitors = no_time_offset_monitors;
+    if (no_monitors) {
+	ErtsTimeOffsetMonitorContext cntxt;
+	Uint alloc_sz;
 	
-	if (ticks <= elapsed) { /* Ooops, better hurry */
-	    rem_time->tv_sec = rem_time->tv_usec = 0;
-	    return;
+	/* Monitor info array size */
+	alloc_sz = no_monitors*sizeof(ErtsTimeOffsetMonitorInfo);
+	/* + template max size */
+	alloc_sz += 6*sizeof(Eterm); /* 5-tuple */
+	alloc_sz += ERTS_MAX_SINT64_HEAP_SIZE*sizeof(Eterm); /* max offset size */
+	tmp = erts_alloc(ERTS_ALC_T_TMP, alloc_sz);
+
+	to_mon_info = (ErtsTimeOffsetMonitorInfo *) tmp;
+	cntxt.ix = 0;
+	cntxt.to_mon_info = to_mon_info;
+
+	erts_doforall_monitors(time_offset_monitors,
+			       save_time_offset_monitor,
+			       &cntxt);
+
+	ASSERT(cntxt.ix == no_monitors);
+    }
+
+    erts_smp_mtx_unlock(&erts_get_time_mtx);
+
+    if (no_monitors) {
+	Eterm *hp, *patch_refp, new_offset_term, message_template;
+	Uint mix, hsz;
+
+	/* Make message template */
+
+	hp = (Eterm *) (tmp + no_monitors*sizeof(ErtsTimeOffsetMonitorInfo));
+
+	hsz = 6; /* 5-tuple */
+	hsz += REF_THING_SIZE;
+	hsz += ERTS_SINT64_HEAP_SIZE(new_offset);
+
+	if (IS_SSMALL(new_offset))
+	    new_offset_term = make_small(new_offset);
+	else
+	    new_offset_term = erts_sint64_to_big(new_offset, &hp);
+	message_template = TUPLE5(hp,
+				  am_CHANGE,
+				  THE_NON_VALUE, /* Patch point for ref */
+				  am_time_offset,
+				  am_clock_service,
+				  new_offset_term);
+	patch_refp = &hp[2];
+
+	ASSERT(*patch_refp == THE_NON_VALUE);
+
+	for (mix = 0; mix < no_monitors; mix++) {
+	    Process *rp = erts_proc_lookup(to_mon_info[mix].pid);
+	    if (rp) {
+		Eterm ref = to_mon_info[mix].ref;
+		ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
+		erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+		if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) {
+		    ErlHeapFragment *bp;
+		    ErlOffHeap *ohp;
+		    Eterm message;
+
+		    hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks);
+		    *patch_refp = ref;
+		    ASSERT(hsz == size_object(message_template));
+		    message = copy_struct(message_template, hsz, &hp, ohp);
+		    erts_queue_message(rp, &rp_locks, bp, message, NIL
+#ifdef USE_VM_PROBES
+				       , NIL
+#endif
+			);
+		}
+		erts_smp_proc_unlock(rp, rp_locks);
+	    }
 	}
-	rem_time->tv_sec = (ticks - elapsed) / 1000;
-	rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000);
+
+	erts_free(ERTS_ALC_T_TMP, tmp);
     }
 }
 
-void erts_get_timeval(SysTimeval *tv)
+static void
+schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset)
 {
-    erts_smp_mtx_lock(&erts_timeofday_mtx);
-    get_tolerant_timeofday(tv);
-    erts_smp_mtx_unlock(&erts_timeofday_mtx);
-    update_approx_time(tv);
+#ifdef ARCH_64
+    void *new_offsetp = (void *) new_offset;
+    ASSERT(sizeof(void *) == sizeof(ErtsMonotonicTime));
+#else
+    void *new_offsetp = erts_alloc(ERTS_ALC_T_NEW_TIME_OFFSET,
+				   sizeof(ErtsMonotonicTime));
+    *((ErtsMonotonicTime *) new_offsetp) = new_offset;
+#endif
+    erts_schedule_misc_aux_work(1,
+				send_time_offset_changed_notifications,
+				new_offsetp);
 }
 
-erts_time_t
-erts_get_time(void)
+static ERTS_INLINE Eterm
+make_time_val(Process *c_p, ErtsMonotonicTime time_val)
 {
-    SysTimeval sys_tv;
-    
-    erts_smp_mtx_lock(&erts_timeofday_mtx);
-    
-    get_tolerant_timeofday(&sys_tv);
-    
-    erts_smp_mtx_unlock(&erts_timeofday_mtx);
+    Sint64 val = (Sint64) time_val;
+    Eterm *hp;
+    Uint sz;
 
-    update_approx_time(&sys_tv);
+    if (IS_SSMALL(val))
+	return make_small(val);
 
-    return sys_tv.tv_sec;
+    sz = ERTS_SINT64_HEAP_SIZE(val);
+    hp = HAlloc(c_p, sz);
+    return erts_sint64_to_big(val, &hp);
 }
 
-#ifdef HAVE_ERTS_NOW_CPU
-void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) {
-  SysCpuTime t;
-  SysTimespec tp;
+Eterm
+erts_get_monotonic_start_time(struct process *c_p)
+{
+    return make_time_val(c_p, ERTS_MONOTONIC_OFFSET_NATIVE);
+}
 
-  sys_get_proc_cputime(t, tp);
-  *microsec = (Uint)(tp.tv_nsec / 1000);
-  t = (tp.tv_sec / 1000000);
-  *megasec = (Uint)(t % 1000000);
-  *sec = (Uint)(tp.tv_sec % 1000000);
+static Eterm
+bld_monotonic_time_source(Uint **hpp, Uint *szp, Sint64 os_mtime)
+{
+#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    return NIL;
+#else
+    int i = 0;
+    Eterm k[5];
+    Eterm v[5];
+
+    if (time_sup.r.o.os_monotonic_disable)
+	return NIL;
+
+    k[i] = erts_bld_atom(hpp, szp, "function");
+    v[i++] = erts_bld_atom(hpp, szp, time_sup.r.o.os_monotonic_func);
+
+    if (time_sup.r.o.os_monotonic_clock_id) {
+	k[i] = erts_bld_atom(hpp, szp, "clock_id");
+	v[i++] = erts_bld_atom(hpp, szp, time_sup.r.o.os_monotonic_clock_id);
+    }
+
+    if (time_sup.r.o.os_monotonic_resolution) {
+	k[i] = erts_bld_atom(hpp, szp, "resolution");
+	v[i++] = erts_bld_uint64(hpp, szp, time_sup.r.o.os_monotonic_resolution);
+    }
+
+    k[i] = erts_bld_atom(hpp, szp, "parallel");
+    v[i++] = time_sup.r.o.os_monotonic_locked ? am_no : am_yes;
+
+    k[i] = erts_bld_atom(hpp, szp, "time");
+    v[i++] = erts_bld_sint64(hpp, szp, os_mtime);
+
+    return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v);
+#endif
 }
+
+Eterm
+erts_monotonic_time_source(struct process *c_p)
+{
+    Uint hsz = 0;
+    Eterm *hp = NULL;
+    Sint64 os_mtime = 0;
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    if (!time_sup.r.o.os_monotonic_disable)
+	os_mtime = (Sint64) erts_os_monotonic_time();
+#endif
+
+    bld_monotonic_time_source(NULL, &hsz, os_mtime);
+    if (hsz)
+	hp = HAlloc(c_p, hsz);
+    return bld_monotonic_time_source(&hp, NULL, os_mtime);
+}
+
+
+#include "bif.h"
+
+static ERTS_INLINE Eterm
+time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonotonicTime muloff)
+{
+    ErtsMonotonicTime result;
+    BIF_RETTYPE ret;
+
+    if (val < 0)
+	goto trap_to_erlang_code;
+
+    /* Convert to common user specified time units */
+    switch (term) {
+    case am_seconds:
+    case make_small(1):
+	result = ERTS_MONOTONIC_TO_SEC(val) + muloff*ERTS_MONOTONIC_OFFSET_SEC;
+	ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result));
+	break;
+    case am_milli_seconds:
+    case make_small(1000):
+	result = ERTS_MONOTONIC_TO_MSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_MSEC;
+	ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result));
+	break;
+    case am_micro_seconds:
+    case make_small(1000*1000):
+	result = ERTS_MONOTONIC_TO_USEC(val) + muloff*ERTS_MONOTONIC_OFFSET_USEC;
+	ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result));
+	break;
+#ifdef ARCH_64
+    case am_nano_seconds:
+    case make_small(1000*1000*1000):
+	result = ERTS_MONOTONIC_TO_NSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_NSEC;
+	ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result));
+	break;
 #endif
+    default: {
+	Eterm value, native_res;
+#ifndef ARCH_64
+	Sint user_res;
+	if (term == am_nano_seconds)
+	    goto to_nano_seconds;
+	if (term_to_Sint(term, &user_res)) {
+	    if (user_res == 1000*1000*1000) {
+	    to_nano_seconds:
+		result = (ERTS_MONOTONIC_TO_NSEC(val)
+			  + muloff*ERTS_MONOTONIC_OFFSET_NSEC);
+		ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result));
+		break;
+	    }
+	    if (user_res <= 0)
+		goto badarg;
+	}
+#else
+	if (is_small(term)) {
+	    if (signed_val(term) <= 0)
+		goto badarg;
+	}
+#endif
+	else if (is_big(term)) {
+	    if (big_sign(term))
+		goto badarg;
+	}
+	else {
+	badarg:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+	    break;
+	}
+
+	trap_to_erlang_code:
+	/* Do it in erlang code instead; pass along values to use... */
+	value = make_time_val(c_p, val + muloff*ERTS_MONOTONIC_OFFSET_NATIVE);
+	native_res = make_time_val(c_p, ERTS_MONOTONIC_TIME_UNIT);
+
+	ERTS_BIF_PREP_TRAP3(ret, erts_convert_time_unit_trap, c_p,
+			    value, native_res, term);
+
+	break;
+    }
+    }
+
+    return ret;
+}
+
+/* Built in functions */
+
+BIF_RETTYPE monotonic_time_0(BIF_ALIST_0)
+{
+    ErtsMonotonicTime mtime = time_sup.r.o.get_time();
+    mtime += ERTS_MONOTONIC_OFFSET_NATIVE;
+    BIF_RET(make_time_val(BIF_P, mtime));
+}
+
+BIF_RETTYPE monotonic_time_1(BIF_ALIST_1)
+{
+    BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, time_sup.r.o.get_time(), 1));
+}
+
+BIF_RETTYPE system_time_0(BIF_ALIST_0)
+{
+    ErtsMonotonicTime mtime, offset;
+    mtime = time_sup.r.o.get_time();
+    offset = get_time_offset();
+    BIF_RET(make_time_val(BIF_P, mtime + offset));
+}
+
+BIF_RETTYPE system_time_1(BIF_ALIST_0)
+{
+    ErtsMonotonicTime mtime, offset;
+    mtime = time_sup.r.o.get_time();
+    offset = get_time_offset();
+    BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime + offset, 0));
+}
+
+BIF_RETTYPE erts_internal_time_unit_0(BIF_ALIST_0)
+{
+    BIF_RET(make_time_val(BIF_P, ERTS_MONOTONIC_TIME_UNIT));
+}
+
+BIF_RETTYPE time_offset_0(BIF_ALIST_0)
+{
+    ErtsMonotonicTime time_offset = get_time_offset();
+    time_offset -= ERTS_MONOTONIC_OFFSET_NATIVE;
+    BIF_RET(make_time_val(BIF_P, time_offset));
+}
+
+BIF_RETTYPE time_offset_1(BIF_ALIST_1)
+{
+    BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, get_time_offset(), -1));
+}
+
+
+BIF_RETTYPE timestamp_0(BIF_ALIST_0)
+{
+    Eterm *hp, res;
+    ErtsMonotonicTime stime, mtime, all_sec, offset;
+    Uint mega_sec, sec, micro_sec;
+
+    mtime = time_sup.r.o.get_time();
+    offset = get_time_offset();
+    stime = ERTS_MONOTONIC_TO_USEC(mtime + offset);
+    all_sec = stime / ERTS_MONOTONIC_TIME_MEGA;
+    mega_sec = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA);
+    sec = (Uint) (all_sec - (((ErtsMonotonicTime) mega_sec)
+			     * ERTS_MONOTONIC_TIME_MEGA));
+    micro_sec = (Uint) (stime - all_sec*ERTS_MONOTONIC_TIME_MEGA);
+
+    ASSERT(((ErtsMonotonicTime) mega_sec)*ERTS_MONOTONIC_TIME_TERA
+	   + ((ErtsMonotonicTime) sec)*ERTS_MONOTONIC_TIME_MEGA
+	   + micro_sec == stime);
+
+    /*
+     * Mega seconds is the only value that potentially
+     * ever could be a bignum. However, that wont happen
+     * during at least the next 4 million years...
+     *
+     * (System time will also have wrapped in the
+     * 64-bit integer before we get there...)
+     */
+
+    ASSERT(IS_USMALL(0, mega_sec));
+    ASSERT(IS_USMALL(0, sec));
+    ASSERT(IS_USMALL(0, micro_sec));
+
+    hp = HAlloc(BIF_P, 4);
+    res = TUPLE3(hp,
+		 make_small(mega_sec),
+		 make_small(sec),
+		 make_small(micro_sec));
+    BIF_RET(res);
+}
+
+BIF_RETTYPE os_system_time_0(BIF_ALIST_0)
+{
+    ErtsMonotonicTime stime;
+    SysTimeval tod;
+    sys_gettimeofday(&tod);
+    stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
+    stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
+    BIF_RET(make_time_val(BIF_P, stime));
+}
+
+BIF_RETTYPE os_system_time_1(BIF_ALIST_0)
+{
+    ErtsMonotonicTime stime;
+    SysTimeval tod;
+    sys_gettimeofday(&tod);
+    stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
+    stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
+    BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0));
+}
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 891046a8b5..e24aef3e3c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -346,8 +346,6 @@ extern Uint display_items;	/* no of items to display in traces etc */
 extern int erts_backtrace_depth;
 extern erts_smp_atomic32_t erts_max_gen_gcs;
 
-extern int erts_disable_tolerant_timeofday;
-
 extern int bif_reductions;      /* reductions + fcalls (when doing call_bif) */
 extern int stackdump_on_exit;
 
@@ -622,9 +620,6 @@ erts_bld_port_info(Eterm **hpp,
 void erts_bif_info_init(void);
 
 /* bif.c */
-Eterm erts_make_ref(Process *);
-Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]);
-void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
 
 ERTS_GLB_INLINE Eterm
 erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 9ae973e108..c0e44b239b 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -47,6 +47,7 @@
 #include "external.h"
 #include "dtrace-wrapper.h"
 #include "erl_map.h"
+#include "erl_bif_unique.h"
 
 extern ErlDrvEntry fd_driver_entry;
 #ifndef __OSE__
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c29d4b3777..7e64623686 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -21,6 +21,25 @@
 #define __SYS_H__
 
 
+#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
+#  undef ERTS_CAN_INLINE
+#  define ERTS_CAN_INLINE 0
+#  undef ERTS_INLINE
+#  define ERTS_INLINE
+#endif
+
+#if ERTS_CAN_INLINE
+#define ERTS_GLB_INLINE static ERTS_INLINE
+#else
+#define ERTS_GLB_INLINE
+#endif
+
+#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
+#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
+#else
+#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
+#endif
+
 #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
 #  define NO_FPE_SIGNALS
 #endif
@@ -132,24 +151,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #  endif
 #endif
 
-#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
-#  undef ERTS_CAN_INLINE
-#  define ERTS_CAN_INLINE 0
-#  undef ERTS_INLINE
-#  define ERTS_INLINE
-#endif
-
-#if ERTS_CAN_INLINE
-#define ERTS_GLB_INLINE static ERTS_INLINE
-#else
-#define ERTS_GLB_INLINE
-#endif
-
-#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
-#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
-#else
-#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
-#endif
+#define ERTS_MK_VSN_INT(Major, Minor, Build) \
+    ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff))
 
 #ifndef ERTS_EXIT_AFTER_DUMP
 #  define ERTS_EXIT_AFTER_DUMP exit
@@ -359,17 +362,45 @@ typedef Sint SWord;
 typedef UWord BeamInstr;
 
 #ifndef HAVE_INT64
-#if SIZEOF_LONG == 8
-#define HAVE_INT64 1
+#  if SIZEOF_LONG == 8
+#    define HAVE_INT64 1
 typedef unsigned long Uint64;
 typedef long          Sint64;
-#elif SIZEOF_LONG_LONG == 8
-#define HAVE_INT64 1
+#    ifdef ULONG_MAX
+#      define ERTS_UINT64_MAX ULONG_MAX
+#    endif
+#    ifdef LONG_MAX
+#      define ERTS_SINT64_MAX LONG_MAX
+#    endif
+#    ifdef LONG_MIN
+#      define ERTS_SINT64_MIN LONG_MIN
+#    endif
+#  elif SIZEOF_LONG_LONG == 8
+#    define HAVE_INT64 1
 typedef unsigned long long Uint64;
 typedef long long          Sint64;
-#else
-#define HAVE_INT64 0
+#    ifdef ULLONG_MAX
+#      define ERTS_UINT64_MAX ULLONG_MAX
+#    endif
+#    ifdef LLONG_MAX
+#      define ERTS_SINT64_MAX LLONG_MAX
+#    endif
+#    ifdef LLONG_MIN
+#      define ERTS_SINT64_MIN LLONG_MIN
+#    endif
+#  else
+#    error "No 64-bit integer type found"
+#  endif
+#endif
+
+#ifndef ERTS_UINT64_MAX
+#  define ERTS_UINT64_MAX (~((Uint64) 0))
 #endif
+#ifndef ERTS_SINT64_MAX
+#  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
+#endif
+#ifndef ERTS_SINT64_MIN
+#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
 #endif
 
 #if SIZEOF_LONG == 4
@@ -646,10 +677,31 @@ extern char *erts_default_arg0;
 
 extern char os_type[];
 
-extern int sys_init_time(void);
+typedef enum {
+    ERTS_NO_TIME_WARP_MODE,
+    ERTS_SINGLE_TIME_WARP_MODE,
+    ERTS_MULTI_TIME_WARP_MODE
+} ErtsTimeWarpMode;
+
+typedef struct {
+    int have_os_monotonic;
+    ErtsMonotonicTime os_monotonic_time_unit;
+    ErtsMonotonicTime sys_clock_resolution;
+    struct {
+	Uint64 resolution;
+	char *func;
+	char *clock_id;
+	int locked_use;
+    } os_monotonic_info;
+} ErtsSysInitTimeResult;
+
+#define ERTS_SYS_INIT_TIME_RESULT_INITER \
+    {0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}
+
+extern void sys_init_time(ErtsSysInitTimeResult *);
 extern void erts_deliver_time(void);
 extern void erts_time_remaining(SysTimeval *);
-extern int erts_init_time_sup(void);
+extern int erts_init_time_sup(int, ErtsTimeWarpMode);
 extern void erts_sys_init_float(void);
 extern void erts_thread_init_float(void);
 extern void erts_thread_disable_fpe(void);
@@ -700,7 +752,7 @@ extern char *erts_sys_ddll_error(int code);
 
 void erts_sys_schedule_interrupt(int set);
 #ifdef ERTS_SMP
-void erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec);
+void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
 void erts_sys_main_thread(void);
 #endif
 
@@ -739,6 +791,7 @@ int univ_to_local(
 int local_to_univ(Sint *year, Sint *month, Sint *day, 
 		  Sint *hour, Sint *minute, Sint *second, int isdst);
 void get_now(Uint*, Uint*, Uint*);
+ErtsMonotonicTime erts_get_monotonic_time(void);
 void get_sys_now(Uint*, Uint*, Uint*);
 void set_break_quit(void (*)(void), void (*)(void));
 
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 2fd8e0cf00..c9f8b68bca 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -83,6 +83,10 @@
 #define ASSERT_NO_LOCKED_LOCKS
 #endif
 
+#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24)
+#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY)
+
+static erts_smp_atomic32_t is_bumping;
 static erts_smp_mtx_t tiw_lock;
 
 
@@ -91,18 +95,20 @@ static erts_smp_mtx_t tiw_lock;
 ** The individual timer cells in tiw are also protected by the same mutex.
 */
 
+/* timing wheel size NEED to be a power of 2 */
 #ifdef SMALL_MEMORY
-#define TIW_SIZE 8192
+#define TIW_SIZE (1 << 13)
 #else
-#define TIW_SIZE 65536		/* timing wheel size (should be a power of 2) */
+#define TIW_SIZE (1 << 20)
 #endif
 static ErlTimer** tiw;		/* the timing wheel, allocated in init_time() */
-static Uint tiw_pos;		/* current position in wheel */
+static ErtsMonotonicTime tiw_pos; /* current position in wheel */
 static Uint tiw_nto;		/* number of timeouts in wheel */
-static Uint tiw_min;
-static ErlTimer *tiw_min_ptr;
-
-/* END tiw_lock protected variables */
+static struct {
+    ErlTimer *head;
+    ErlTimer **tail;
+    Uint nto;
+} tiw_at_once;
 
 /* Actual interval time chosen by sys_init_time() */
 
@@ -114,77 +120,97 @@ static int tiw_itime; /* Constant after init */
 #  define TIW_ITIME tiw_itime
 #endif
 
-erts_smp_atomic32_t do_time;	/* set at clock interrupt */
-static ERTS_INLINE erts_short_time_t do_time_read(void)
-{
-    return erts_smp_atomic32_read_acqb(&do_time);
-}
+static int true_next_timeout_time;
+static ErtsMonotonicTime next_timeout_time;
+erts_atomic64_t erts_next_timeout__;
 
-static ERTS_INLINE erts_short_time_t do_time_update(void)
+static ERTS_INLINE void
+init_next_timeout(ErtsMonotonicTime time)
 {
-    return do_time_read();
+    erts_atomic64_init_nob(&erts_next_timeout__,
+			   (erts_aint64_t) time);
 }
 
-static ERTS_INLINE void do_time_init(void)
+static ERTS_INLINE void
+set_next_timeout(ErtsMonotonicTime time, int true_timeout)
 {
-    erts_smp_atomic32_init_nob(&do_time, 0);
+    true_next_timeout_time = true_timeout;
+    next_timeout_time = time;
+    erts_atomic64_set_relb(&erts_next_timeout__,
+			   (erts_aint64_t) time);
 }
 
 /* get the time (in units of TIW_ITIME) to the next timeout,
    or -1 if there are no timeouts                     */
 
-static erts_short_time_t next_time_internal(void) /* PRE: tiw_lock taken by caller */
+static ERTS_INLINE ErtsMonotonicTime
+find_next_timeout(ErtsMonotonicTime curr_time,
+		  ErtsMonotonicTime max_search_time)
 {
-    int i, tm, nto;
-    Uint32 min;
-    ErlTimer* p;
-    erts_short_time_t dt;
-  
-    if (tiw_nto == 0)
-	return -1;	/* no timeouts in wheel */
+    int start_ix, tiw_pos_ix;
+    ErlTimer *p;
+    int true_min_timeout;
+    ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos, timeout_limit;
 
-    if (tiw_min_ptr) {
-	min = tiw_min;
-	dt  = do_time_read();
-	return ((min >= dt) ? (min - dt) : 0);
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw_lock));
+
+    if (true_next_timeout_time)
+	return next_timeout_time;
+
+    /* We never set next timeout beyond timeout_limit */
+    timeout_limit = curr_time + ERTS_MONOTONIC_DAY;
+
+    if (tiw_nto == 0) { /* no timeouts in wheel */
+	true_min_timeout = true_next_timeout_time = 0;
+	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(timeout_limit);
+	goto found_next;
     }
-  
-    /* start going through wheel to find next timeout */
-    tm = nto = 0;
-    min = (Uint32) -1;	/* max Uint32 */
-    i = tiw_pos;
+
+    /*
+     * Don't want others entering trying to bump
+     * timers while we are checking...
+     */
+    set_next_timeout(timeout_limit, 0);
+
+    true_min_timeout = 1;
+    slot_timeout_pos = tiw_pos;
+    min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
+
+    start_ix = tiw_pos_ix = (int) (tiw_pos & (TIW_SIZE-1));
+
     do {
-	p = tiw[i];
-	while (p != NULL) {
-	    nto++;
-	    if (p->count == 0) {
-		/* found next timeout */
-		dt = do_time_read();
-		/* p->count is zero */
-		tiw_min_ptr = p;
-		tiw_min     = tm;
-		return ((tm >= dt) ? (tm - dt) : 0);
-	    } else {
-		/* keep shortest time in 'min' */
-		if (tm + p->count*TIW_SIZE < min) {
-		    min = tm + p->count*TIW_SIZE;
-		    tiw_min_ptr = p;
-		    tiw_min     = min;
-		}
+	slot_timeout_pos++;
+	if (slot_timeout_pos >= min_timeout_pos) {
+	    true_min_timeout = 0;
+	    break;
+	}
+
+	p = tiw[tiw_pos_ix];
+
+	while (p) {
+	    ErtsMonotonicTime timeout_pos;
+	    ASSERT(p != p->next);
+	    timeout_pos = p->timeout_pos;
+	    if (min_timeout_pos > timeout_pos) {
+		min_timeout_pos = timeout_pos;
+		if (min_timeout_pos <= slot_timeout_pos)
+		    goto found_next;
 	    }
 	    p = p->next;
 	}
-	/* when we have found all timeouts the shortest time will be in min */
-	if (nto == tiw_nto) break;
-	tm++;
-	i = (i + 1) % TIW_SIZE;
-    } while (i != tiw_pos);
-    dt = do_time_read();
-    if (min <= (Uint32) dt)
-	return 0;
-    if ((min - (Uint32) dt) > (Uint32) ERTS_SHORT_TIME_T_MAX)
-	return ERTS_SHORT_TIME_T_MAX;
-    return (erts_short_time_t) (min - (Uint32) dt);
+
+	tiw_pos_ix++;
+	if (tiw_pos_ix == TIW_SIZE)
+	    tiw_pos_ix = 0;
+    } while (start_ix != tiw_pos_ix);
+
+found_next:
+
+    min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos);
+    if (min_timeout != next_timeout_time)
+	set_next_timeout(min_timeout, true_min_timeout);
+
+    return min_timeout;
 }
 
 static void remove_timer(ErlTimer *p) {
@@ -212,72 +238,153 @@ static void remove_timer(ErlTimer *p) {
     tiw_nto--;
 }
 
-/* Private export to erl_time_sup.c */
-erts_short_time_t erts_next_time(void)
+ErtsMonotonicTime
+erts_check_next_timeout_time(ErtsMonotonicTime max_search_time)
 {
-    erts_short_time_t ret;
+    ErtsMonotonicTime next, curr;
+
+    curr = erts_get_monotonic_time();
 
     erts_smp_mtx_lock(&tiw_lock);
-    (void)do_time_update();
-    ret = next_time_internal();
+
+    next = find_next_timeout(curr, max_search_time);
+
     erts_smp_mtx_unlock(&tiw_lock);
-    return ret;
+
+    return next;
 }
 
-static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lock is write-locked */
+#ifndef DEBUG
+#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TO) ((void) 0)
+#else
+#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TO) debug_check_safe_to_skip_to((TO))
+static void
+debug_check_safe_to_skip_to(ErtsMonotonicTime skip_to_pos)
 {
-    Uint keep_pos;
-    Uint count;
-    ErlTimer *p, **prev, *timeout_head, **timeout_tail;
-    Uint dtime = (Uint) dt;  
+    int slots, ix;
+    ErlTimer *tmr;
+    ErtsMonotonicTime tmp;
+
+    ix = (int) (tiw_pos & (TIW_SIZE-1));
+    tmp = skip_to_pos - tiw_pos;
+    ASSERT(tmp >= 0);
+    if (tmp < (ErtsMonotonicTime) TIW_SIZE)
+	slots = (int) tmp;
+    else
+	slots = TIW_SIZE;
+
+     while (slots > 0) {
+	tmr = tiw[ix];
+	while (tmr) {
+	    ASSERT(tmr->timeout_pos > skip_to_pos);
+	    tmr = tmr->next;
+	}
+	ix++;
+	if (ix == TIW_SIZE)
+	    ix = 0;
+	slots--;
+    }
+}
+#endif
+
+void erts_bump_timers(ErtsMonotonicTime curr_time)
+{
+    int tiw_pos_ix, slots;
+    ErlTimer *p, *timeout_head, **timeout_tail;
+    ErtsMonotonicTime bump_to, tmp_slots;
+
+    if (erts_smp_atomic32_cmpxchg_nob(&is_bumping, 1, 0) != 0)
+	return; /* Another thread is currently bumping... */
+
+    bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
+
+    erts_smp_mtx_lock(&tiw_lock);
+
+    if (tiw_pos >= bump_to) {
+	timeout_head = NULL;
+	goto done;
+    }
+
+    /* Don't want others here while we are bumping... */
+    set_next_timeout(curr_time + ERTS_MONOTONIC_DAY, 0);
+
+    if (!tiw_at_once.head) {
+	timeout_head = NULL;
+	timeout_tail = &timeout_head;
+    }
+    else {
+	ASSERT(tiw_nto >= tiw_at_once.nto);
+	timeout_head = tiw_at_once.head;
+	timeout_tail = tiw_at_once.tail;
+	tiw_nto -= tiw_at_once.nto;
+	tiw_at_once.head = NULL;
+	tiw_at_once.tail = &tiw_at_once.head;
+	tiw_at_once.nto = 0;
+    }
 
-    /* no need to bump the position if there aren't any timeouts */
     if (tiw_nto == 0) {
-	erts_smp_mtx_unlock(&tiw_lock);
-	return;
+	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(bump_to);
+	tiw_pos = bump_to;
+	goto done;
     }
 
-    /* if do_time > TIW_SIZE we want to go around just once */
-    count = (Uint)(dtime / TIW_SIZE) + 1;
-    keep_pos = (tiw_pos + dtime) % TIW_SIZE;
-    if (dtime > TIW_SIZE) dtime = TIW_SIZE;
-  
-    timeout_head = NULL;
-    timeout_tail = &timeout_head;
-    while (dtime > 0) {
-	/* this is to decrease the counters with the right amount */
-	/* when dtime >= TIW_SIZE */
-	if (tiw_pos == keep_pos) count--;
-	prev = &tiw[tiw_pos];
-	while ((p = *prev) != NULL) {
-	    ASSERT( p != p->next);
-	    if (p->count < count) {     /* we have a timeout */
-		/* remove min time */
-		if (tiw_min_ptr == p) {
-		    tiw_min_ptr = NULL;
-		    tiw_min = 0;
-		}
+    if (true_next_timeout_time) {
+	ErtsMonotonicTime skip_until_pos;
+	/*
+	 * No need inspecting slots where we know no timeouts
+	 * to trigger should reside.
+	 */
 
+	skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(next_timeout_time);
+	if (skip_until_pos > bump_to)
+	    skip_until_pos = bump_to;
+
+	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(skip_until_pos);
+	ASSERT(skip_until_pos > tiw_pos);
+
+	tiw_pos = skip_until_pos - 1;
+    }
+
+    tiw_pos_ix = (int) ((tiw_pos+1) & (TIW_SIZE-1));
+    tmp_slots = (bump_to - tiw_pos);
+    if (tmp_slots < (ErtsMonotonicTime) TIW_SIZE)
+	slots = (int) tmp_slots;
+    else
+	slots = TIW_SIZE;
+ 
+    while (slots > 0) {
+	p = tiw[tiw_pos_ix];
+	while (p) {
+	    ErlTimer *next = p->next;
+	    ASSERT(p != next);
+	    if (p->timeout_pos <= bump_to) { /* we have a timeout */
 		/* Remove from list */
 		remove_timer(p);
 		*timeout_tail = p;	/* Insert in timeout queue */
 		timeout_tail = &p->next;
 	    }
-	    else {
-		/* no timeout, just decrease counter */
-		p->count -= count;
-		prev = &p->next;
-	    }
+	    p = next;
 	}
-	tiw_pos = (tiw_pos + 1) % TIW_SIZE;
-	dtime--;
+	tiw_pos_ix++;
+	if (tiw_pos_ix == TIW_SIZE)
+	    tiw_pos_ix = 0;
+	slots--;
     }
-    tiw_pos = keep_pos;
-    if (tiw_min_ptr)
-	tiw_min -= dt;
-    
+
+    ASSERT(tmp_slots >= (ErtsMonotonicTime) TIW_SIZE
+	   || tiw_pos_ix == (int) ((bump_to+1) & (TIW_SIZE-1)));
+
+    tiw_pos = bump_to;
+
+    /* Search at most two seconds ahead... */
+    (void) find_next_timeout(curr_time, ERTS_SEC_TO_MONOTONIC(2));
+
+done:
+
     erts_smp_mtx_unlock(&tiw_lock);
     
+    erts_smp_atomic32_set_nob(&is_bumping, 0);
+
     /* Call timedout timers callbacks */
     while (timeout_head) {
 	p = timeout_head;
@@ -288,6 +395,7 @@ static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lo
 	 * accesses any field until the ->timeout
 	 * callback is called.
 	 */
+	ASSERT(p->timeout_pos <= bump_to);
 	p->next = NULL;
 	p->prev = NULL;
 	p->slot = 0;
@@ -295,12 +403,6 @@ static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lo
     }
 }
 
-void erts_bump_timer(erts_short_time_t dt) /* dt is value from do_time */
-{
-    erts_smp_mtx_lock(&tiw_lock);
-    bump_timer_internal(dt);
-}
-
 Uint
 erts_timer_wheel_memory_size(void)
 {
@@ -310,13 +412,14 @@ erts_timer_wheel_memory_size(void)
 /* this routine links the time cells into a free list at the start
    and sets the time queue as empty */
 void
-erts_init_time(void)
+erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
 {
+    ErtsMonotonicTime mtime;
     int i, itime;
 
     /* system dependent init; must be done before do_time_init()
        if timer thread is enabled */
-    itime = erts_init_time_sup();
+    itime = erts_init_time_sup(time_correction, time_warp_mode);
 #ifdef TIW_ITIME_IS_CONSTANT 
     if (itime != TIW_ITIME) {
 	erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME);
@@ -325,16 +428,23 @@ erts_init_time(void)
     tiw_itime = itime;
 #endif
 
+    erts_smp_atomic32_init_nob(&is_bumping, 0);
     erts_smp_mtx_init(&tiw_lock, "timer_wheel");
 
     tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
 				  TIW_SIZE * sizeof(ErlTimer*));
     for(i = 0; i < TIW_SIZE; i++)
 	tiw[i] = NULL;
-    do_time_init();
-    tiw_pos = tiw_nto = 0;
-    tiw_min_ptr = NULL;
-    tiw_min = 0;
+
+    mtime = erts_get_monotonic_time();
+    tiw_pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
+    tiw_nto = 0;
+    tiw_at_once.head = NULL;
+    tiw_at_once.tail = &tiw_at_once.head;
+    tiw_at_once.nto = 0;
+    init_next_timeout(mtime + ERTS_MONOTONIC_DAY);
+
+    erts_late_init_time_sup();
 }
 
 
@@ -343,58 +453,62 @@ erts_init_time(void)
 /*
 ** Insert a process into the time queue, with a timeout 't'
 */
-static void
-insert_timer(ErlTimer* p, Uint t)
+static ErtsMonotonicTime
+insert_timer(ErlTimer* p, ErtsMonotonicTime curr_time, ErtsMonotonicTime to)
 {
-    Uint tm;
-    Uint64 ticks;
+    ErtsMonotonicTime timeout_time, timeout_pos;
 
-    /* The current slot (tiw_pos) in timing wheel is the next slot to be
-     * be processed. Hence no extra time tick is needed.
-     *
-     * (x + y - 1)/y is precisely the "number of bins" formula.
-     */
-    ticks = (t + (TIW_ITIME - 1)) / TIW_ITIME;
+    if (to == 0) {
+	timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
+	tiw_nto++;
+	tiw_at_once.nto++;
+	*tiw_at_once.tail = p;
+	p->next = NULL;
+	p->timeout_pos = timeout_pos;
+	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+    }
+    else {
+	int tm;
+	ErtsMonotonicTime ticks;
 
-    /* 
-     * Ticks must be a Uint64, or the addition may overflow here,
-     * resulting in an incorrect value for p->count below.
-     */
-    ticks += do_time_update(); /* Add backlog of unprocessed time */
-    
-    /* calculate slot */
-    tm = (ticks + tiw_pos) % TIW_SIZE;
-    p->slot = (Uint) tm;
-    p->count = (Uint) (ticks / TIW_SIZE);
+	ticks = ERTS_MSEC_TO_CLKTCKS(to);
+	timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time - 1) + 1 + ticks;
+
+	/* calculate slot */
+	tm = (int) (timeout_pos & (TIW_SIZE-1));
+	p->slot = (Uint) tm;
   
-    /* insert at head of list at slot */
-    p->next = tiw[tm];
-    p->prev = NULL;
-    if (p->next != NULL)
-	p->next->prev = p;
-    tiw[tm] = p;
+	/* insert at head of list at slot */
+	p->next = tiw[tm];
+	p->prev = NULL;
+	if (p->next != NULL)
+	    p->next->prev = p;
+	tiw[tm] = p;
 
+	tiw_nto++;
 
-    /* insert min time */
-    if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) {
-	tiw_min = ticks;
-	tiw_min_ptr = p;
-    }
-    if ((tiw_min_ptr == p) && (ticks > tiw_min)) {
-	/* some other timer might be 'min' now */
-	tiw_min = 0;
-	tiw_min_ptr = NULL;
+	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+	p->timeout_pos = timeout_pos;
+
+	ASSERT(ERTS_MSEC_TO_MONOTONIC(to) <= timeout_time - curr_time);
+	ASSERT(timeout_time - curr_time
+	       < ERTS_MSEC_TO_MONOTONIC(to) + ERTS_CLKTCKS_TO_MONOTONIC(1));
     }
 
-    tiw_nto++;
+    if (timeout_time < next_timeout_time)
+	set_next_timeout(timeout_time, 1);
+
+    return timeout_time;
 }
 
 void
 erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
 	      void* arg, Uint t)
 {
-
-    erts_deliver_time();
+#ifdef ERTS_SMP
+    ErtsMonotonicTime timeout_time;
+#endif
+    ErtsMonotonicTime current_time = erts_get_monotonic_time();
     erts_smp_mtx_lock(&tiw_lock);
     if (p->active) { /* XXX assert ? */
 	erts_smp_mtx_unlock(&tiw_lock);
@@ -404,11 +518,15 @@ erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
     p->cancel = cancel;
     p->arg = arg;
     p->active = 1;
-    insert_timer(p, t);
+#ifdef ERTS_SMP
+    timeout_time =
+#else
+	(void)
+#endif
+	insert_timer(p, current_time, (ErtsMonotonicTime) t);
     erts_smp_mtx_unlock(&tiw_lock);
 #if defined(ERTS_SMP)
-    if (t <= (Uint) ERTS_SHORT_TIME_T_MAX)
-	erts_sys_schedule_interrupt_timed(1, (erts_short_time_t) t);
+    erts_sys_schedule_interrupt_timed(1, timeout_time);
 #endif
 }
 
@@ -421,14 +539,8 @@ erts_cancel_timer(ErlTimer* p)
 	return;
     }
 
-    /* is it the 'min' timer, remove min */
-    if (p == tiw_min_ptr) {
-	tiw_min_ptr = NULL;
-	tiw_min     = 0;
-    }
-
     remove_timer(p);
-    p->slot = p->count = 0;
+    p->slot = 0;
 
     if (p->cancel != NULL) {
 	erts_smp_mtx_unlock(&tiw_lock);
@@ -447,8 +559,7 @@ erts_cancel_timer(ErlTimer* p)
 Uint
 erts_time_left(ErlTimer *p)
 {
-    Uint left;
-    erts_short_time_t dt;
+    ErtsMonotonicTime current_time, timeout_time;
 
     erts_smp_mtx_lock(&tiw_lock);
 
@@ -457,45 +568,43 @@ erts_time_left(ErlTimer *p)
 	return 0;
     }
 
-    if (p->slot < tiw_pos)
-	left = (p->count + 1) * TIW_SIZE + p->slot - tiw_pos;
-    else
-	left = p->count * TIW_SIZE + p->slot - tiw_pos;
-    dt = do_time_read();
-    if (left < dt)
-	left = 0;
-    else
-	left -= dt;
+    timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos);
 
     erts_smp_mtx_unlock(&tiw_lock);
 
-    return (Uint) left * TIW_ITIME;
+    current_time = erts_get_monotonic_time();
+    if (timeout_time <= current_time)
+	return 0;
+    return (Uint) ERTS_MONOTONIC_TO_MSEC(timeout_time - current_time);
 }
 
 #ifdef DEBUG
 void erts_p_slpq(void)
 {
+    ErtsMonotonicTime current_time = erts_get_monotonic_time();
     int i;
     ErlTimer* p;
   
     erts_smp_mtx_lock(&tiw_lock);
 
     /* print the whole wheel, starting at the current position */
-    erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto);
+    erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n",
+		current_time, tiw_pos, tiw_nto);
     i = tiw_pos;
     if (tiw[i] != NULL) {
 	erts_printf("%d:\n", i);
 	for(p = tiw[i]; p != NULL; p = p->next) {
-	    erts_printf(" (count %d, slot %d)\n",
-			p->count, p->slot);
+	    erts_printf(" (timeout time %bps, slot %d)\n",
+			ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos),
+			p->slot);
 	}
     }
-    for(i = (i+1)%TIW_SIZE; i != tiw_pos; i = (i+1)%TIW_SIZE) {
+    for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw_pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) {
 	if (tiw[i] != NULL) {
 	    erts_printf("%d:\n", i);
 	    for(p = tiw[i]; p != NULL; p = p->next) {
-		erts_printf(" (count %d, slot %d)\n",
-			    p->count, p->slot);
+		erts_printf(" (timeout time %bps, slot %d)\n",
+			    ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot);
 	    }
 	}
     }
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index c505c44905..54f1a122c4 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -49,6 +49,7 @@
 #include "beam_bp.h"
 #include "erl_ptab.h"
 #include "erl_check_io.h"
+#include "erl_bif_unique.h"
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
@@ -4353,8 +4354,8 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
  */
 Uint64 erts_timestamp_millis(void)
 {
-#ifdef HAVE_GETHRTIME
-    return (Uint64) (sys_gethrtime() / 1000000);
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    return ERTS_MONOTONIC_TO_MSEC(erts_os_monotonic_time());
 #else
     Uint64 res;
     SysTimeval tv;
-- 
cgit v1.2.3


From fa7b2c00cbf9212c4a3551980939b92fc6606510 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Feb 2015 03:18:16 +0100
Subject: Implement ethread events with timeout

---
 erts/emulator/beam/erl_process.c |  6 +++---
 erts/emulator/beam/erl_threads.h | 23 +++++++++++++++++++++++
 2 files changed, 26 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 81bc3d2429..9dcaf2fdca 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2806,7 +2806,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
 			do {
-			    res = erts_tse_wait(ssi->event);
+			    res = erts_tse_twait(ssi->event, -1);
 			} while (res == EINTR);
 		    }
 		}
@@ -6594,7 +6594,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			    int res;
 
 			    do {
-				res = erts_tse_wait(ssi->event);
+				res = erts_tse_twait(ssi->event, -1);
 			    } while (res == EINTR);
 			}
 		    }
@@ -6800,7 +6800,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			    int res;
 
 			    do {
-				res = erts_tse_wait(ssi->event);
+				res = erts_tse_twait(ssi->event, -1);
 			    } while (res == EINTR);
 			}
 		    }
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 7214f3ea33..b9f34d4c12 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -651,6 +651,8 @@ ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
 ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
 ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
 ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
+ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo);
+ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo);
 ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
 ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
 ERTS_GLB_INLINE int erts_thr_get_main_status(void);
@@ -3473,6 +3475,27 @@ ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
 #endif
 }
 
+ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
+{
+#ifdef USE_THREADS
+    return ethr_event_twait(&((ethr_ts_event *) ep)->event,
+			    (ethr_sint64_t) tmo);
+#else
+    return ENOTSUP;
+#endif
+}
+
+ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
+{
+#ifdef USE_THREADS
+    return ethr_event_stwait(&((ethr_ts_event *) ep)->event,
+			     spincount,
+			     (ethr_sint64_t) tmo);
+#else
+    return ENOTSUP;
+#endif
+}
+
 ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-- 
cgit v1.2.3


From 6b1921d767de5cd1a980234f83b36dbfa13d9fc7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Feb 2015 00:25:57 +0100
Subject: Erlang based BIF timer implementation for scalability

---
 erts/emulator/beam/atom.names      |   1 +
 erts/emulator/beam/bif.c           |  35 +++++++--
 erts/emulator/beam/bif.tab         |   5 --
 erts/emulator/beam/erl_alloc.types |   2 +
 erts/emulator/beam/erl_bif_info.c  |  21 ++++-
 erts/emulator/beam/erl_bif_timer.c | 157 +++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_bif_timer.h |   1 +
 erts/emulator/beam/erl_db_hash.c   |  51 +++++++++++-
 erts/emulator/beam/erl_gc.c        |  71 +++++++++--------
 erts/emulator/beam/erl_init.c      |  14 +++-
 erts/emulator/beam/erl_message.c   |   2 +-
 erts/emulator/beam/erl_message.h   |  14 +++-
 erts/emulator/beam/erl_process.c   |  36 ++++++++-
 erts/emulator/beam/erl_process.h   |  22 +++++-
 erts/emulator/beam/erl_time_sup.c  |   1 +
 erts/emulator/beam/global.h        |   8 +-
 16 files changed, 375 insertions(+), 66 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index ced35be265..e2760c18b4 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -109,6 +109,7 @@ atom bag
 atom band
 atom big
 atom bif_return_trap
+atom bif_timer_server
 atom binary
 atom binary_bin_to_list_trap
 atom binary_copy_trap
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ec5122292e..a48f65d3a2 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -615,14 +615,12 @@ erts_queue_monitor_message(Process *p,
 }
 
 static BIF_RETTYPE
-local_pid_monitor(Process *p, Eterm target)
+local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool)
 {
     BIF_RETTYPE ret;
-    Eterm mon_ref;
     Process *rp;
     ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK;
 
-    mon_ref = erts_make_ref(p);
     ERTS_BIF_PREP_RET(ret, mon_ref);
     if (target == p->common.id) {
 	return ret;
@@ -635,12 +633,18 @@ local_pid_monitor(Process *p, Eterm target)
     if (!rp) {
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	p_locks &= ~ERTS_PROC_LOCK_LINK;
-	erts_queue_monitor_message(p, &p_locks,
-				   mon_ref, am_process, target, am_noproc);
+	if (bool)
+	    ret = am_false;
+	else
+	    erts_queue_monitor_message(p, &p_locks,
+				       mon_ref, am_process, target, am_noproc);
     }
     else {
 	ASSERT(rp != p);
 
+	if (bool)
+	    ret = am_true;
+
 	erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL);
 	erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL);
 
@@ -785,7 +789,7 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
 
     if (is_internal_pid(target)) {
     local_pid:
-	ret = local_pid_monitor(BIF_P, target);
+	ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0);
     } else if (is_external_pid(target)) {
 	dep = external_pid_dist_entry(target);
 	if (dep == erts_this_dist_entry)
@@ -828,6 +832,25 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
     return ret;
 }
 
+BIF_RETTYPE erts_internal_monitor_process_2(BIF_ALIST_2)
+{
+    if (is_not_internal_pid(BIF_ARG_1)) {
+	if (is_external_pid(BIF_ARG_1)
+	    && (external_pid_dist_entry(BIF_ARG_1)
+		== erts_this_dist_entry)) {
+	    BIF_RET(am_false);
+	}
+	goto badarg;
+    }
+
+    if (is_not_internal_ref(BIF_ARG_2))
+	goto badarg;
+
+    BIF_RET(local_pid_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, 1));
+
+badarg:
+    BIF_ERROR(BIF_P, BADARG);
+}
 
 /**********************************************************************/
 /* this is a combination of the spawn and link BIFs */
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index db8feb681b..8ac1fba733 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -217,11 +217,6 @@ bif math:sqrt/1
 bif math:atan2/2
 bif math:pow/2
 
-bif erlang:start_timer/3
-bif erlang:send_after/3
-bif erlang:cancel_timer/1
-bif erlang:read_timer/1
-
 bif erlang:make_tuple/2
 bif erlang:append_element/2
 bif erlang:make_tuple/3
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 4cd4ad100c..43279715b8 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -365,6 +365,7 @@ type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	STANDARD_LOW	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD_LOW	SYSTEM		port_data_heap
+type	BIF_TIMER_DATA	LONG_LIVED_LOW	SYSTEM		bif_timer_data
 
 +else # "fullword"
 
@@ -385,6 +386,7 @@ type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
+type	BIF_TIMER_DATA	LONG_LIVED	SYSTEM		bif_timer_data
 
 +endif
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 80d49c7ce2..70062b2305 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -538,6 +538,7 @@ pi_locks(Eterm info)
     switch (info) {
     case am_status:
     case am_priority:
+    case am_trap_exit:
 	return ERTS_PROC_LOCK_STATUS;
     case am_links:
     case am_monitors:
@@ -590,7 +591,7 @@ static Eterm pi_args[] = {
     am_min_bin_vheap_size,
     am_current_location,
     am_current_stacktrace,
-};    
+};
 
 #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
 
@@ -3701,6 +3702,24 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1)
+{
+    if (is_internal_pid(BIF_ARG_1)) {
+	Process *rp = erts_proc_lookup(BIF_ARG_1);
+	if (rp && (rp->static_flags & ERTS_STC_FLG_SYSTEM_PROC))
+	    BIF_RET(am_true);
+	BIF_RET(am_false);
+    }
+
+    if (is_external_pid(BIF_ARG_1)
+	&& external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) {
+	BIF_RET(am_false);
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+
 static erts_smp_atomic_t hipe_test_reschedule_flag;
 
 
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index c9b02b48f5..8b444f2b01 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -490,8 +490,9 @@ setup_bif_timer(Uint32 xflags,
     return ref;
 }
 
+BIF_RETTYPE old_send_after_3(BIF_ALIST_3);
 /* send_after(Time, Pid, Message) -> Ref */
-BIF_RETTYPE send_after_3(BIF_ALIST_3)
+BIF_RETTYPE old_send_after_3(BIF_ALIST_3)
 {
     Eterm res;
 
@@ -511,8 +512,9 @@ BIF_RETTYPE send_after_3(BIF_ALIST_3)
     }
 }
 
+BIF_RETTYPE old_start_timer_3(BIF_ALIST_3);
 /* start_timer(Time, Pid, Message) -> Ref */
-BIF_RETTYPE start_timer_3(BIF_ALIST_3)
+BIF_RETTYPE old_start_timer_3(BIF_ALIST_3)
 {
     Eterm res;
 
@@ -532,8 +534,9 @@ BIF_RETTYPE start_timer_3(BIF_ALIST_3)
     }
 }
 
+BIF_RETTYPE old_cancel_timer_1(BIF_ALIST_1);
 /* cancel_timer(Ref) -> false | RemainingTime */
-BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
+BIF_RETTYPE old_cancel_timer_1(BIF_ALIST_1)
 {
     Eterm res;
     ErtsBifTimer *btm;
@@ -570,8 +573,9 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
     BIF_RET(res);
 }
 
+BIF_RETTYPE old_read_timer_1(BIF_ALIST_1);
 /* read_timer(Ref) -> false | RemainingTime */
-BIF_RETTYPE read_timer_1(BIF_ALIST_1)
+BIF_RETTYPE old_read_timer_1(BIF_ALIST_1)
 {
     Eterm res;
     ErtsBifTimer *btm;
@@ -653,7 +657,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks)
     erts_smp_btm_rwunlock();
 }
 
-void erts_bif_timer_init(void)
+static void erts_old_bif_timer_init(void)
 {
     int i;
     no_bif_timers = 0;
@@ -704,3 +708,146 @@ erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *),
 	}
     }
 }
+
+typedef struct {
+    Uint ref_heap[REF_THING_SIZE];
+    Eterm pid[1];
+} ErtsBifTimerServers;
+
+static ErtsBifTimerServers *bif_timer_servers;
+
+void erts_bif_timer_init(void)
+{
+    erts_old_bif_timer_init();
+}
+
+void
+erts_bif_timer_start_servers(Eterm parent)
+{
+    Process *parent_proc;
+    Eterm *hp, btr_ref, arg_list_end;
+    ErlSpawnOpts so;
+    int i;
+
+    bif_timer_servers = erts_alloc(ERTS_ALC_T_BIF_TIMER_DATA,
+				   (sizeof(ErtsBifTimerServers)
+				    + (sizeof(Eterm)*(erts_no_schedulers-1))));
+
+    so.flags = SPO_USE_ARGS|SPO_SYSTEM_PROC|SPO_PREFER_SCHED|SPO_OFF_HEAP_MSGS;
+    so.min_heap_size  = H_MIN_SIZE;
+    so.min_vheap_size = BIN_VH_MIN_SIZE;
+    so.priority       = PRIORITY_MAX;
+    so.max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
+
+    /*
+     * Parent is "init" and schedulers have not yet been started, so it
+     * *should* be alive and well...
+     */
+    ASSERT(is_internal_pid(parent));
+    parent_proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
+							internal_pid_index(parent));
+    ASSERT(parent_proc);
+    ASSERT(parent_proc->common.id == parent);
+    ASSERT(!ERTS_PROC_IS_EXITING(parent_proc));
+
+    erts_smp_proc_lock(parent_proc, ERTS_PROC_LOCK_MAIN);
+
+    hp = HAlloc(parent_proc, 2*erts_no_schedulers + 2 + REF_THING_SIZE);
+
+    btr_ref = erts_make_ref_in_buffer(hp);
+    hp += REF_THING_SIZE;
+
+    arg_list_end = CONS(hp, btr_ref, NIL);
+    hp += 2;
+
+    for (i = 0; i < erts_no_schedulers; i++) {
+	int sched = i+1;
+	Eterm arg_list = CONS(hp, make_small(i+1), arg_list_end);
+	hp += 2;
+
+	so.scheduler = sched; /* Preferred scheduler */
+
+	bif_timer_servers->pid[i] = erl_create_process(parent_proc,
+						       am_erts_internal,
+						       am_bif_timer_server,
+						       arg_list,
+						       &so);
+    }
+
+    erts_smp_proc_unlock(parent_proc, ERTS_PROC_LOCK_MAIN);
+
+    hp = internal_ref_val(btr_ref);
+    for (i = 0; i < REF_THING_SIZE; i++)
+	bif_timer_servers->ref_heap[i] = hp[i];
+}
+
+BIF_RETTYPE
+erts_internal_get_bif_timer_servers_0(BIF_ALIST_0)
+{
+    int i;
+    Eterm *hp, res = NIL;
+
+    hp = HAlloc(BIF_P, erts_no_schedulers*2);
+    for (i = erts_no_schedulers-1; i >= 0; i--) {
+	res = CONS(hp, bif_timer_servers->pid[i], res);
+	hp += 2;
+    }
+    BIF_RET(res);
+}
+
+BIF_RETTYPE
+erts_internal_access_bif_timer_1(BIF_ALIST_1)
+{
+    int ix;
+    Uint32 *rdp;
+    Eterm ref, pid, *hp, res;
+
+    if (is_not_internal_ref(BIF_ARG_1)) {
+	if (is_not_ref(BIF_ARG_1))
+	    BIF_ERROR(BIF_P, BADARG);
+	BIF_RET(am_undefined);
+    }
+
+    rdp = internal_ref_numbers(BIF_ARG_1);
+    ix = (int) erts_get_ref_numbers_thr_id(rdp);
+    if (ix < 1 || erts_no_schedulers < ix)
+	BIF_RET(am_undefined);
+
+    pid = bif_timer_servers->pid[ix-1];
+    ASSERT(is_internal_pid(pid));
+
+    hp = HAlloc(BIF_P, 3 /* 2-tuple */ + REF_THING_SIZE);
+    for (ix = 0; ix < REF_THING_SIZE; ix++)
+	hp[ix] = bif_timer_servers->ref_heap[ix];
+    ref = make_internal_ref(&hp[0]);
+    hp += REF_THING_SIZE;
+
+    res = TUPLE2(hp, ref, pid);
+    BIF_RET(res);
+}
+
+BIF_RETTYPE
+erts_internal_create_bif_timer_0(BIF_ALIST_0)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(BIF_P);
+    Eterm *hp, btr_ref, t_ref, pid, res;
+    int ix;
+
+    hp = HAlloc(BIF_P, 4 /* 3-tuple */ + 2*REF_THING_SIZE);
+    for (ix = 0; ix < REF_THING_SIZE; ix++)
+	hp[ix] = bif_timer_servers->ref_heap[ix];
+    btr_ref = make_internal_ref(&hp[0]);
+    hp += REF_THING_SIZE;
+
+    t_ref = erts_sched_make_ref_in_buffer(esdp, hp);
+    hp += REF_THING_SIZE;
+
+    ASSERT(erts_get_ref_numbers_thr_id(internal_ref_numbers(t_ref))
+	   == (Uint32) esdp->no);
+
+    pid = bif_timer_servers->pid[((int) esdp->no) - 1];
+    
+    res = TUPLE3(hp, btr_ref, pid, t_ref);
+
+    BIF_RET(res);
+}
diff --git a/erts/emulator/beam/erl_bif_timer.h b/erts/emulator/beam/erl_bif_timer.h
index 1197c176f5..c2f5dfd3c3 100644
--- a/erts/emulator/beam/erl_bif_timer.h
+++ b/erts/emulator/beam/erl_bif_timer.h
@@ -33,4 +33,5 @@ void erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks);
 void erts_bif_timer_init(void);
 void erts_bif_timer_foreach(void (*func)(Eterm,Eterm,ErlHeapFragment *,void *),
 			    void *arg);
+void erts_bif_timer_start_servers(Eterm);
 #endif
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 06dac8f161..063808eb79 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -171,10 +171,53 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
 #define MAX_HASH 0xEFFFFFFFUL
 #define INVALID_HASH 0xFFFFFFFFUL
 
-/* optimised version of make_hash (normal case? atomic key) */
-#define MAKE_HASH(term) \
-    ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
-      make_hash2(term)) % MAX_HASH)
+static ERTS_INLINE HashValue
+MAKE_HASH(Eterm term)
+{
+    if (is_atom(term)) {
+	/*
+	 * optimised version of make_hash, although poor hashvalue
+	 * (normal case? atomic key)
+	 */
+	return atom_tab(atom_val(term))->slot.bucket.hvalue;
+    }
+    if (is_ref(term)) {
+	/*
+	 * make_hash2() produce poor hash values
+	 * for refs.
+	 */
+	int no;
+	Uint32 *ref;
+	HashValue hval;
+	if (is_internal_ref(term)) {
+	    no = (int) internal_ref_no_of_numbers(term);
+	    ref = internal_ref_numbers(term);
+	}
+	else {
+	    no = (int) external_ref_no_of_numbers(term);
+	    ref = external_ref_numbers(term);
+	}
+	switch (no) {
+	case 3:
+	ref_limit:
+	    if (!ref[2])
+		no = 2;
+	case 2:
+	    if (!ref[1])
+		no = 1;
+	case 1:
+	    break;
+	default:
+	    no = 3;
+	    goto ref_limit;
+	}
+	hval = (HashValue) block_hash((byte *) ref,
+				      no * sizeof(Uint32),
+				      0x08d12e65);
+	return hval % MAX_HASH;
+    }
+    return make_hash2(term) % MAX_HASH;
+}
 
 #ifdef ERTS_SMP
 #  define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1)
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index fea9b16e90..9a05e5b23a 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -95,10 +95,10 @@ typedef struct {
 
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
-static Uint combined_message_size(Process* p);
+static Uint combined_message_size(Process* p, int off_heap_msgs);
 static void remove_message_buffers(Process* p);
-static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
-static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
+static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs);
+static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs);
 static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
 static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size);
 static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size);
@@ -401,7 +401,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
     Uint reclaimed_now = 0;
     int done = 0;
+    int off_heap_msgs;
     Uint ms1, s1, us1;
+    erts_aint32_t state;
     ErtsSchedulerData *esdp;
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
@@ -418,7 +420,8 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
         trace_gc(p, am_gc_start);
     }
 
-    erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
+    state = erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
+    off_heap_msgs = state & ERTS_PSFLG_OFF_HEAP_MSGS;
     if (erts_system_monitor_long_gc != 0) {
 	get_now(&ms1, &s1, &us1);
     }
@@ -444,11 +447,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     while (!done) {
 	if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) {
 	    DTRACE2(gc_major_start, pidbuf, need);
-	    done = major_collection(p, need, objv, nobj, &reclaimed_now);
+	    done = major_collection(p, need, objv, nobj, &reclaimed_now, off_heap_msgs);
 	    DTRACE2(gc_major_end, pidbuf, reclaimed_now);
 	} else {
 	    DTRACE2(gc_minor_start, pidbuf, need);
-	    done = minor_collection(p, need, objv, nobj, &reclaimed_now);
+	    done = minor_collection(p, need, objv, nobj, &reclaimed_now, off_heap_msgs);
 	    DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
 	}
     }
@@ -831,7 +834,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 }
 
 static int
-minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
+minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs)
 {
     Uint mature = HIGH_WATER(p) - HEAP_START(p);
 
@@ -870,20 +873,22 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	Uint size_after;
 	Uint need_after;
 	Uint stack_size = STACK_SZ_ON_HEAP(p);
-	Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
+	Uint fragments = MBUF_SIZE(p) + combined_message_size(p, off_heap_msgs);
 	Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p));
 	Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0);
 
         do_minor(p, new_sz, objv, nobj);
 
-	/*
-	 * Copy newly received message onto the end of the new heap.
-	 */
-	ErtsGcQuickSanityCheck(p);
-	for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-	    if (msgp->data.attached) {
-		erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
-		ErtsGcQuickSanityCheck(p);
+	if (!off_heap_msgs) {
+	    /*
+	     * Copy newly received message onto the end of the new heap.
+	     */
+	    ErtsGcQuickSanityCheck(p);
+	    for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+		if (msgp->data.attached) {
+		    erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
+		    ErtsGcQuickSanityCheck(p);
+		}
 	    }
 	}
 	ErtsGcQuickSanityCheck(p);
@@ -1209,7 +1214,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
  */
 
 static int
-major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
+major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs)
 {
     Rootset rootset;
     Roots* roots;
@@ -1222,8 +1227,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
     Uint n;
     Uint new_sz;
-    Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
-    ErlMessage *msgp;
+    Uint fragments = MBUF_SIZE(p) + combined_message_size(p, off_heap_msgs);
 
     size_before = fragments + (HEAP_TOP(p) - HEAP_START(p));
 
@@ -1433,13 +1437,16 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 
     ErtsGcQuickSanityCheck(p);
 
-    /*
-     * Copy newly received message onto the end of the new heap.
-     */
-    for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-	if (msgp->data.attached) {
-	    erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
-	    ErtsGcQuickSanityCheck(p);
+    if (!off_heap_msgs) {
+	ErlMessage *msgp;
+	/*
+	 * Copy newly received message onto the end of the new heap.
+	 */
+	for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+	    if (msgp->data.attached) {
+		erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
+		ErtsGcQuickSanityCheck(p);
+	    }
 	}
     }
 
@@ -1500,15 +1507,17 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int
  * mbuf list.
  */
 static Uint
-combined_message_size(Process* p)
+combined_message_size(Process* p, int off_heap_msgs)
 {
-    Uint sz = 0;
+    Uint sz;
     ErlMessage *msgp;
 
-    for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-	if (msgp->data.attached) {
+    if (off_heap_msgs)
+	return 0;
+
+    for (sz = 0, msgp = p->msg.first; msgp; msgp = msgp->next) {
+	if (msgp->data.attached)
 	    sz += erts_msg_attached_data_size(msgp);
-	}
     }
     return sz;
 }
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 0e128c9b99..be2c5ced9e 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -389,12 +389,13 @@ erl_init(int ncpu,
     erl_nif_init();
 }
 
-static void
+static Eterm
 erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv)
 {
     int i;
     Eterm start_mod;
     Eterm args;
+    Eterm res;
     Eterm* hp;
     Process parent;
     ErlSpawnOpts so;
@@ -424,10 +425,11 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
     hp += 2;
     args = CONS(hp, env, args);
 
-    so.flags = 0;
-    (void) erl_create_process(&parent, start_mod, am_start, args, &so);
+    so.flags = SPO_SYSTEM_PROC;
+    res = erl_create_process(&parent, start_mod, am_start, args, &so);
     erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN);
     erts_cleanup_empty_process(&parent);
+    return res;
 }
 
 Eterm
@@ -2086,7 +2088,11 @@ erl_start(int argc, char **argv)
 
     erts_initialized = 1;
 
-    erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv);
+    {
+	Eterm init = erl_first_process_otp("otp_ring0", NULL, 0,
+					   boot_argc, boot_argv);
+	erts_bif_timer_start_servers(init);
+    }
 
 #ifdef ERTS_SMP
     erts_start_schedulers();
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 8870fac7d9..ce91acb6a4 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -994,7 +994,7 @@ erts_send_message(Process* sender,
 #endif
 	    );
         BM_SWAP_TIMER(send,system);
-    } else if (sender == receiver) {
+    } else if (sender == receiver && !(sender->flags & F_OFF_HEAP_MSGS)) {
 	/* Drop message if receiver has a pending exit ... */
 #ifdef ERTS_SMP
 	ErtsProcLocks need_locks = (~(*receiver_locks)
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 0f3bb8d281..a2a7193ea9 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -198,15 +198,25 @@ do {									\
     if ((M)->data.attached) {						\
 	Uint need__ = erts_msg_attached_data_size((M));			\
  	if ((ST) - (HT) >= need__) {					\
-	    Uint *htop__ = (HT);					\
+	    Uint *htop__;						\
+	move__attached__msg__data____:					\
+	    htop__ = (HT);						\
 	    erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\
 	    ASSERT(htop__ - (HT) <= need__);				\
 	    (HT) = htop__;						\
 	}								\
 	else {								\
+	    int off_heap_msgs__ = (int) (P)->flags & F_OFF_HEAP_MSGS;	\
+	    if (!off_heap_msgs__)					\
+		need__ = 0;						\
 	    { SWPO ; }							\
-	    (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
+	    (FC) -= erts_garbage_collect((P), need__, NULL, 0);		\
 	    { SWPI ; }							\
+	    if (off_heap_msgs__) {					\
+		ASSERT((M)->data.attached);				\
+		ASSERT((ST) - (HT) >= need__);				\
+		goto move__attached__msg__data____;			\
+	    }								\
 	}								\
 	ASSERT(!(M)->data.attached);					\
     }									\
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9dcaf2fdca..6e562e16c8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5844,6 +5844,13 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
     int check_emigration_need;
 #endif
 
+#ifdef ERTS_SMP
+    if ((p->static_flags & ERTS_STC_FLG_PREFER_SCHED)
+	&& p->preferred_run_queue != RUNQ_READ_RQ(&p->run_queue)) {
+	RUNQ_SET_RQ(&p->run_queue, p->preferred_run_queue);
+    }
+#endif
+
     a = state;
 
     while (1) {
@@ -5882,6 +5889,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 	    free_proxy_proc(proxy);
 
 	erts_smp_runq_lock(c_rq);
+
 	return 0;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -10511,7 +10519,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	    int ix = so->scheduler-1;
 	    ASSERT(0 <= ix && ix < erts_no_run_queues);
 	    rq = ERTS_RUNQ_IX(ix);
-	    state |= ERTS_PSFLG_BOUND;
+	    if (!(so->flags & SPO_PREFER_SCHED)) {
+		/* Unsupported feature... */
+		state |= ERTS_PSFLG_BOUND;
+	    }
 	}
 	prio = (erts_aint32_t) so->priority;
     }
@@ -10519,6 +10530,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET)
 	      | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET));
 
+    if (so->flags & SPO_OFF_HEAP_MSGS)
+	state |= ERTS_PSFLG_OFF_HEAP_MSGS;
+
     if (!rq)
 	rq = erts_get_runq_proc(parent);
 
@@ -10542,11 +10556,25 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     heap_need = arg_size;
 
     p->flags = erts_default_process_flags;
+    if (so->flags & SPO_OFF_HEAP_MSGS)
+	p->flags |= F_OFF_HEAP_MSGS;
 
+#ifdef ERTS_SMP
+    p->preferred_run_queue = NULL;
+#endif
+    p->static_flags = 0;
+    if (so->flags & SPO_SYSTEM_PROC)
+	p->static_flags |= ERTS_STC_FLG_SYSTEM_PROC;
     if (so->flags & SPO_USE_ARGS) {
 	p->min_heap_size  = so->min_heap_size;
 	p->min_vheap_size = so->min_vheap_size;
 	p->max_gen_gcs    = so->max_gen_gcs;
+	if (so->flags & SPO_PREFER_SCHED) {
+#ifdef ERTS_SMP
+	    p->preferred_run_queue = rq;
+#endif
+	    p->static_flags |= ERTS_STC_FLG_PREFER_SCHED;
+	}
     } else {
 	p->min_heap_size  = H_MIN_SIZE;
 	p->min_vheap_size = BIN_VH_MIN_SIZE;
@@ -10867,6 +10895,8 @@ void erts_init_empty_process(Process *p)
 
     p->parent = NIL;
     p->approx_started = 0;
+    p->static_flags = 0;
+
     p->common.u.alive.started_interval = 0;
 
 #ifdef HIPE
@@ -10892,6 +10922,7 @@ void erts_init_empty_process(Process *p)
     p->pending_suspenders = NULL;
     p->pending_exit.reason = THE_NON_VALUE;
     p->pending_exit.bp = NULL;
+    p->preferred_run_queue = NULL;
     erts_proc_lock_init(p);
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
@@ -11805,6 +11836,9 @@ erts_do_exit_process(Process* p, Eterm reason)
     }
 #endif
 
+    if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC)
+	erl_exit(1, "System process %T terminated: %T\n", p->common.id, reason);
+
 #ifdef ERTS_SMP
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
     /* By locking all locks (main lock is already locked) when going
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 6ef56b1974..53a992e115 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -938,6 +938,8 @@ struct process {
     Eterm parent;		/* Pid of process that created this process. */
     erts_approx_time_t approx_started; /* Time when started. */
 
+    Uint32 static_flags;        /* Flags that do *not* change */
+
     /* This is the place, where all fields that differs between memory
      * architectures, have gone to.
      */
@@ -969,6 +971,7 @@ struct process {
     ErtsSchedulerData *scheduler_data;
     Eterm suspendee;
     ErtsPendingSuspend *pending_suspenders;
+    ErtsRunQueue *preferred_run_queue;
     erts_smp_atomic_t run_queue;
 #ifdef HIPE
     struct hipe_process_state_smp hipe_smp;
@@ -1078,11 +1081,12 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
+#define ERTS_PSFLG_OFF_HEAP_MSGS	ERTS_PSFLG_BIT(18)
 #ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
-#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
-#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
-#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(19)
+#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(22)
 #endif
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
@@ -1097,6 +1101,12 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
     (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
 
+/*
+ * Static flags that do not change after process creation.
+ */
+#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)
+#define ERTS_STC_FLG_PREFER_SCHED	(((Uint32) 1) << 1)
+
 /* The sequential tracing token is a tuple of size 5:
  *
  *    {Flags, Label, Serial, Sender}
@@ -1124,6 +1134,9 @@ void erts_check_for_holes(Process* p);
 #define SPO_LINK 1
 #define SPO_USE_ARGS 2
 #define SPO_MONITOR 4
+#define SPO_OFF_HEAP_MSGS 8
+#define SPO_SYSTEM_PROC 16
+#define SPO_PREFER_SCHED 32
 
 /*
  * The following struct contains options for a process to be spawned.
@@ -1211,6 +1224,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC */
+#define F_OFF_HEAP_MSGS      (1 << 12)
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 1534fb8058..7dfa7d8743 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1984,3 +1984,4 @@ BIF_RETTYPE os_system_time_1(BIF_ALIST_0)
     stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
     BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0));
 }
+
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index e24aef3e3c..96c21d5320 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1154,7 +1154,9 @@ erts_alloc_message_heap_state(Uint size,
     state = erts_smp_atomic32_read_acqb(&receiver->state);
     if (statep)
 	*statep = state;
-    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+    if (state & (ERTS_PSFLG_OFF_HEAP_MSGS
+		 | ERTS_PSFLG_EXITING
+		 | ERTS_PSFLG_PENDING_EXIT))
 	goto allocate_in_mbuf;
 #endif
 
@@ -1174,7 +1176,9 @@ erts_alloc_message_heap_state(Uint size,
 	state = erts_smp_atomic32_read_nob(&receiver->state);
 	if (statep)
 	    *statep = state;
-	if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+	if ((state & (ERTS_PSFLG_OFF_HEAP_MSGS
+		      | ERTS_PSFLG_EXITING
+		      | ERTS_PSFLG_PENDING_EXIT))
 	    || (receiver->flags & F_DISABLE_GC)
 	    || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
 	    /*
-- 
cgit v1.2.3


From 1d9350693fe2c4d1d6b2baa504aacd070e023a1a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Feb 2015 10:12:02 +0100
Subject: Multiple timer wheels

---
 erts/emulator/beam/erl_bif_timer.c |   2 +-
 erts/emulator/beam/erl_init.c      |   1 +
 erts/emulator/beam/erl_process.c   | 242 +++++++++++++++++------
 erts/emulator/beam/erl_process.h   |   4 +
 erts/emulator/beam/erl_time.h      |  30 ++-
 erts/emulator/beam/erl_time_sup.c  |   1 +
 erts/emulator/beam/time.c          | 387 +++++++++++++++++++++----------------
 erts/emulator/beam/utils.c         |   2 +-
 8 files changed, 424 insertions(+), 245 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 8b444f2b01..0bd8d20c34 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -481,7 +481,7 @@ setup_bif_timer(Uint32 xflags,
 
     tab_insert(btm);
     ASSERT(btm == tab_find(ref));
-    btm->tm.active = 0; /* MUST be initalized */
+    erts_init_timer(&btm->tm);
     erts_set_timer(&btm->tm,
 		  (ErlTimeoutProc) bif_timer_timeout,
 		  (ErlCancelProc) bif_timer_cleanup,
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index be2c5ced9e..6708324a29 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -340,6 +340,7 @@ erl_init(int ncpu,
 			 no_dirty_io_schedulers
 #endif
 			 );
+    erts_late_init_time_sup();
     erts_init_cpu_topology(); /* Must be after init_scheduling */
     erts_init_gc(); /* Must be after init_scheduling */
     erts_alloc_late_init();
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6e562e16c8..45b6fc5fb3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2187,7 +2187,7 @@ aux_work_timeout_late_init(void)
 {
     aux_work_tmo->initialized = 1;
     if (erts_atomic32_read_nob(&aux_work_tmo->refc)) {
-	aux_work_tmo->timer.data.active = 0;
+	erts_init_timer(&aux_work_tmo->timer.data);
 	erts_set_timer(&aux_work_tmo->timer.data,
 		       aux_work_timeout,
 		       NULL,
@@ -2220,7 +2220,6 @@ aux_work_timeout(void *unused)
     if (refc != 1
 	|| 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) {
 	/* Setup next timeout... */
-	aux_work_tmo->timer.data.active = 0;
 	erts_set_timer(&aux_work_tmo->timer.data,
 		       aux_work_timeout,
 		       NULL,
@@ -2239,7 +2238,7 @@ setup_aux_work_timer(void)
     else
 #endif
     {
-	aux_work_tmo->timer.data.active = 0;
+	erts_init_timer(&aux_work_tmo->timer.data);
 	erts_set_timer(&aux_work_tmo->timer.data,
 		       aux_work_timeout,
 		       NULL,
@@ -2641,6 +2640,13 @@ thr_prgr_fin_wait(void *vssi)
 
 static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp);
 
+void
+erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time)
+{
+    /* TODO only poke when needed (based on timeout_time) */
+    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1));
+}
+
 static void *
 aux_thread(void *unused)
 {
@@ -2649,6 +2655,11 @@ aux_thread(void *unused)
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     int thr_prgr_active = 1;
+    ErtsTimerWheel *timer_wheel = erts_default_timer_wheel;
+    ErtsNextTimeoutRef nxt_tmo_ref = erts_get_next_timeout_reference(timer_wheel);
+
+    if (!timer_wheel)
+	ERTS_INTERNAL_ERROR("Missing aux timer wheel");
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -2672,6 +2683,7 @@ aux_thread(void *unused)
     sched_prep_spin_wait(ssi);
 
     while (1) {
+	ErtsMonotonicTime current_time;
 	erts_aint32_t flgs;
 
 	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
@@ -2683,28 +2695,56 @@ aux_thread(void *unused)
 		erts_thr_progress_leader_update(NULL);
 	}
 
-	if (!aux_work) {
-	    if (thr_prgr_active)
-		erts_thr_progress_active(NULL, thr_prgr_active = 0);
-	    erts_thr_progress_prepare_wait(NULL);
+	if (aux_work) {
+	    current_time = erts_get_monotonic_time();
+	    if (current_time >= erts_next_timeout_time(nxt_tmo_ref)) {
+		if (!thr_prgr_active)
+		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
+		erts_bump_timers(timer_wheel, current_time);
+	    }
+	}
+	else {
+	    ErtsMonotonicTime timeout_time;
+	    timeout_time = erts_check_next_timeout_time(timer_wheel,
+							ERTS_SEC_TO_MONOTONIC(10*60));
+	    current_time = erts_get_monotonic_time();
+	    if (current_time >= timeout_time) {
+		if (!thr_prgr_active)
+		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
+	    }
+	    else {
+		if (thr_prgr_active)
+		    erts_thr_progress_active(NULL, thr_prgr_active = 0);
+		erts_thr_progress_prepare_wait(NULL);
 
-	    ERTS_SCHED_FAIR_YIELD();
+		ERTS_SCHED_FAIR_YIELD();
 
-	    flgs = sched_spin_wait(ssi, 0);
+		flgs = sched_spin_wait(ssi, 0);
 
-	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		    int res;
-		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		    do {
-			res = erts_tse_wait(ssi->event);
-		    } while (res == EINTR);
+		    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+			int res;
+			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+			current_time = erts_get_monotonic_time();
+			do {
+			    Sint64 timeout;
+			    if (current_time >= timeout_time)
+				break;
+			    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+							     - current_time
+							     - 1) + 1;
+			    res = erts_tse_twait(ssi->event, timeout);
+			    current_time = erts_get_monotonic_time();
+			} while (res == EINTR);
+		    }
 		}
+		erts_thr_progress_finalize_wait(NULL);
 	    }
-	    erts_thr_progress_finalize_wait(NULL);
+	    if (current_time >= timeout_time)
+		erts_bump_timers(timer_wheel, current_time);
 	}
 
 	flgs = sched_prep_spin_wait(ssi);
@@ -2771,6 +2811,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    sched_wall_time_change(esdp, thr_prgr_active);
 
 	while (1) {
+	    ErtsMonotonicTime current_time;
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
@@ -2784,34 +2825,65 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    erts_thr_progress_leader_update(esdp);
 	    }
 
-	    if (aux_work)
+	    if (aux_work) {
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+		current_time = erts_get_monotonic_time();
+		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			sched_wall_time_change(esdp, 1);
+		    }
+		    erts_bump_timers(esdp->timer_wheel, current_time);
+		}
+	    }
 	    else {
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		    if (thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			sched_wall_time_change(esdp, 0);
+		ErtsMonotonicTime timeout_time;
+		timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
+							    ERTS_SEC_TO_MONOTONIC(10*60));
+		current_time = erts_get_monotonic_time();
+		if (current_time >= timeout_time) {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			sched_wall_time_change(esdp, 1);
 		    }
-		    erts_thr_progress_prepare_wait(esdp);
 		}
+		else {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			if (thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			    sched_wall_time_change(esdp, 0);
+			}
+			erts_thr_progress_prepare_wait(esdp);
+		    }
 
-		ERTS_SCHED_FAIR_YIELD();
+		    ERTS_SCHED_FAIR_YIELD();
 
-		flgs = sched_spin_wait(ssi, spincount);
-		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+		    flgs = sched_spin_wait(ssi, spincount);
 		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-			int res;
-			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			do {
-			    res = erts_tse_twait(ssi->event, -1);
-			} while (res == EINTR);
+			flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+			if (flgs & ERTS_SSI_FLG_SLEEPING) {
+			    int res;
+			    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+			    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+			    current_time = erts_get_monotonic_time();
+			    do {
+				Sint64 timeout;
+				if (current_time >= timeout_time)
+				    break;
+				timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+								 - current_time
+								 - 1) + 1;
+				res = erts_tse_twait(ssi->event, timeout);
+				current_time = erts_get_monotonic_time();
+			    } while (res == EINTR);
+			}
 		    }
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+			erts_thr_progress_finalize_wait(esdp);
 		}
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-		    erts_thr_progress_finalize_wait(esdp);
+		if (current_time >= timeout_time)
+		    erts_bump_timers(esdp->timer_wheel, current_time);
 	    }
 
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
@@ -2879,8 +2951,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    erl_sys_schedule(1); /* Might give us something to do */
 
 	    current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time())
-		erts_bump_timers(current_time);
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		erts_bump_timers(esdp->timer_wheel, current_time);
 
 	sys_aux_work:
 #ifndef ERTS_SMP
@@ -2997,8 +3069,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	{
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time())
-		erts_bump_timers(current_time);
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		erts_bump_timers(esdp->timer_wheel, current_time);
 	}
 
 #ifndef ERTS_SMP
@@ -5269,6 +5341,10 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 #else
     esdp->no = (Uint) num;
 #endif
+
+    esdp->timer_wheel = erts_default_timer_wheel;
+    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
+
     esdp->ssi = ssi;
     esdp->current_process = NULL;
     esdp->current_port = NULL;
@@ -6765,6 +6841,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
 	    while (1) {
+		ErtsMonotonicTime current_time;
 		erts_aint32_t qmask;
 		erts_aint32_t flgs;
 
@@ -6789,30 +6866,64 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		    }
 		}
 
-		if (!aux_work) {
-		    if (thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			sched_wall_time_change(esdp, 0);
+		if (aux_work) {
+		    current_time = erts_get_monotonic_time();
+		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
+			}
+			erts_bump_timers(esdp->timer_wheel, current_time);
 		    }
-		    erts_thr_progress_prepare_wait(esdp);
-		    flgs = sched_spin_suspended(ssi,
-						ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
-		    if (flgs == (ERTS_SSI_FLG_SLEEPING
-				 | ERTS_SSI_FLG_WAITING
-				 | ERTS_SSI_FLG_SUSPENDED)) {
-			flgs = sched_set_suspended_sleeptype(ssi);
+		}
+		else {
+		    ErtsMonotonicTime timeout_time;
+		    timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
+								ERTS_SEC_TO_MONOTONIC(60*60));
+		    current_time = erts_get_monotonic_time();
+
+		    if (current_time >= timeout_time) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
+			}
+		    }
+		    else {		    
+			if (thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			    sched_wall_time_change(esdp, 0);
+			}
+			erts_thr_progress_prepare_wait(esdp);
+			flgs = sched_spin_suspended(ssi,
+						    ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
 			if (flgs == (ERTS_SSI_FLG_SLEEPING
-				     | ERTS_SSI_FLG_TSE_SLEEPING
 				     | ERTS_SSI_FLG_WAITING
 				     | ERTS_SSI_FLG_SUSPENDED)) {
-			    int res;
-
-			    do {
-				res = erts_tse_twait(ssi->event, -1);
-			    } while (res == EINTR);
+			    flgs = sched_set_suspended_sleeptype(ssi);
+			    if (flgs == (ERTS_SSI_FLG_SLEEPING
+					 | ERTS_SSI_FLG_TSE_SLEEPING
+					 | ERTS_SSI_FLG_WAITING
+					 | ERTS_SSI_FLG_SUSPENDED)) {
+				int res;
+
+				current_time = erts_get_monotonic_time();
+				do {
+				    Sint64 timeout;
+				    if (current_time >= timeout_time)
+					break;
+				    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+								     - current_time
+								     - 1) + 1;
+				    res = erts_tse_twait(ssi->event, timeout);
+				    current_time = erts_get_monotonic_time();
+				} while (res == EINTR);
+			    }
 			}
+			erts_thr_progress_finalize_wait(esdp);
 		    }
-		    erts_thr_progress_finalize_wait(esdp);
+
+		    if (current_time >= timeout_time)
+			erts_bump_timers(esdp->timer_wheel, current_time);
 		}
 
 		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
@@ -7631,6 +7742,9 @@ sched_thread_func(void *vesdp)
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
     Uint no = esdp->no;
+
+    esdp->timer_wheel = erts_create_timer_wheel((int) no);
+    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
 #ifdef ERTS_SMP
     ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
@@ -9048,9 +9162,9 @@ Process *schedule(Process *p, int calls)
 
 	{
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time()) {
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
 		erts_smp_runq_unlock(rq);
-		erts_bump_timers(current_time);
+		erts_bump_timers(esdp->timer_wheel, current_time);
 		erts_smp_runq_lock(rq);
 	    }
 	}
@@ -9213,8 +9327,8 @@ Process *schedule(Process *p, int calls)
 	    erl_sys_schedule(1);
 
 	    current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time())
-		erts_bump_timers(current_time);
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		erts_bump_timers(esdp->timer_wheel, current_time);
 
 #ifdef ERTS_SMP
 	    erts_smp_runq_lock(rq);
@@ -10652,7 +10766,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef ERTS_SMP
     p->common.u.alive.ptimer = NULL;
 #else
-    sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer));
+    erts_init_timer(&p->common.u.alive.tm);
 #endif
 
     p->common.u.alive.reg = NULL;
@@ -10845,7 +10959,7 @@ void erts_init_empty_process(Process *p)
 #ifdef ERTS_SMP
     p->common.u.alive.ptimer = NULL;
 #else
-    memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer));
+    erts_init_timer(&p->common.u.alive.tm);
 #endif
     p->next = NULL;
     p->off_heap.first = NULL;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 53a992e115..77a4b45b09 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -562,6 +562,8 @@ struct ErtsSchedulerData_ {
     Eterm* x_reg_array;		/* X registers */
     FloatDef* f_reg_array;	/* Floating point registers. */
 
+    ErtsTimerWheel *timer_wheel;
+    ErtsNextTimeoutRef next_tmo_ref;
 #ifdef ERTS_SMP
     ethr_tid tid;		/* Thread id */
     struct erl_bits_state erl_bits_state; /* erl_bits.c state */
@@ -2237,6 +2239,8 @@ extern int erts_disable_proc_not_running_opt;
 
 void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
 
+void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);
+
 #ifdef ERTS_SMP
 void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
 ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index e461594e9c..7933f4973d 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -26,6 +26,10 @@
 #define ERTS_TIME_ASSERT(B) ((void) 1)
 #endif
 
+typedef struct ErtsTimerWheel_ ErtsTimerWheel;
+typedef erts_atomic64_t * ErtsNextTimeoutRef;
+extern ErtsTimerWheel *erts_default_timer_wheel;
+
 extern SysTimeval erts_first_emu_time;
 
 /*
@@ -35,8 +39,8 @@ typedef struct erl_timer {
     struct erl_timer* next;	/* next entry tiw slot or chain */
     struct erl_timer* prev;	/* prev entry tiw slot or chain */
     Uint slot;			/* slot in timer wheel */
+    erts_smp_atomic_t wheel;
     ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
-    int    active;		/* 1=activated, 0=deactivated */
     /* called when timeout */
     void (*timeout)(void*);
     /* called when cancel (may be NULL) */
@@ -63,7 +67,6 @@ union ErtsSmpPTimer_ {
     ErtsSmpPTimer *next;
 };
 
-
 void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
 			    Eterm id,
 			    ErlTimeoutProc timeout_func,
@@ -79,28 +82,35 @@ void erts_late_init_time_sup(void);
 
 /* timer-wheel api */
 
+ErtsTimerWheel *erts_create_timer_wheel(int);
+ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *);
 void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode);
 void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
 void erts_cancel_timer(ErlTimer*);
-void erts_bump_timers(ErtsMonotonicTime);
-Uint erts_timer_wheel_memory_size(void);
 Uint erts_time_left(ErlTimer *);
+void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime);
+Uint erts_timer_wheel_memory_size(void);
 
 #ifdef DEBUG
 void erts_p_slpq(void);
 #endif
 
-ErtsMonotonicTime erts_check_next_timeout_time(ErtsMonotonicTime);
+ErtsMonotonicTime erts_check_next_timeout_time(ErtsTimerWheel *,
+					       ErtsMonotonicTime);
 
-extern erts_atomic64_t erts_next_timeout__;
-
-ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(void);
+ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p);
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(void)
+ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p)
+{
+    erts_smp_atomic_init_nob(&p->wheel, (erts_aint_t) NULL);
+}
+
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
 {
-    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&erts_next_timeout__);
+    return (ErtsMonotonicTime) erts_atomic64_read_acqb((erts_atomic64_t *) nxt_tmo_ref);
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 7dfa7d8743..d47d1682d7 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -690,6 +690,7 @@ static void
 late_init_time_correction(void)
 {
     if (time_sup.inf.c.finalized_offset) {
+	erts_init_timer(&time_sup.inf.c.parmon.timer);
 	erts_set_timer(&time_sup.inf.c.parmon.timer,
 #ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
 		       init_check_time_correction,
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index c9f8b68bca..9f997e1d0b 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -86,9 +86,6 @@
 #define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24)
 #define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY)
 
-static erts_smp_atomic32_t is_bumping;
-static erts_smp_mtx_t tiw_lock;
-
 
 /* BEGIN tiw_lock protected variables 
 **
@@ -101,14 +98,6 @@ static erts_smp_mtx_t tiw_lock;
 #else
 #define TIW_SIZE (1 << 20)
 #endif
-static ErlTimer** tiw;		/* the timing wheel, allocated in init_time() */
-static ErtsMonotonicTime tiw_pos; /* current position in wheel */
-static Uint tiw_nto;		/* number of timeouts in wheel */
-static struct {
-    ErlTimer *head;
-    ErlTimer **tail;
-    Uint nto;
-} tiw_at_once;
 
 /* Actual interval time chosen by sys_init_time() */
 
@@ -120,23 +109,52 @@ static int tiw_itime; /* Constant after init */
 #  define TIW_ITIME tiw_itime
 #endif
 
-static int true_next_timeout_time;
-static ErtsMonotonicTime next_timeout_time;
-erts_atomic64_t erts_next_timeout__;
+struct ErtsTimerWheel_ {
+    ErlTimer *w[TIW_SIZE];
+    ErtsMonotonicTime pos;
+    Uint nto;
+    struct {
+	ErlTimer *head;
+	ErlTimer **tail;
+	Uint nto;
+    } at_once;
+    int true_next_timeout_time;
+    ErtsMonotonicTime next_timeout_time;
+    erts_atomic64_t next_timeout;
+    erts_smp_atomic32_t is_bumping;
+    erts_smp_mtx_t lock;
+};
+
+ErtsTimerWheel *erts_default_timer_wheel; /* managed by aux thread */
+
+static ERTS_INLINE ErtsTimerWheel *
+get_timer_wheel(ErlTimer *p)
+{
+    return (ErtsTimerWheel *) erts_smp_atomic_read_acqb(&p->wheel);
+}
+
+static ERTS_INLINE void
+set_timer_wheel(ErlTimer *p, ErtsTimerWheel *tiw)
+{
+    erts_smp_atomic_set_relb(&p->wheel, (erts_aint_t) tiw);
+}
 
 static ERTS_INLINE void
-init_next_timeout(ErtsMonotonicTime time)
+init_next_timeout(ErtsTimerWheel *tiw,
+		  ErtsMonotonicTime time)
 {
-    erts_atomic64_init_nob(&erts_next_timeout__,
+    erts_atomic64_init_nob(&tiw->next_timeout,
 			   (erts_aint64_t) time);
 }
 
 static ERTS_INLINE void
-set_next_timeout(ErtsMonotonicTime time, int true_timeout)
+set_next_timeout(ErtsTimerWheel *tiw,
+		 ErtsMonotonicTime time,
+		 int true_timeout)
 {
-    true_next_timeout_time = true_timeout;
-    next_timeout_time = time;
-    erts_atomic64_set_relb(&erts_next_timeout__,
+    tiw->true_next_timeout_time = true_timeout;
+    tiw->next_timeout_time = time;
+    erts_atomic64_set_relb(&tiw->next_timeout,
 			   (erts_aint64_t) time);
 }
 
@@ -144,7 +162,8 @@ set_next_timeout(ErtsMonotonicTime time, int true_timeout)
    or -1 if there are no timeouts                     */
 
 static ERTS_INLINE ErtsMonotonicTime
-find_next_timeout(ErtsMonotonicTime curr_time,
+find_next_timeout(ErtsTimerWheel *tiw,
+		  ErtsMonotonicTime curr_time,
 		  ErtsMonotonicTime max_search_time)
 {
     int start_ix, tiw_pos_ix;
@@ -152,16 +171,16 @@ find_next_timeout(ErtsMonotonicTime curr_time,
     int true_min_timeout;
     ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos, timeout_limit;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw_lock));
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw->lock));
 
-    if (true_next_timeout_time)
-	return next_timeout_time;
+    if (tiw->true_next_timeout_time)
+	return tiw->next_timeout_time;
 
     /* We never set next timeout beyond timeout_limit */
     timeout_limit = curr_time + ERTS_MONOTONIC_DAY;
 
-    if (tiw_nto == 0) { /* no timeouts in wheel */
-	true_min_timeout = true_next_timeout_time = 0;
+    if (tiw->nto == 0) { /* no timeouts in wheel */
+	true_min_timeout = tiw->true_next_timeout_time = 0;
 	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(timeout_limit);
 	goto found_next;
     }
@@ -170,13 +189,13 @@ find_next_timeout(ErtsMonotonicTime curr_time,
      * Don't want others entering trying to bump
      * timers while we are checking...
      */
-    set_next_timeout(timeout_limit, 0);
+    set_next_timeout(tiw, timeout_limit, 0);
 
     true_min_timeout = 1;
-    slot_timeout_pos = tiw_pos;
+    slot_timeout_pos = tiw->pos;
     min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
 
-    start_ix = tiw_pos_ix = (int) (tiw_pos & (TIW_SIZE-1));
+    start_ix = tiw_pos_ix = (int) (tiw->pos & (TIW_SIZE-1));
 
     do {
 	slot_timeout_pos++;
@@ -185,7 +204,7 @@ find_next_timeout(ErtsMonotonicTime curr_time,
 	    break;
 	}
 
-	p = tiw[tiw_pos_ix];
+	p = tiw->w[tiw_pos_ix];
 
 	while (p) {
 	    ErtsMonotonicTime timeout_pos;
@@ -207,16 +226,18 @@ find_next_timeout(ErtsMonotonicTime curr_time,
 found_next:
 
     min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos);
-    if (min_timeout != next_timeout_time)
-	set_next_timeout(min_timeout, true_min_timeout);
+    if (min_timeout != tiw->next_timeout_time)
+	set_next_timeout(tiw, min_timeout, true_min_timeout);
 
     return min_timeout;
 }
 
-static void remove_timer(ErlTimer *p) {
+static void
+remove_timer(ErtsTimerWheel *tiw, ErlTimer *p)
+{
     /* first */
     if (!p->prev) {
-	tiw[p->slot] = p->next;
+	tiw->w[p->slot] = p->next;
 	if(p->next)
 	    p->next->prev = NULL;
     } else {
@@ -233,40 +254,41 @@ static void remove_timer(ErlTimer *p) {
 
     p->next = NULL;
     p->prev = NULL;
-    /* Make sure cancel callback isn't called */
-    p->active = 0;
-    tiw_nto--;
+
+    set_timer_wheel(p, NULL);
+    tiw->nto--;
 }
 
 ErtsMonotonicTime
-erts_check_next_timeout_time(ErtsMonotonicTime max_search_time)
+erts_check_next_timeout_time(ErtsTimerWheel *tiw,
+			     ErtsMonotonicTime max_search_time)
 {
     ErtsMonotonicTime next, curr;
 
     curr = erts_get_monotonic_time();
 
-    erts_smp_mtx_lock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
 
-    next = find_next_timeout(curr, max_search_time);
+    next = find_next_timeout(tiw, curr, max_search_time);
 
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
 
     return next;
 }
 
 #ifndef DEBUG
-#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TO) ((void) 0)
+#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0)
 #else
-#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TO) debug_check_safe_to_skip_to((TO))
+#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO))
 static void
-debug_check_safe_to_skip_to(ErtsMonotonicTime skip_to_pos)
+debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos)
 {
     int slots, ix;
     ErlTimer *tmr;
     ErtsMonotonicTime tmp;
 
-    ix = (int) (tiw_pos & (TIW_SIZE-1));
-    tmp = skip_to_pos - tiw_pos;
+    ix = (int) (tiw->pos & (TIW_SIZE-1));
+    tmp = skip_to_pos - tiw->pos;
     ASSERT(tmp >= 0);
     if (tmp < (ErtsMonotonicTime) TIW_SIZE)
 	slots = (int) tmp;
@@ -274,7 +296,7 @@ debug_check_safe_to_skip_to(ErtsMonotonicTime skip_to_pos)
 	slots = TIW_SIZE;
 
      while (slots > 0) {
-	tmr = tiw[ix];
+	tmr = tiw->w[ix];
 	while (tmr) {
 	    ASSERT(tmr->timeout_pos > skip_to_pos);
 	    tmr = tmr->next;
@@ -287,79 +309,80 @@ debug_check_safe_to_skip_to(ErtsMonotonicTime skip_to_pos)
 }
 #endif
 
-void erts_bump_timers(ErtsMonotonicTime curr_time)
+void
+erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
     int tiw_pos_ix, slots;
     ErlTimer *p, *timeout_head, **timeout_tail;
     ErtsMonotonicTime bump_to, tmp_slots;
 
-    if (erts_smp_atomic32_cmpxchg_nob(&is_bumping, 1, 0) != 0)
+    if (erts_smp_atomic32_cmpxchg_nob(&tiw->is_bumping, 1, 0) != 0)
 	return; /* Another thread is currently bumping... */
 
     bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 
-    erts_smp_mtx_lock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
 
-    if (tiw_pos >= bump_to) {
+    if (tiw->pos >= bump_to) {
 	timeout_head = NULL;
 	goto done;
     }
 
     /* Don't want others here while we are bumping... */
-    set_next_timeout(curr_time + ERTS_MONOTONIC_DAY, 0);
+    set_next_timeout(tiw, curr_time + ERTS_MONOTONIC_DAY, 0);
 
-    if (!tiw_at_once.head) {
+    if (!tiw->at_once.head) {
 	timeout_head = NULL;
 	timeout_tail = &timeout_head;
     }
     else {
-	ASSERT(tiw_nto >= tiw_at_once.nto);
-	timeout_head = tiw_at_once.head;
-	timeout_tail = tiw_at_once.tail;
-	tiw_nto -= tiw_at_once.nto;
-	tiw_at_once.head = NULL;
-	tiw_at_once.tail = &tiw_at_once.head;
-	tiw_at_once.nto = 0;
+	ASSERT(tiw->nto >= tiw->at_once.nto);
+	timeout_head = tiw->at_once.head;
+	timeout_tail = tiw->at_once.tail;
+	tiw->nto -= tiw->at_once.nto;
+	tiw->at_once.head = NULL;
+	tiw->at_once.tail = &tiw->at_once.head;
+	tiw->at_once.nto = 0;
     }
 
-    if (tiw_nto == 0) {
-	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(bump_to);
-	tiw_pos = bump_to;
+    if (tiw->nto == 0) {
+	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to);
+	tiw->pos = bump_to;
 	goto done;
     }
 
-    if (true_next_timeout_time) {
+    if (tiw->true_next_timeout_time) {
 	ErtsMonotonicTime skip_until_pos;
 	/*
 	 * No need inspecting slots where we know no timeouts
 	 * to trigger should reside.
 	 */
 
-	skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(next_timeout_time);
+	skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
 	if (skip_until_pos > bump_to)
 	    skip_until_pos = bump_to;
 
-	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(skip_until_pos);
-	ASSERT(skip_until_pos > tiw_pos);
+	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos);
+	ASSERT(skip_until_pos > tiw->pos);
 
-	tiw_pos = skip_until_pos - 1;
+	tiw->pos = skip_until_pos - 1;
     }
 
-    tiw_pos_ix = (int) ((tiw_pos+1) & (TIW_SIZE-1));
-    tmp_slots = (bump_to - tiw_pos);
+    tiw_pos_ix = (int) ((tiw->pos+1) & (TIW_SIZE-1));
+    tmp_slots = (bump_to - tiw->pos);
     if (tmp_slots < (ErtsMonotonicTime) TIW_SIZE)
 	slots = (int) tmp_slots;
     else
 	slots = TIW_SIZE;
  
     while (slots > 0) {
-	p = tiw[tiw_pos_ix];
+	p = tiw->w[tiw_pos_ix];
 	while (p) {
 	    ErlTimer *next = p->next;
 	    ASSERT(p != next);
 	    if (p->timeout_pos <= bump_to) { /* we have a timeout */
 		/* Remove from list */
-		remove_timer(p);
+		remove_timer(tiw, p);
 		*timeout_tail = p;	/* Insert in timeout queue */
 		timeout_tail = &p->next;
 	    }
@@ -374,19 +397,21 @@ void erts_bump_timers(ErtsMonotonicTime curr_time)
     ASSERT(tmp_slots >= (ErtsMonotonicTime) TIW_SIZE
 	   || tiw_pos_ix == (int) ((bump_to+1) & (TIW_SIZE-1)));
 
-    tiw_pos = bump_to;
+    tiw->pos = bump_to;
 
     /* Search at most two seconds ahead... */
-    (void) find_next_timeout(curr_time, ERTS_SEC_TO_MONOTONIC(2));
+    (void) find_next_timeout(tiw, curr_time, ERTS_SEC_TO_MONOTONIC(2));
 
 done:
 
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
     
-    erts_smp_atomic32_set_nob(&is_bumping, 0);
+    erts_smp_atomic32_set_nob(&tiw->is_bumping, 0);
 
     /* Call timedout timers callbacks */
     while (timeout_head) {
+	ErlTimeoutProc timeout;
+	void *arg;
 	p = timeout_head;
 	timeout_head = p->next;
 	/* Here comes hairy use of the timer fields!
@@ -399,23 +424,61 @@ done:
 	p->next = NULL;
 	p->prev = NULL;
 	p->slot = 0;
-	(*p->timeout)(p->arg);
+	timeout = p->timeout;
+	arg = p->arg;
+	(*timeout)(arg);
     }
 }
 
 Uint
 erts_timer_wheel_memory_size(void)
 {
-    return (Uint) TIW_SIZE * sizeof(ErlTimer*);
+#ifdef ERTS_SMP
+    return sizeof(ErtsTimerWheel)*(1 + erts_no_schedulers);
+#else
+    return sizeof(ErtsTimerWheel);
+#endif
 }
 
+ErtsTimerWheel *
+erts_create_timer_wheel(int no)
+{
+    ErtsMonotonicTime mtime;
+    int i;
+    ErtsTimerWheel *tiw;
+    tiw = (ErtsTimerWheel *) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
+					sizeof(ErtsTimerWheel));
+    for(i = 0; i < TIW_SIZE; i++)
+	tiw->w[i] = NULL;
+
+    erts_smp_atomic32_init_nob(&tiw->is_bumping, 0);
+    erts_smp_mtx_init_x(&tiw->lock, "timer_wheel", make_small(no));
+
+    mtime = erts_get_monotonic_time();
+    tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
+    tiw->nto = 0;
+    tiw->at_once.head = NULL;
+    tiw->at_once.tail = &tiw->at_once.head;
+    tiw->at_once.nto = 0;
+    tiw->true_next_timeout_time = 0;
+    tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
+    init_next_timeout(tiw, mtime + ERTS_MONOTONIC_DAY);
+    return tiw;
+}
+
+ErtsNextTimeoutRef
+erts_get_next_timeout_reference(ErtsTimerWheel *tiw)
+{
+    return (ErtsNextTimeoutRef) &tiw->next_timeout;
+}
+
+
 /* this routine links the time cells into a free list at the start
    and sets the time queue as empty */
 void
 erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
 {
-    ErtsMonotonicTime mtime;
-    int i, itime;
+    int itime;
 
     /* system dependent init; must be done before do_time_init()
        if timer thread is enabled */
@@ -428,41 +491,39 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
     tiw_itime = itime;
 #endif
 
-    erts_smp_atomic32_init_nob(&is_bumping, 0);
-    erts_smp_mtx_init(&tiw_lock, "timer_wheel");
-
-    tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
-				  TIW_SIZE * sizeof(ErlTimer*));
-    for(i = 0; i < TIW_SIZE; i++)
-	tiw[i] = NULL;
-
-    mtime = erts_get_monotonic_time();
-    tiw_pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
-    tiw_nto = 0;
-    tiw_at_once.head = NULL;
-    tiw_at_once.tail = &tiw_at_once.head;
-    tiw_at_once.nto = 0;
-    init_next_timeout(mtime + ERTS_MONOTONIC_DAY);
-
-    erts_late_init_time_sup();
+    erts_default_timer_wheel = erts_create_timer_wheel(0);
 }
 
+void
+erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout,
+	       ErlCancelProc cancel, void *arg, Uint to)
+{
+    ErtsMonotonicTime timeout_time, timeout_pos;
+    ErtsMonotonicTime curr_time;
+    ErtsTimerWheel *tiw;
+    ErtsSchedulerData *esdp;
+
+    curr_time = erts_get_monotonic_time();
+    esdp = erts_get_scheduler_data();
+    if (esdp)
+	tiw = esdp->timer_wheel;
+    else
+	tiw = erts_default_timer_wheel;
 
+    erts_smp_mtx_lock(&tiw->lock);
 
+    if (get_timer_wheel(p))
+	ERTS_INTERNAL_ERROR("Double set timer");
 
-/*
-** Insert a process into the time queue, with a timeout 't'
-*/
-static ErtsMonotonicTime
-insert_timer(ErlTimer* p, ErtsMonotonicTime curr_time, ErtsMonotonicTime to)
-{
-    ErtsMonotonicTime timeout_time, timeout_pos;
+    p->timeout = timeout;
+    p->cancel = cancel;
+    p->arg = arg;
 
     if (to == 0) {
 	timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
-	tiw_nto++;
-	tiw_at_once.nto++;
-	*tiw_at_once.tail = p;
+	tiw->nto++;
+	tiw->at_once.nto++;
+	*tiw->at_once.tail = p;
 	p->next = NULL;
 	p->timeout_pos = timeout_pos;
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
@@ -479,13 +540,13 @@ insert_timer(ErlTimer* p, ErtsMonotonicTime curr_time, ErtsMonotonicTime to)
 	p->slot = (Uint) tm;
   
 	/* insert at head of list at slot */
-	p->next = tiw[tm];
+	p->next = tiw->w[tm];
 	p->prev = NULL;
 	if (p->next != NULL)
 	    p->next->prev = p;
-	tiw[tm] = p;
+	tiw->w[tm] = p;
 
-	tiw_nto++;
+	tiw->nto++;
 
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
 	p->timeout_pos = timeout_pos;
@@ -495,59 +556,45 @@ insert_timer(ErlTimer* p, ErtsMonotonicTime curr_time, ErtsMonotonicTime to)
 	       < ERTS_MSEC_TO_MONOTONIC(to) + ERTS_CLKTCKS_TO_MONOTONIC(1));
     }
 
-    if (timeout_time < next_timeout_time)
-	set_next_timeout(timeout_time, 1);
+    if (timeout_time < tiw->next_timeout_time)
+	set_next_timeout(tiw, timeout_time, 1);
 
-    return timeout_time;
-}
+    set_timer_wheel(p, tiw);
+
+    erts_smp_mtx_unlock(&tiw->lock);
 
-void
-erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
-	      void* arg, Uint t)
-{
-#ifdef ERTS_SMP
-    ErtsMonotonicTime timeout_time;
-#endif
-    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-    erts_smp_mtx_lock(&tiw_lock);
-    if (p->active) { /* XXX assert ? */
-	erts_smp_mtx_unlock(&tiw_lock);
-	return;
-    }
-    p->timeout = timeout;
-    p->cancel = cancel;
-    p->arg = arg;
-    p->active = 1;
-#ifdef ERTS_SMP
-    timeout_time =
-#else
-	(void)
-#endif
-	insert_timer(p, current_time, (ErtsMonotonicTime) t);
-    erts_smp_mtx_unlock(&tiw_lock);
 #if defined(ERTS_SMP)
-    erts_sys_schedule_interrupt_timed(1, timeout_time);
+    if (tiw == erts_default_timer_wheel)
+	erts_interupt_aux_thread_timed(timeout_time);
 #endif
+
 }
 
 void
-erts_cancel_timer(ErlTimer* p)
+erts_cancel_timer(ErlTimer *p)
 {
-    erts_smp_mtx_lock(&tiw_lock);
-    if (!p->active) { /* allow repeated cancel (drivers) */
-	erts_smp_mtx_unlock(&tiw_lock);
-	return;
-    }
-
-    remove_timer(p);
-    p->slot = 0;
+    ErtsTimerWheel *tiw;
+    ErlCancelProc cancel;
+    void *arg;
 
-    if (p->cancel != NULL) {
-	erts_smp_mtx_unlock(&tiw_lock);
-	(*p->cancel)(p->arg);
+    tiw = get_timer_wheel(p);
+    if (!tiw)
 	return;
+    
+    erts_smp_mtx_lock(&tiw->lock);
+    if (tiw != get_timer_wheel(p))
+	cancel = NULL;
+    else {
+	remove_timer(tiw, p);
+	p->slot = 0;
+
+	cancel = p->cancel;
+	arg = p->arg;
     }
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
+
+    if (cancel)
+	(*cancel)(arg);
 }
 
 /*
@@ -559,18 +606,19 @@ erts_cancel_timer(ErlTimer* p)
 Uint
 erts_time_left(ErlTimer *p)
 {
+    ErtsTimerWheel *tiw;
     ErtsMonotonicTime current_time, timeout_time;
 
-    erts_smp_mtx_lock(&tiw_lock);
-
-    if (!p->active) {
-	erts_smp_mtx_unlock(&tiw_lock);
+    tiw = get_timer_wheel(p);
+    if (!tiw)
 	return 0;
-    }
 
-    timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos);
-
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
+    if (tiw != get_timer_wheel(p))
+	timeout_time = ERTS_MONOTONIC_TIME_MIN;
+    else
+	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos);
+    erts_smp_mtx_unlock(&tiw->lock);
 
     current_time = erts_get_monotonic_time();
     if (timeout_time <= current_time)
@@ -581,34 +629,35 @@ erts_time_left(ErlTimer *p)
 #ifdef DEBUG
 void erts_p_slpq(void)
 {
+    ErtsTimerWheel *tiw = erts_default_timer_wheel;
     ErtsMonotonicTime current_time = erts_get_monotonic_time();
     int i;
     ErlTimer* p;
   
-    erts_smp_mtx_lock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
 
     /* print the whole wheel, starting at the current position */
     erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n",
-		current_time, tiw_pos, tiw_nto);
-    i = tiw_pos;
-    if (tiw[i] != NULL) {
+		current_time, tiw->pos, tiw->nto);
+    i = tiw->pos;
+    if (tiw->w[i] != NULL) {
 	erts_printf("%d:\n", i);
-	for(p = tiw[i]; p != NULL; p = p->next) {
+	for(p = tiw->w[i]; p != NULL; p = p->next) {
 	    erts_printf(" (timeout time %bps, slot %d)\n",
 			ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos),
 			p->slot);
 	}
     }
-    for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw_pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) {
-	if (tiw[i] != NULL) {
+    for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw->pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) {
+	if (tiw->w[i] != NULL) {
 	    erts_printf("%d:\n", i);
-	    for(p = tiw[i]; p != NULL; p = p->next) {
+	    for(p = tiw->w[i]; p != NULL; p = p->next) {
 		erts_printf(" (timeout time %bps, slot %d)\n",
 			    ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot);
 	    }
 	}
     }
 
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
 }
 #endif /* DEBUG */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 54f1a122c4..da03960b59 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3777,7 +3777,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
     res->timer.timeout_func = timeout_func;
     res->timer.timer_ref = timer_ref;
     res->timer.id = id;
-    res->timer.tm.active = 0; /* MUST be initalized */
+    erts_init_timer(&res->timer.tm);
 
     ASSERT(!*timer_ref);
 
-- 
cgit v1.2.3


From 6071cbf00680b94d107906e1c7f1e13f79676479 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 20 Mar 2015 16:36:34 +0100
Subject: erts: Fix hashmap overestimation

Old overestimation assumed an average of k/3 nodes.
This can be bad for large maps (with tight standard deviations)
as the average vary between 0.3*k and up to almost 0.4*k.
---
 erts/emulator/beam/erl_map.c | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index bd6da0a6f1..bcbda65da0 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2549,19 +2549,24 @@ static int int_sqrt_ceiling(Uint x)
     }
 }
 
-Uint hashmap_over_estimated_heap_size(Uint n)
+Uint hashmap_over_estimated_heap_size(Uint k)
 {
-    /* n is nr of key-value pairs.
-       Average nr of nodes is about n/3.
-       Standard deviation is about sqrt(n)/3.
-       Assuming normal probability distribution,
-       we overestimate nr of nodes by 14 std.devs, which gives a probability
-       for overrun of 1.0e-49 (same magnitude as a git SHA1 collision).
+    /* k is nr of key-value pairs.
+       N(k) is expected nr of nodes in hamt.
+
+       Observation:
+       For uniformly distributed hash values, average of N varies between
+       0.3*k and 0.4*k (with a beautiful sine curve)
+       and standard deviation of N is about sqrt(k)/3.
+
+       Assuming normal probability distribution, we overestimate nr of nodes
+       by 15 std.devs above the average, which gives a probability for overrun
+       less than 1.0e-49 (same magnitude as a git SHA1 collision).
      */
-    Uint nodes = (n + int_sqrt_ceiling(n)*14) / 3;
-    return (n*2 +     /* leaf cons cells */
-	    n +       /* leaf list terms */
-	    nodes*2); /* headers + parent refs */
+    Uint max_nodes = 2*k/5 + (15/3)*int_sqrt_ceiling(k);
+    return (k*2 +     /* leaf cons cells */
+	    k +       /* leaf list terms */
+	    max_nodes*2); /* headers + parent boxed terms */
 }
 
 
-- 
cgit v1.2.3


From c1be2e5baed4794f7017dab9558afe8a737b63ee Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 20 Mar 2015 16:37:18 +0100
Subject: erts: Add test map_SUITE:t_hashmap_balance

---
 erts/emulator/beam/erl_bif_info.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d750e34be3..0774ab51af 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3597,6 +3597,26 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 
 		BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups));
 	    }
+	    else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) {
+		Uint hash = (Uint) make_internal_hash(tp[2]);
+		Uint hsz = 0;
+		Eterm* hp;
+		erts_bld_uint(NULL, &hsz, hash);
+		hp = HAlloc(BIF_P,hsz);
+		return erts_bld_uint(&hp, NULL, hash);
+	    }
+	    else if (ERTS_IS_ATOM_STR("atom", tp[1])) {
+		Uint ix;
+		if (!term_to_Uint(tp[2], &ix))
+		    BIF_ERROR(BIF_P, BADARG);
+		while (ix >= atom_table_size()) {
+		    char tmp[20];
+		    erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size());
+		    erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1);
+		}
+		return make_atom(ix);
+	    }
+
 	    break;
 	}
 	default:
-- 
cgit v1.2.3


From 50713b7d6179477d6018e6a9ca5610617e3de1fe Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Sat, 21 Mar 2015 11:59:34 +0100
Subject: Unbreak lcnt

---
 erts/emulator/beam/erl_init.c     | 1 -
 erts/emulator/beam/erl_time.h     | 1 -
 erts/emulator/beam/erl_time_sup.c | 3 +--
 erts/emulator/beam/sys.h          | 1 +
 4 files changed, 2 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 6708324a29..8d38ff9c2c 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1170,7 +1170,6 @@ early_init(int *argc, char **argv) /*
 
     /* Creates threads on Windows that depend on the arguments, so has to be after erl_sys_args */
     erl_sys_init();
-    erts_early_init_time_sup();
 
     erts_ets_realloc_always_moves = 0;
     erts_ets_always_compress = 0;
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 7933f4973d..924b0b6091 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -77,7 +77,6 @@ void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer);
 void erts_monitor_time_offset(Eterm id, Eterm ref);
 int erts_demonitor_time_offset(Eterm ref);
 
-void erts_early_init_time_sup(void);
 void erts_late_init_time_sup(void);
 
 /* timer-wheel api */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index d47d1682d7..b809fa8316 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -765,8 +765,7 @@ erts_has_time_correction(void)
     return time_sup.r.o.correction;
 }
 
-void
-erts_early_init_time_sup(void)
+void erts_init_sys_time_sup(void)
 {
     ErtsSysInitTimeResult sys_init_time_res
 	= ERTS_SYS_INIT_TIME_RESULT_INITER;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 7e64623686..a108e819c6 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -698,6 +698,7 @@ typedef struct {
 #define ERTS_SYS_INIT_TIME_RESULT_INITER \
     {0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}
 
+extern void erts_init_sys_time_sup(void);
 extern void sys_init_time(ErtsSysInitTimeResult *);
 extern void erts_deliver_time(void);
 extern void erts_time_remaining(SysTimeval *);
-- 
cgit v1.2.3


From d07319f790eb38c867bd04a23de674e2393825ff Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Sun, 22 Mar 2015 20:20:28 +0100
Subject: Better support for poor os monotonic sources

---
 erts/emulator/beam/erl_time.h     |  17 +--
 erts/emulator/beam/erl_time_sup.c | 237 +++++++++++++++++++++++++++-----------
 erts/emulator/beam/sys.h          |   2 +
 3 files changed, 174 insertions(+), 82 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 924b0b6091..35a1442dbd 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -219,6 +219,10 @@ erts_time_unit_conversion(Uint64 value,
  * it is assumed (and need) to be a power of 10.
  */
 
+#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT < 1000*1000
+#  error Compile time time unit needs to be at least 1000000
+#endif
+
 #define ERTS_MONOTONIC_TIME_UNIT \
     ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT)
 
@@ -248,19 +252,6 @@ erts_time_unit_conversion(Uint64 value,
 #define ERTS_USEC_TO_MONOTONIC__(USEC) ((ErtsMonotonicTime) (USEC))
 #define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/1000)
 
-#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000
-/* Milli-second time unit */
-
-#define ERTS_MONOTONIC_TO_SEC__(MSEC) ((USEC)/(1000))
-#define ERTS_MONOTONIC_TO_MSEC__(MSEC) (MSEC)
-#define ERTS_MONOTONIC_TO_USEC__(MSEC) ((MSEC)*1000)
-#define ERTS_MONOTONIC_TO_NSEC__(MSEC) ((MSEC)*(1000*1000))
-
-#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*1000)
-#define ERTS_MSEC_TO_MONOTONIC__(MSEC) ((ErtsMonotonicTime) (MSEC))
-#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))/1000)
-#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/(1000*1000))
-
 #else
 #error Missing implementation for monotonic time unit
 #endif
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index b809fa8316..d961fae81a 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -21,6 +21,8 @@
  * Support routines for the time
  */
 
+/* #define ERTS_TIME_CORRECTION_PRINT */
+
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
@@ -75,12 +77,12 @@ schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset);
 
 #else /* ARCH_64 */
 
-#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 1000*1000
+#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000
 
 /*
  * Using micro second time unit or lower. Start at zero since
  * time will remain an immediate for a very long time anyway
- * (18279 years in the micro second case)...
+ * (1827 years in the 10 micro second case)...
  */
 #define ERTS_MONOTONIC_TIME_START ((ErtsMonotonicTime) 0)
 
@@ -99,11 +101,16 @@ schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset);
 
 #endif /* ARCH_64 */
 
-#define ERTS_MONOTONIC_OFFSET_NATIVE ERTS_MONOTONIC_TIME_START
-#define ERTS_MONOTONIC_OFFSET_NSEC ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_TIME_START)
-#define ERTS_MONOTONIC_OFFSET_USEC ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_TIME_START)
-#define ERTS_MONOTONIC_OFFSET_MSEC ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_TIME_START)
-#define ERTS_MONOTONIC_OFFSET_SEC ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_TIME_START)
+#define ERTS_MONOTONIC_OFFSET_NATIVE \
+    (ERTS_MONOTONIC_TIME_START - ERTS_MONOTONIC_TIME_UNIT)
+#define ERTS_MONOTONIC_OFFSET_NSEC					\
+    ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
+#define ERTS_MONOTONIC_OFFSET_USEC					\
+    ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
+#define ERTS_MONOTONIC_OFFSET_MSEC					\
+    ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
+#define ERTS_MONOTONIC_OFFSET_SEC					\
+    ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
 
 #else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
 
@@ -120,25 +127,6 @@ schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset);
 
 #endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
 
-#define ERTS_MONOTONIC_TO_SYS_TIME_VAL(TVP, MT)				\
-    do {								\
-	ErtsMonotonicTime sec__, usec__;				\
-	sec__ = ERTS_MONOTONIC_TO_SEC((MT));				\
-	usec__ = ERTS_MONOTONIC_TO_USEC((MT)) - sec__*1000000;		\
-	ASSERT(usec__ < 1000000);					\
-	(TVP)->tv_sec = sec__;						\
-	(TVP)->tv_usec = usec__;					\
-    } while (0)
-
-#define ERTS_MAX_SYSTEM_TIME_DIFF ERTS_MSEC_TO_MONOTONIC(10)
-#define ERTS_SYSTEM_TIME_DIFF_EXCEED_LIMIT(ESYSTIME, OSSYSTIME)		\
-    (((Uint64) (ESYSTIME)) - (((Uint64) (OSSYSTIME))			\
-			      - ERTS_MAX_SYSTEM_TIME_DIFF)		\
-     > 2*ERTS_MAX_SYSTEM_TIME_DIFF)
-
-#define ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF (ERTS_MAX_SYSTEM_TIME_DIFF/2)
-#define ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(500)
-
 struct time_sup_read_only__ {
     ErtsMonotonicTime (*get_time)(void);
     int correction;
@@ -150,6 +138,7 @@ struct time_sup_read_only__ {
     char *os_monotonic_clock_id;
     int os_monotonic_locked;
     Uint64 os_monotonic_resolution;
+    Uint64 os_monotonic_extended;
 #endif
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
     ErtsMonotonicTime start;
@@ -161,6 +150,10 @@ struct time_sup_read_only__ {
 	ErtsMonotonicTime sec;
     } start_offset;
 #endif
+    struct {
+	ErtsMonotonicTime large_diff;
+	ErtsMonotonicTime small_diff;
+    } adj;
 };
 
 typedef struct {
@@ -338,13 +331,77 @@ static ErtsMonotonicTime get_corrected_time(void)
     return calc_corrected_erl_mtime(os_mtime, cip, NULL);
 }
 
+#ifdef ERTS_TIME_CORRECTION_PRINT
+
+static ERTS_INLINE void
+print_correction(int change,
+		 ErtsMonotonicTime sdiff,
+		 ErtsMonotonicTime old_ecorr,
+		 ErtsMonotonicTime old_dcorr,
+		 ErtsMonotonicTime new_ecorr,
+		 ErtsMonotonicTime new_dcorr,
+		 Uint tmo)
+{
+    ErtsMonotonicTime usec_sdiff;
+    if (sdiff < 0)
+	usec_sdiff = -1*ERTS_MONOTONIC_TO_USEC(-1*sdiff);
+    else
+	usec_sdiff = ERTS_MONOTONIC_TO_USEC(sdiff);
+
+    if (!change)
+	fprintf(stderr,
+		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppm] : "
+		"tmo = %lld msec\r\n",
+		(long long) usec_sdiff,
+		(long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
+		(long long) (1000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		(long long) tmo);
+    else
+	fprintf(stderr,
+		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppm] "
+		"-> [ec=%lld ppm, dc=%lld ppm] : tmo = %lld msec\r\n",
+		(long long) usec_sdiff,
+		(long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
+		(long long) (1000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		(long long) (1000000*new_ecorr) / ERTS_TCORR_ERR_UNIT,
+		(long long) (1000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		(long long) tmo);
+
+}
+
+#endif
+
 static void
 check_time_correction(void *unused)
 {
+#ifndef ERTS_TIME_CORRECTION_PRINT
+#  define ERTS_PRINT_CORRECTION
+#else
+#  ifdef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+#    define ERTS_PRINT_CORRECTION		\
+    print_correction(set_new_correction,	\
+		     sdiff,			\
+		     cip->correction.error,	\
+		     0,				\
+		     new_correction.error,	\
+		     0,				\
+		     timeout)
+#  else
+#    define ERTS_PRINT_CORRECTION		\
+    print_correction(set_new_correction,	\
+		     sdiff,			\
+		     cip->correction.error,	\
+		     cip->correction.drift,	\
+		     new_correction.error,	\
+		     new_correction.drift,	\
+		     timeout)
+#  endif
+#endif
     ErtsMonotonicCorrectionData cdata;
     ErtsMonotonicCorrection new_correction;
     ErtsMonotonicCorrectionInstance *cip;
-    ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime, erl_stime, time_offset;
+    ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime,
+	erl_stime, time_offset;
     Uint timeout;
     SysTimeval tod;
     int set_new_correction, begin_short_intervals = 0;
@@ -377,8 +434,8 @@ check_time_correction(void *unused)
     new_correction = cip->correction;
     
     if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE
-	&& (sdiff < -2*ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF
-	    || 2*ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF < sdiff)) {
+	&& (sdiff < -2*time_sup.r.o.adj.small_diff
+	    || 2*time_sup.r.o.adj.small_diff < sdiff)) {
 	/* System time diff exeeded limits; change time offset... */
 	time_offset -= sdiff;
 	sdiff = 0;
@@ -393,16 +450,16 @@ check_time_correction(void *unused)
 	}
     }
     else if (cdata.curr.correction.error == 0) {
-	if (sdiff < -ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	if (sdiff < -time_sup.r.o.adj.small_diff) {
 	    set_new_correction = 1;
-	    if (sdiff < -ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+	    if (sdiff < -time_sup.r.o.adj.large_diff)
 		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
 	    else
 		new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ;
 	}
-	else if (sdiff > ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	else if (sdiff > time_sup.r.o.adj.small_diff) {
 	    set_new_correction = 1;
-	    if (sdiff > ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+	    if (sdiff > time_sup.r.o.adj.large_diff)
 		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
 	    else
 		new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ;
@@ -414,16 +471,16 @@ check_time_correction(void *unused)
     else if (cdata.curr.correction.error > 0) {
 	if (sdiff < 0) {
 	    if (cdata.curr.correction.error == ERTS_TCORR_ERR_LARGE_ADJ
-		|| -ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF <= sdiff)
+		|| -time_sup.r.o.adj.large_diff <= sdiff)
 		set_new_correction = 0;
 	    else {
 		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
 		set_new_correction = 1;
 	    }
 	}
-	else if (sdiff > ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	else if (sdiff > time_sup.r.o.adj.small_diff) {
 	    set_new_correction = 1;
-	    if (sdiff > ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+	    if (sdiff > time_sup.r.o.adj.large_diff)
 		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
 	    else
 		new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ;
@@ -436,7 +493,7 @@ check_time_correction(void *unused)
     else /* if (cdata.curr.correction.error < 0) */ { 
 	if (0 < sdiff) {
 	    if (cdata.curr.correction.error == -ERTS_TCORR_ERR_LARGE_ADJ
-		|| sdiff <= ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+		|| sdiff <= time_sup.r.o.adj.large_diff)
 		set_new_correction = 0;
 	    else {
 		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
@@ -444,9 +501,9 @@ check_time_correction(void *unused)
 	    }
 	    set_new_correction = 0;
 	}
-	else if (sdiff < -ERTS_TIME_CORRECTION_SMALL_ADJ_DIFF) {
+	else if (sdiff < -time_sup.r.o.adj.small_diff) {
 	    set_new_correction = 1;
-	    if (sdiff < -ERTS_TIME_CORRECTION_LARGE_ADJ_DIFF)
+	    if (sdiff < -time_sup.r.o.adj.large_diff)
 		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
 	    else
 		new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ;
@@ -475,11 +532,13 @@ check_time_correction(void *unused)
 	    mtime_acc = ddp->acc.mon;
 	    stime_acc = ddp->acc.sys;
 
-	    avg_drift_adj = ((stime_acc - mtime_acc)*ERTS_MONOTONIC_TIME_UNIT) / mtime_acc;
+	    avg_drift_adj = (((stime_acc - mtime_acc)*ERTS_MONOTONIC_TIME_UNIT)
+			     / mtime_acc);
 
 	    mtime_diff = os_mtime - old_os_mtime;
 	    stime_diff = os_stime - old_os_stime;
-	    drift_adj = ((stime_diff - mtime_diff)*ERTS_MONOTONIC_TIME_UNIT) / mtime_diff;
+	    drift_adj = (((stime_diff - mtime_diff)*ERTS_MONOTONIC_TIME_UNIT)
+			 / mtime_diff);
 	    
 	    ix++;
 	    if (ix >= ERTS_DRIFT_INTERVALS)
@@ -499,10 +558,11 @@ check_time_correction(void *unused)
 	    ddp->acc.sys = stime_acc;
 
 	    /*
-	     * If calculated drift adjustment is if off by more than 20% from the
-	     * average drift we interpret this as a discontinous leap in system
-	     * time and ignore it. If it actually is a change in drift we will
-	     * later detect this when the average drift change.
+	     * If calculated drift adjustment is if off by more than 20%
+	     * from the average drift we interpret this as a discontinous
+	     * leap in system time and ignore it. If it actually is a
+	     * change in drift we will later detect this when the average
+	     * drift change.
 	     */
 	    drift_adj_diff = avg_drift_adj - drift_adj;
 	    if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF
@@ -512,7 +572,8 @@ check_time_correction(void *unused)
 	    }
 	    else {
 		if (ddp->dirty_counter <= 0) {
-		    drift_adj = ((stime_acc - mtime_acc)*ERTS_MONOTONIC_TIME_UNIT) / mtime_acc;
+		    drift_adj = ((stime_acc - mtime_acc)
+				 *ERTS_MONOTONIC_TIME_UNIT) / mtime_acc;
 		}
 		if (ddp->dirty_counter >= 0) {
 		    if (ddp->dirty_counter == 0) {
@@ -573,7 +634,9 @@ check_time_correction(void *unused)
 	&& time_sup.inf.c.parmon.cdata.short_check_interval) {
 	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK);
     }
-    
+
+    ERTS_PRINT_CORRECTION;
+
     if (set_new_correction) {
 	erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx);
 
@@ -610,6 +673,8 @@ check_time_correction(void *unused)
 		   NULL,
 		   NULL,
 		   timeout);
+
+#undef ERTS_PRINT_CORRECTION
 }
 
 #ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
@@ -618,7 +683,8 @@ static void
 init_check_time_correction(void *unused)
 {
     ErtsMonotonicDriftData *ddp;
-    ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff, stime_diff;
+    ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff,
+	stime_diff;
     int ix;
     SysTimeval tod;
 
@@ -690,6 +756,7 @@ static void
 late_init_time_correction(void)
 {
     if (time_sup.inf.c.finalized_offset) {
+
 	erts_init_timer(&time_sup.inf.c.parmon.timer);
 	erts_set_timer(&time_sup.inf.c.parmon.timer,
 #ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
@@ -792,6 +859,8 @@ void erts_init_sys_time_sup(void)
 	= sys_init_time_res.os_monotonic_info.locked_use;
     time_sup.r.o.os_monotonic_resolution
 	= sys_init_time_res.os_monotonic_info.resolution;
+    time_sup.r.o.os_monotonic_extended
+	= sys_init_time_res.os_monotonic_info.extended;
 #endif
 }
 
@@ -799,7 +868,7 @@ int
 erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 {
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
-    ErtsMonotonicTime abs_start;
+    ErtsMonotonicTime abs_native_offset, native_offset;
 #endif
 
     ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX);
@@ -822,42 +891,68 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
     time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
     time_sup.r.o.start += ERTS_MONOTONIC_TIME_UNIT;
-    abs_start = time_sup.r.o.start;
+    native_offset = time_sup.r.o.start - ERTS_MONOTONIC_TIME_UNIT;
+    native_offset = native_offset;
 #else /* ARCH_64 */
-    if (ERTS_MONOTONIC_TIME_UNIT <= 1000*1000)
-	abs_start = time_sup.r.o.start = 0;
+    if (ERTS_MONOTONIC_TIME_UNIT <= 10*1000*1000) {
+	time_sup.r.o.start = 0;
+	native_offset = -ERTS_MONOTONIC_TIME_UNIT;
+	abs_native_offset = ERTS_MONOTONIC_TIME_UNIT;
+    }
     else {
 	time_sup.r.o.start = ((ErtsMonotonicTime) MIN_SMALL);
 	time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
 	time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
-	abs_start = -1*time_sup.r.o.start;
+	native_offset = time_sup.r.o.start - ERTS_MONOTONIC_TIME_UNIT;
+	abs_native_offset = -1*native_offset;
     }
 #endif
 
-    time_sup.r.o.start_offset.native = time_sup.r.o.start;
+    time_sup.r.o.start_offset.native = (time_sup.r.o.start
+					- ERTS_MONOTONIC_TIME_UNIT);
     time_sup.r.o.start_offset.nsec = (ErtsMonotonicTime)
-	erts_time_unit_conversion((Uint64) abs_start,
+	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1000*1000*1000);
     time_sup.r.o.start_offset.usec = (ErtsMonotonicTime)
-	erts_time_unit_conversion((Uint64) abs_start,
+	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1000*1000);
     time_sup.r.o.start_offset.msec = (ErtsMonotonicTime)
-	erts_time_unit_conversion((Uint64) abs_start,
+	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1000);
     time_sup.r.o.start_offset.sec = (ErtsMonotonicTime)
-	erts_time_unit_conversion((Uint64) abs_start,
+	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1);
-    if (time_sup.r.o.start < 0) {
+    if (native_offset < 0) {
 	time_sup.r.o.start_offset.nsec *= -1;
 	time_sup.r.o.start_offset.usec *= -1;
 	time_sup.r.o.start_offset.msec *= -1;
 	time_sup.r.o.start_offset.sec *= -1;
     }
 
+#endif
+
+    time_sup.r.o.adj.large_diff = erts_time_sup__.r.o.monotonic_time_unit;
+    time_sup.r.o.adj.large_diff *= 50;
+    time_sup.r.o.adj.large_diff /= time_sup.r.o.os_monotonic_resolution;
+    if (time_sup.r.o.adj.large_diff < ERTS_MSEC_TO_MONOTONIC(5))
+	time_sup.r.o.adj.large_diff = ERTS_MSEC_TO_MONOTONIC(5);
+    time_sup.r.o.adj.small_diff = time_sup.r.o.adj.large_diff/10;
+
+#ifdef ERTS_TIME_CORRECTION_PRINT
+    fprintf(stderr, "start         = %lld\n\r", (long long) ERTS_MONOTONIC_TIME_START);
+    fprintf(stderr, "native offset = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_NATIVE);
+    fprintf(stderr, "nsec offset   = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_NSEC);
+    fprintf(stderr, "usec offset   = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_USEC);
+    fprintf(stderr, "msec offset   = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_MSEC);
+    fprintf(stderr, "sec offset    = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_SEC);
+    fprintf(stderr, "large diff    = %lld usec\r\n",
+	    (long long) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.large_diff));
+    fprintf(stderr, "small diff    = %lld usec\r\n",
+	    (long long) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.small_diff));
 #endif
 
     if (ERTS_MONOTONIC_TIME_UNIT < ERTS_CLKTCK_RESOLUTION)
@@ -878,6 +973,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	time_sup.r.o.moffset = -1*time_sup.inf.c.minit;
 	offset = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
 	offset += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	offset -= ERTS_MONOTONIC_TIME_UNIT;
 	init_time_offset(offset);
 
 	rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
@@ -897,7 +993,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	cdatap->curr.correction.drift = 0;
 #endif
 	cdatap->curr.correction.error = 0;
-	cdatap->curr.erl_mtime = 0;
+	cdatap->curr.erl_mtime = ERTS_MONOTONIC_TIME_UNIT;
 	cdatap->curr.os_mtime = time_sup.inf.c.minit;
 	cdatap->last_check = time_sup.inf.c.minit;
 	cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER;
@@ -910,9 +1006,10 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     {
 	ErtsMonotonicTime stime, offset;
 	time_sup.r.o.get_time = get_not_corrected_time;
+	sys_gettimeofday(&time_sup.inf.c.inittv);
 	stime = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
-	stime += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
-	offset = stime;
+ 	stime += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	offset = stime - ERTS_MONOTONIC_TIME_UNIT;
 	time_sup.inf.c.not_corrected_moffset = offset;
 	init_time_offset(offset);
 	time_sup.f.c.last_not_corrected_time = 0;
@@ -937,6 +1034,7 @@ erts_late_init_time_sup(void)
     if (time_sup.r.o.get_time == get_corrected_time)
 	late_init_time_correction();
 #endif
+    erts_late_sys_init_time();
 }
 
 ErtsTimeWarpMode erts_time_warp_mode(void)
@@ -1737,7 +1835,7 @@ make_time_val(Process *c_p, ErtsMonotonicTime time_val)
 Eterm
 erts_get_monotonic_start_time(struct process *c_p)
 {
-    return make_time_val(c_p, ERTS_MONOTONIC_OFFSET_NATIVE);
+    return make_time_val(c_p, ERTS_MONOTONIC_TIME_START);
 }
 
 static Eterm
@@ -1747,8 +1845,8 @@ bld_monotonic_time_source(Uint **hpp, Uint *szp, Sint64 os_mtime)
     return NIL;
 #else
     int i = 0;
-    Eterm k[5];
-    Eterm v[5];
+    Eterm k[6];
+    Eterm v[6];
 
     if (time_sup.r.o.os_monotonic_disable)
 	return NIL;
@@ -1761,10 +1859,11 @@ bld_monotonic_time_source(Uint **hpp, Uint *szp, Sint64 os_mtime)
 	v[i++] = erts_bld_atom(hpp, szp, time_sup.r.o.os_monotonic_clock_id);
     }
 
-    if (time_sup.r.o.os_monotonic_resolution) {
-	k[i] = erts_bld_atom(hpp, szp, "resolution");
-	v[i++] = erts_bld_uint64(hpp, szp, time_sup.r.o.os_monotonic_resolution);
-    }
+    k[i] = erts_bld_atom(hpp, szp, "resolution");
+    v[i++] = erts_bld_uint64(hpp, szp, time_sup.r.o.os_monotonic_resolution);
+
+    k[i] = erts_bld_atom(hpp, szp, "extended");
+    v[i++] = time_sup.r.o.os_monotonic_extended ? am_yes : am_no;
 
     k[i] = erts_bld_atom(hpp, szp, "parallel");
     v[i++] = time_sup.r.o.os_monotonic_locked ? am_no : am_yes;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index a108e819c6..d83bdf8d31 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -692,6 +692,7 @@ typedef struct {
 	char *func;
 	char *clock_id;
 	int locked_use;
+	int extended;
     } os_monotonic_info;
 } ErtsSysInitTimeResult;
 
@@ -700,6 +701,7 @@ typedef struct {
 
 extern void erts_init_sys_time_sup(void);
 extern void sys_init_time(ErtsSysInitTimeResult *);
+extern void erts_late_sys_init_time(void);
 extern void erts_deliver_time(void);
 extern void erts_time_remaining(SysTimeval *);
 extern int erts_init_time_sup(int, ErtsTimeWarpMode);
-- 
cgit v1.2.3


From 12bf6b0c82762744506e153720076524401c8512 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 23 Mar 2015 17:07:45 +0100
Subject: erts: Rename to flatmap_from_validated_list

from map_to_validated_list
---
 erts/emulator/beam/erl_map.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index bcbda65da0..26233a98c6 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -85,7 +85,7 @@ static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
-static Eterm map_from_validated_list(Process *p, Eterm list, Uint size);
+static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys);
 static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root);
@@ -277,7 +277,7 @@ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) {
 	if (size > MAP_SMALL_MAP_LIMIT) {
 	    BIF_RET(hashmap_from_validated_list(BIF_P, BIF_ARG_1, size));
 	} else {
-	    BIF_RET(map_from_validated_list(BIF_P, BIF_ARG_1, size));
+	    BIF_RET(flatmap_from_validated_list(BIF_P, BIF_ARG_1, size));
 	}
     }
 
@@ -286,7 +286,7 @@ error:
     BIF_ERROR(BIF_P, BADARG);
 }
 
-static Eterm map_from_validated_list(Process *p, Eterm list, Uint size) {
+static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size) {
     Eterm *kv, item = list;
     Eterm *hp, *thp,*vs, *ks, keys, res;
     flatmap_t *mp;
-- 
cgit v1.2.3


From e52829ceb614b36c31a650dea455a463fc490698 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Mar 2015 16:39:56 +0100
Subject: erts_sys_hrtime() for lcnt

---
 erts/emulator/beam/erl_lock_count.c | 13 +++++--------
 erts/emulator/beam/sys.h            |  1 -
 2 files changed, 5 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index ddeb56a6be..c6d8f4df95 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -104,16 +104,13 @@ static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) {
 }
 
 static void lcnt_time(erts_lcnt_time_t *time) {
-#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
-    ErtsMonotonicTime mtime = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time());
+    /*
+     * erts_sys_hrtime() is the highest resolution
+     * we could find, it may or may not be monotonic...
+     */
+    ErtsMonotonicTime mtime = erts_sys_hrtime();
     time->s  = (unsigned long) (mtime / 1000000000LL);
     time->ns = (unsigned long) (mtime - 1000000000LL*time->s);
-#else
-  SysTimeval tv;
-  sys_gettimeofday(&tv);
-  time->s  = tv.tv_sec;
-  time->ns = tv.tv_usec*1000LL;
-#endif
 }
 
 static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) {
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d83bdf8d31..c51b665db9 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -708,7 +708,6 @@ extern int erts_init_time_sup(int, ErtsTimeWarpMode);
 extern void erts_sys_init_float(void);
 extern void erts_thread_init_float(void);
 extern void erts_thread_disable_fpe(void);
-
 ERTS_GLB_INLINE int erts_block_fpe(void);
 ERTS_GLB_INLINE void erts_unblock_fpe(int);
 
-- 
cgit v1.2.3


From 18464e275987eaa69c627b1a2784a9deeb216c03 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Mar 2015 00:38:18 +0100
Subject: Fix zero timout timers

---
 erts/emulator/beam/time.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 9f997e1d0b..3dfd3f79d4 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -524,6 +524,7 @@ erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout,
 	tiw->nto++;
 	tiw->at_once.nto++;
 	*tiw->at_once.tail = p;
+	tiw->at_once.tail = &p->next;
 	p->next = NULL;
 	p->timeout_pos = timeout_pos;
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
-- 
cgit v1.2.3


From c20482023b70768bd84d25f1e34dbbc2fe09cf31 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Mar 2015 15:28:11 +0100
Subject: Better OS system time implementation

---
 erts/emulator/beam/erl_bif_info.c |   2 +
 erts/emulator/beam/erl_time.h     |   1 +
 erts/emulator/beam/erl_time_sup.c | 214 ++++++++++++++++++++++----------------
 erts/emulator/beam/sys.h          |  10 +-
 4 files changed, 134 insertions(+), 93 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 70062b2305..8491c01b75 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2120,6 +2120,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	}
     } else if (ERTS_IS_ATOM_STR("os_monotonic_time_source", BIF_ARG_1)) {
 	BIF_RET(erts_monotonic_time_source(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("os_system_time_source", BIF_ARG_1)) {
+	BIF_RET(erts_system_time_source(BIF_P));
     } else if (ERTS_IS_ATOM_STR("time_correction", BIF_ARG_1)) {
 	BIF_RET(erts_has_time_correction() ? am_true : am_false);
     } else if (ERTS_IS_ATOM_STR("start_time", BIF_ARG_1)) {
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 35a1442dbd..7f8c4e0ed5 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -148,6 +148,7 @@ ErtsTimeOffsetState erts_finalize_time_offset(void);
 struct process;
 Eterm erts_get_monotonic_start_time(struct process *c_p);
 Eterm erts_monotonic_time_source(struct process*c_p);
+Eterm erts_system_time_source(struct process*c_p);
 
 #ifdef SYS_CLOCK_RESOLUTION
 #define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000))
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index d961fae81a..ef39f4b5f4 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -133,12 +133,17 @@ struct time_sup_read_only__ {
     ErtsTimeWarpMode warp_mode;
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
     ErtsMonotonicTime moffset;
-    int os_monotonic_disable;
-    char *os_monotonic_func;
-    char *os_monotonic_clock_id;
-    int os_monotonic_locked;
-    Uint64 os_monotonic_resolution;
-    Uint64 os_monotonic_extended;
+    int os_monotonic_time_disable;
+    char *os_monotonic_time_func;
+    char *os_monotonic_time_clock_id;
+    int os_monotonic_time_locked;
+    Uint64 os_monotonic_time_resolution;
+    Uint64 os_monotonic_time_extended;
+    char *os_system_time_func;
+    char *os_system_time_clock_id;
+    int os_system_time_locked;
+    Uint64 os_system_time_resolution;
+    Uint64 os_system_time_extended;
 #endif
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
     ErtsMonotonicTime start;
@@ -209,7 +214,7 @@ struct time_sup_infrequently_changed__ {
     ErtsMonotonicTime minit;
 #endif
     int finalized_offset;
-    SysTimeval inittv; /* Used everywhere, the initial time-of-day */
+    ErtsSystemTime sinit;
     ErtsMonotonicTime not_corrected_moffset;
     erts_atomic64_t offset;
 };
@@ -242,10 +247,8 @@ ErtsTimeSupData erts_time_sup__ erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 erts_approx_time_t
 erts_get_approx_time(void)
 {
-    SysTimeval tv;
-    sys_gettimeofday(&tv);
-
-    return (erts_approx_time_t) tv.tv_sec;
+    ErtsSystemTime stime = erts_os_system_time();
+    return (erts_approx_time_t) ERTS_MONOTONIC_TO_SEC(stime);
 }
 
 static ERTS_INLINE void
@@ -403,7 +406,6 @@ check_time_correction(void *unused)
     ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime,
 	erl_stime, time_offset;
     Uint timeout;
-    SysTimeval tod;
     int set_new_correction, begin_short_intervals = 0;
 
     erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
@@ -411,7 +413,7 @@ check_time_correction(void *unused)
     ASSERT(time_sup.inf.c.finalized_offset);
 
     os_mtime = erts_os_monotonic_time();
-    sys_gettimeofday(&tod);
+    os_stime = erts_os_system_time();
 
     cdata = time_sup.inf.c.parmon.cdata;
 
@@ -426,9 +428,6 @@ check_time_correction(void *unused)
     time_offset = get_time_offset();
     erl_stime = erl_mtime + time_offset;
 
-    os_stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
-    os_stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
-
     sdiff = erl_stime - os_stime;
 
     new_correction = cip->correction;
@@ -686,7 +685,6 @@ init_check_time_correction(void *unused)
     ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff,
 	stime_diff;
     int ix;
-    SysTimeval tod;
 
     ddp = &time_sup.inf.c.parmon.cdata.drift;
     ix = ddp->ix;
@@ -694,10 +692,7 @@ init_check_time_correction(void *unused)
     old_stime = ddp->intervals[0].time.sys;
 
     mtime = erts_os_monotonic_time();
-    sys_gettimeofday(&tod);
-
-    stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
-    stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
+    stime = erts_os_system_time();
 
     mtime_diff = mtime - old_mtime;
     stime_diff = stime - old_stime;
@@ -729,7 +724,7 @@ init_check_time_correction(void *unused)
 #endif
 
 static ErtsMonotonicTime
-finalize_corrected_time_offset(SysTimeval *todp)
+finalize_corrected_time_offset(ErtsSystemTime *stimep)
 {
     ErtsMonotonicTime os_mtime;
     ErtsMonotonicCorrectionData cdata;
@@ -738,7 +733,7 @@ finalize_corrected_time_offset(SysTimeval *todp)
     erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
     os_mtime = erts_os_monotonic_time();
-    sys_gettimeofday(todp);
+    *stimep = erts_os_system_time();
 
     cdata = time_sup.inf.c.parmon.cdata;
 
@@ -774,15 +769,11 @@ late_init_time_correction(void)
 
 static ErtsMonotonicTime get_not_corrected_time(void)
 {
-    SysTimeval tmp_tv;
     ErtsMonotonicTime stime, mtime;
 
     erts_smp_mtx_lock(&erts_get_time_mtx);
 
-    sys_gettimeofday(&tmp_tv);
-
-    stime = ERTS_SEC_TO_MONOTONIC(tmp_tv.tv_sec);
-    stime += ERTS_USEC_TO_MONOTONIC(tmp_tv.tv_usec);
+    stime = erts_os_system_time();
 
     mtime = stime - time_sup.inf.c.not_corrected_moffset;
 
@@ -820,7 +811,7 @@ int erts_check_time_adj_support(int time_correction,
     /* User wants time correction */
 
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
-    return !time_sup.r.o.os_monotonic_disable;
+    return !time_sup.r.o.os_monotonic_time_disable;
 #else
     return 0;
 #endif
@@ -849,24 +840,33 @@ void erts_init_sys_time_sup(void)
 #endif
 
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
-    time_sup.r.o.os_monotonic_disable
-	= !sys_init_time_res.have_os_monotonic;
-    time_sup.r.o.os_monotonic_func
-	= sys_init_time_res.os_monotonic_info.func;
-    time_sup.r.o.os_monotonic_clock_id
-	= sys_init_time_res.os_monotonic_info.clock_id;
-    time_sup.r.o.os_monotonic_locked
-	= sys_init_time_res.os_monotonic_info.locked_use;
-    time_sup.r.o.os_monotonic_resolution
-	= sys_init_time_res.os_monotonic_info.resolution;
-    time_sup.r.o.os_monotonic_extended
-	= sys_init_time_res.os_monotonic_info.extended;
+    time_sup.r.o.os_monotonic_time_disable
+	= !sys_init_time_res.have_os_monotonic_time;
+    time_sup.r.o.os_monotonic_time_func
+	= sys_init_time_res.os_monotonic_time_info.func;
+    time_sup.r.o.os_monotonic_time_clock_id
+	= sys_init_time_res.os_monotonic_time_info.clock_id;
+    time_sup.r.o.os_monotonic_time_locked
+	= sys_init_time_res.os_monotonic_time_info.locked_use;
+    time_sup.r.o.os_monotonic_time_resolution
+	= sys_init_time_res.os_monotonic_time_info.resolution;
+    time_sup.r.o.os_monotonic_time_extended
+	= sys_init_time_res.os_monotonic_time_info.extended;
+    time_sup.r.o.os_system_time_func
+	= sys_init_time_res.os_system_time_info.func;
+    time_sup.r.o.os_system_time_clock_id
+	= sys_init_time_res.os_system_time_info.clock_id;
+    time_sup.r.o.os_system_time_locked
+	= sys_init_time_res.os_system_time_info.locked_use;
+    time_sup.r.o.os_system_time_resolution
+	= sys_init_time_res.os_system_time_info.resolution;
 #endif
 }
 
 int 
 erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 {
+    ErtsMonotonicTime resolution;
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
     ErtsMonotonicTime abs_native_offset, native_offset;
 #endif
@@ -935,11 +935,15 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
 #endif
 
+    resolution = time_sup.r.o.os_monotonic_time_resolution;
+    if (resolution > time_sup.r.o.os_system_time_resolution)
+	resolution = time_sup.r.o.os_system_time_resolution;
+
     time_sup.r.o.adj.large_diff = erts_time_sup__.r.o.monotonic_time_unit;
     time_sup.r.o.adj.large_diff *= 50;
-    time_sup.r.o.adj.large_diff /= time_sup.r.o.os_monotonic_resolution;
-    if (time_sup.r.o.adj.large_diff < ERTS_MSEC_TO_MONOTONIC(5))
-	time_sup.r.o.adj.large_diff = ERTS_MSEC_TO_MONOTONIC(5);
+    time_sup.r.o.adj.large_diff /= resolution;
+    if (time_sup.r.o.adj.large_diff < ERTS_USEC_TO_MONOTONIC(500))
+	time_sup.r.o.adj.large_diff = ERTS_USEC_TO_MONOTONIC(500);
     time_sup.r.o.adj.small_diff = time_sup.r.o.adj.large_diff/10;
 
 #ifdef ERTS_TIME_CORRECTION_PRINT
@@ -961,7 +965,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 #ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
     time_sup.r.o.correction = 0;
 #else
-    if (time_sup.r.o.os_monotonic_disable)
+    if (time_sup.r.o.os_monotonic_time_disable)
 	time_sup.r.o.correction = 0;
 
     if (time_sup.r.o.correction) {
@@ -969,10 +973,9 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	ErtsMonotonicTime offset;
 	time_sup.inf.c.minit = erts_os_monotonic_time();
-	sys_gettimeofday(&time_sup.inf.c.inittv);
+	time_sup.inf.c.sinit = erts_os_system_time();
 	time_sup.r.o.moffset = -1*time_sup.inf.c.minit;
-	offset = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
-	offset += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	offset = time_sup.inf.c.sinit;
 	offset -= ERTS_MONOTONIC_TIME_UNIT;
 	init_time_offset(offset);
 
@@ -985,10 +988,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	cdatap = &time_sup.inf.c.parmon.cdata;
     
 #ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
-	cdatap->drift.intervals[0].time.sys
-	    = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
-	cdatap->drift.intervals[0].time.sys
-	    += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	cdatap->drift.intervals[0].time.sys = time_sup.inf.c.sinit;
 	cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit;
 	cdatap->curr.correction.drift = 0;
 #endif
@@ -1006,9 +1006,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     {
 	ErtsMonotonicTime stime, offset;
 	time_sup.r.o.get_time = get_not_corrected_time;
-	sys_gettimeofday(&time_sup.inf.c.inittv);
-	stime = ERTS_SEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_sec);
- 	stime += ERTS_USEC_TO_MONOTONIC(time_sup.inf.c.inittv.tv_usec);
+	stime = time_sup.inf.c.sinit = erts_os_system_time();
 	offset = stime - ERTS_MONOTONIC_TIME_UNIT;
 	time_sup.inf.c.not_corrected_moffset = offset;
 	init_time_offset(offset);
@@ -1085,17 +1083,12 @@ erts_finalize_time_offset(void)
 
 	if (!time_sup.inf.c.finalized_offset) {
 	    ErtsMonotonicTime mtime, new_offset;
-	    SysTimeval tv;
 
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
 	    if (!time_sup.r.o.correction)
 #endif
 	    {
-		ErtsMonotonicTime stime;
-		sys_gettimeofday(&tv);
-
-		stime = ERTS_SEC_TO_MONOTONIC(tv.tv_sec);
-		stime += ERTS_USEC_TO_MONOTONIC(tv.tv_usec);
+		ErtsMonotonicTime stime = erts_os_system_time();
 
 		mtime = stime - time_sup.inf.c.not_corrected_moffset;
 
@@ -1114,11 +1107,9 @@ erts_finalize_time_offset(void)
 	    }
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
 	    else {
-		mtime = finalize_corrected_time_offset(&tv);
-		new_offset = ERTS_SEC_TO_MONOTONIC(tv.tv_sec);
-		new_offset += ERTS_USEC_TO_MONOTONIC(tv.tv_usec);
-		new_offset -= mtime;
-
+		ErtsSystemTime stime;
+		mtime = finalize_corrected_time_offset(&stime);
+		new_offset = stime - mtime;
 	    }
 #endif
 	    new_offset = ERTS_MONOTONIC_TO_USEC(new_offset);
@@ -1613,13 +1604,16 @@ erts_get_monotonic_time(void)
 void
 get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
 {
-    SysTimeval now;
-    
-    sys_gettimeofday(&now);
-    
-    *megasec = (Uint) (now.tv_sec / 1000000);
-    *sec = (Uint) (now.tv_sec % 1000000);
-    *microsec = (Uint) (now.tv_usec);
+    ErtsSystemTime stime = erts_os_system_time();
+    ErtsSystemTime ms, s, us;
+
+    us = ERTS_MONOTONIC_TO_USEC(stime);
+    s = us / (1000*1000);
+    ms = s / (1000*1000);
+
+    *megasec = (Uint) ms;
+    *sec = (Uint) (s - ms*(1000*1000));
+    *microsec = (Uint) (us - s*(1000*1000));
 }
 
 #ifdef HAVE_ERTS_NOW_CPU
@@ -1848,25 +1842,28 @@ bld_monotonic_time_source(Uint **hpp, Uint *szp, Sint64 os_mtime)
     Eterm k[6];
     Eterm v[6];
 
-    if (time_sup.r.o.os_monotonic_disable)
+    if (time_sup.r.o.os_monotonic_time_disable)
 	return NIL;
 
     k[i] = erts_bld_atom(hpp, szp, "function");
-    v[i++] = erts_bld_atom(hpp, szp, time_sup.r.o.os_monotonic_func);
+    v[i++] = erts_bld_atom(hpp, szp,
+			   time_sup.r.o.os_monotonic_time_func);
 
-    if (time_sup.r.o.os_monotonic_clock_id) {
+    if (time_sup.r.o.os_monotonic_time_clock_id) {
 	k[i] = erts_bld_atom(hpp, szp, "clock_id");
-	v[i++] = erts_bld_atom(hpp, szp, time_sup.r.o.os_monotonic_clock_id);
+	v[i++] = erts_bld_atom(hpp, szp,
+			       time_sup.r.o.os_monotonic_time_clock_id);
     }
 
     k[i] = erts_bld_atom(hpp, szp, "resolution");
-    v[i++] = erts_bld_uint64(hpp, szp, time_sup.r.o.os_monotonic_resolution);
+    v[i++] = erts_bld_uint64(hpp, szp,
+			     time_sup.r.o.os_monotonic_time_resolution);
 
     k[i] = erts_bld_atom(hpp, szp, "extended");
-    v[i++] = time_sup.r.o.os_monotonic_extended ? am_yes : am_no;
+    v[i++] = time_sup.r.o.os_monotonic_time_extended ? am_yes : am_no;
 
     k[i] = erts_bld_atom(hpp, szp, "parallel");
-    v[i++] = time_sup.r.o.os_monotonic_locked ? am_no : am_yes;
+    v[i++] = time_sup.r.o.os_monotonic_time_locked ? am_no : am_yes;
 
     k[i] = erts_bld_atom(hpp, szp, "time");
     v[i++] = erts_bld_sint64(hpp, szp, os_mtime);
@@ -1882,7 +1879,7 @@ erts_monotonic_time_source(struct process *c_p)
     Eterm *hp = NULL;
     Sint64 os_mtime = 0;
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
-    if (!time_sup.r.o.os_monotonic_disable)
+    if (!time_sup.r.o.os_monotonic_time_disable)
 	os_mtime = (Sint64) erts_os_monotonic_time();
 #endif
 
@@ -1892,6 +1889,49 @@ erts_monotonic_time_source(struct process *c_p)
     return bld_monotonic_time_source(&hp, NULL, os_mtime);
 }
 
+static Eterm
+bld_system_time_source(Uint **hpp, Uint *szp, Sint64 os_stime)
+{
+    int i = 0;
+    Eterm k[5];
+    Eterm v[5];
+
+    k[i] = erts_bld_atom(hpp, szp, "function");
+    v[i++] = erts_bld_atom(hpp, szp,
+			   time_sup.r.o.os_system_time_func);
+
+    if (time_sup.r.o.os_system_time_clock_id) {
+	k[i] = erts_bld_atom(hpp, szp, "clock_id");
+	v[i++] = erts_bld_atom(hpp, szp,
+			       time_sup.r.o.os_system_time_clock_id);
+    }
+
+    k[i] = erts_bld_atom(hpp, szp, "resolution");
+    v[i++] = erts_bld_uint64(hpp, szp,
+			     time_sup.r.o.os_system_time_resolution);
+
+    k[i] = erts_bld_atom(hpp, szp, "parallel");
+    v[i++] = am_yes;
+
+    k[i] = erts_bld_atom(hpp, szp, "time");
+    v[i++] = erts_bld_sint64(hpp, szp, os_stime);
+
+    return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v);
+}
+
+Eterm
+erts_system_time_source(struct process *c_p)
+{
+    Uint hsz = 0;
+    Eterm *hp = NULL;
+    Sint64 os_stime = (Sint64) erts_os_system_time();
+
+    bld_system_time_source(NULL, &hsz, os_stime);
+    if (hsz)
+	hp = HAlloc(c_p, hsz);
+    return bld_system_time_source(&hp, NULL, os_stime);
+}
+
 
 #include "bif.h"
 
@@ -2066,21 +2106,13 @@ BIF_RETTYPE timestamp_0(BIF_ALIST_0)
 
 BIF_RETTYPE os_system_time_0(BIF_ALIST_0)
 {
-    ErtsMonotonicTime stime;
-    SysTimeval tod;
-    sys_gettimeofday(&tod);
-    stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
-    stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
+    ErtsSystemTime stime = erts_os_system_time();
     BIF_RET(make_time_val(BIF_P, stime));
 }
 
 BIF_RETTYPE os_system_time_1(BIF_ALIST_0)
 {
-    ErtsMonotonicTime stime;
-    SysTimeval tod;
-    sys_gettimeofday(&tod);
-    stime = ERTS_SEC_TO_MONOTONIC(tod.tv_sec);
-    stime += ERTS_USEC_TO_MONOTONIC(tod.tv_usec);
+    ErtsSystemTime stime = erts_os_system_time();
     BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0));
 }
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c51b665db9..27a55e0ec7 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -684,7 +684,7 @@ typedef enum {
 } ErtsTimeWarpMode;
 
 typedef struct {
-    int have_os_monotonic;
+    int have_os_monotonic_time;
     ErtsMonotonicTime os_monotonic_time_unit;
     ErtsMonotonicTime sys_clock_resolution;
     struct {
@@ -693,7 +693,13 @@ typedef struct {
 	char *clock_id;
 	int locked_use;
 	int extended;
-    } os_monotonic_info;
+    } os_monotonic_time_info;
+    struct {
+	Uint64 resolution;
+	char *func;
+	char *clock_id;
+	int locked_use;
+    } os_system_time_info;
 } ErtsSysInitTimeResult;
 
 #define ERTS_SYS_INIT_TIME_RESULT_INITER \
-- 
cgit v1.2.3


From a1520d8bd2b467d5128998a5069611b9e6252653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 24 Mar 2015 17:05:58 +0100
Subject: erts: Fix comparison of exact terms

Comparison of exact terms could cause faulty term tests.
This was caused by a faulty (too small) internal type.

Symptom:

   -1 = erts_internal:cmp_term(2147483648,0). %% wrong

Correct:

   1 = erts_internal:cmp_term(2147483648,0).

Reported-by: Jesper Louis Andersen
---
 erts/emulator/beam/bif.c     | 2 +-
 erts/emulator/beam/erl_map.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 42dd160e38..5ac74032fa 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4659,7 +4659,7 @@ BIF_RETTYPE bump_reductions_1(BIF_ALIST_1)
 }
 
 BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) {
-    int res = CMP_TERM(BIF_ARG_1,BIF_ARG_2);
+    Sint res = CMP_TERM(BIF_ARG_1,BIF_ARG_2);
 
     /* ensure -1, 0, 1 result */
     if (res < 0) {
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 5e740aacdd..a180047f6c 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -400,7 +400,7 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
 	Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2;
 	map_t *mp1,*mp2,*mp_new;
 	Uint n1,n2,i1,i2,need,unused_size=0;
-	int c = 0;
+	Sint c = 0;
 
 	mp1  = (map_t*)map_val(BIF_ARG_1);
 	mp2  = (map_t*)map_val(BIF_ARG_2);
@@ -798,7 +798,7 @@ int erts_validate_and_sort_map(map_t* mp)
     Uint   sz  = map_get_size(mp);
     Uint   ix,jx;
     Eterm  tmp;
-    int c;
+    Sint c;
 
     /* sort */
 
-- 
cgit v1.2.3


From 1f23b603f022c7345dd573bf917926a575ae030d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 24 Mar 2015 20:01:11 +0100
Subject: erts: Fix bug in binary_to_term for hamt when yielding

Must save hamt_list in context.
---
 erts/emulator/beam/external.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 82c60840e5..c99b60ed09 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3878,6 +3878,7 @@ dec_term_atom_common:
                     ctx->u.dc.next = next;
                     ctx->u.dc.hp = hp;
 		    ctx->u.dc.maps_list = maps_list;
+		    ctx->u.dc.hamt_list = hamt_list;
                     ctx->reds = 0;
                     return NULL;
                 }
-- 
cgit v1.2.3


From db54eaa94562b49c81b677948a8e9139ebdb010e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 24 Mar 2015 15:27:11 +0100
Subject: erts: Remove HAMT_SUBTAG_NODE_ARRAY

This will also fix a bug in term_to_binary
treating full nodes as tuples and emiting LIST_EXT for leafs.
---
 erts/emulator/beam/erl_map.c  | 88 ++++++-------------------------------------
 erts/emulator/beam/erl_map.h  |  6 ---
 erts/emulator/beam/external.c |  6 +--
 erts/emulator/beam/utils.c    |  5 +--
 4 files changed, 15 insertions(+), 90 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 26233a98c6..ef806c2cfa 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -775,7 +775,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
 		sz    = hashmap_bitcount(hdr);
 		hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
 		nhp   = hp;
-		*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+		*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr);
 		*hp++ = res; sz--;
 		while (sz--) { *hp++ = ESTACK_POP(stack); }
 		ASSERT((hp - nhp) < 18);
@@ -824,7 +824,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
 	sz    = hashmap_bitcount(hdr);
 	hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA);
 	nhp   = hp;
-	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+	*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr);
 	*hp++ = res; sz--;
 
 	while (sz--) { *hp++ = ESTACK_POP(stack); }
@@ -846,7 +846,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
 	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr);
 	*hp++ = n;
     } else {
-	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_NODE_ARRAY : MAP_HEADER_HAMT_NODE_BITMAP(hdr);
+	*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr);
     }
 
     *hp++ = res; sz--;
@@ -1163,8 +1163,8 @@ recurse:
 	    sp->abm = 1 << hashmap_index(ahx);
 	    sp->srcA = &nodeA;
 	    switch(hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+                sp->srcB++;
 		sp->bbm = 0xffff;
 		break;
 
@@ -1189,16 +1189,16 @@ recurse:
 	hdrA = *sp->srcA++;
 	ASSERT(is_header(hdrA));
 	switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
-	case HAMT_SUBTAG_HEAD_ARRAY: sp->srcA++;
-	case HAMT_SUBTAG_NODE_ARRAY: {
+	case HAMT_SUBTAG_HEAD_ARRAY: {
+            sp->srcA++;
 	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
 	    sp->abm = 0xffff;
 	    sp->srcB = boxed_val(nodeB);
 	    hdrB = *sp->srcB++;
 	    ASSERT(is_header(hdrB));
 	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+                sp->srcB++;
 		sp->bbm = 0xffff;
 		break;
 	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
@@ -1218,8 +1218,8 @@ recurse:
 	    hdrB = *sp->srcB++;
 	    ASSERT(is_header(hdrB));
 	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+                sp->srcB++;
 		sp->bbm = 0xffff;
 		break;
 	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
@@ -1296,8 +1296,7 @@ recurse:
 	} else {
 	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
 	    hp = nhp;
-	    *hp++ = (sp->ix == 16 ? make_arityval(16)
-		     : MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm));
+	    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm);
 	}
 	memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
 	res = make_boxed(nhp);
@@ -1748,7 +1747,6 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
 	    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
 	    case HAMT_SUBTAG_HEAD_ARRAY:
 		ptr++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
 		sz = 16;
 		break;
 	    case HAMT_SUBTAG_HEAD_BITMAP:
@@ -1799,7 +1797,6 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
 	    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
 	    case HAMT_SUBTAG_HEAD_ARRAY:
 		ptr++;
-	    case HAMT_SUBTAG_NODE_ARRAY:
 		sz = 16;
 		break;
 	    case HAMT_SUBTAG_HEAD_BITMAP:
@@ -1862,11 +1859,6 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
                 ASSERT(is_header(hdr));
 
                 switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-                    case HAMT_SUBTAG_NODE_ARRAY:
-                        ix   = hashmap_index(hx);
-                        hx   = hashmap_shift_hash(th,hx,lvl,key);
-                        node = ptr[ix+1];
-                        break;
                     case HAMT_SUBTAG_HEAD_ARRAY:
                         ix   = hashmap_index(hx);
                         hx   = hashmap_shift_hash(th,hx,lvl,key);
@@ -1964,13 +1956,6 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 		ASSERT(is_header(hdr));
 
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(th,hx,lvl,key);
-			size += HAMT_NODE_ARRAY_SZ;
-			ESTACK_PUSH2(*sp, ix, node);
-			node  = ptr[ix+1];
-			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			ix    = hashmap_index(hx);
 			hx    = hashmap_shift_hash(th,hx,lvl,key);
@@ -2100,14 +2085,6 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 		ASSERT(is_header(hdr));
 
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			slot  = (Uint) ESTACK_POP(*sp);
-			nhp   = hp;
-			n     = HAMT_NODE_ARRAY_SZ;
-			while(n--) { *hp++ = *ptr++; }
-			nhp[slot+1] = res;
-			res = make_hashmap(nhp);
-			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			slot  = (Uint) ESTACK_POP(*sp);
 			nhp   = hp;
@@ -2132,9 +2109,6 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 			if (hval & bp) { ptr++; n--; }
 			while(n--) { *hp++ = *ptr++; }
 
-			if ((hval | bp) == 0xffff) {
-			    *nhp = make_arityval(16);
-			}
 			res = make_hashmap(nhp);
 			break;
 		    case HAMT_SUBTAG_HEAD_BITMAP:
@@ -2230,13 +2204,6 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
 		ASSERT(is_header(hdr));
 
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			ix    = hashmap_index(hx);
-			hx    = hashmap_shift_hash(th,hx,lvl,key);
-			size += HAMT_NODE_ARRAY_SZ;
-			ESTACK_PUSH2(stack, ix, node);
-			node  = ptr[ix+1];
-			break;
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			ix    = hashmap_index(hx);
 			hx    = hashmap_shift_hash(th,hx,lvl,key);
@@ -2351,24 +2318,6 @@ unroll:
 	ASSERT(is_header(hdr));
 
 	switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_NODE_ARRAY:
-		ix  = (Uint) ESTACK_POP(stack);
-		nhp = hp;
-		if (res == THE_NON_VALUE) {
-		    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(0xffff ^ (1 << ix)); ptr++;
-		    n     = 16;
-		    n    -= ix;
-		    while(ix--) { *hp++ = *ptr++; }
-		    ptr++; n--;
-		    while(n--) { *hp++ = *ptr++; }
-		    res = make_hashmap(nhp);
-		} else {
-		    n = HAMT_NODE_ARRAY_SZ;
-		    while(n--) { *hp++ = *ptr++; }
-		    nhp[ix+1] = res;
-		    res = make_hashmap(nhp);
-		}
-		break;
 	    case HAMT_SUBTAG_HEAD_ARRAY:
 		ix  = (Uint) ESTACK_POP(stack);
 		nhp = hp;
@@ -2610,7 +2559,6 @@ BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
             case HAMT_SUBTAG_HEAD_ARRAY:
             case HAMT_SUBTAG_HEAD_BITMAP:
                 BIF_RET(AM_hashmap);
-            case HAMT_SUBTAG_NODE_ARRAY:
             case HAMT_SUBTAG_NODE_BITMAP:
                 BIF_RET(AM_hashmap_node);
             default:
@@ -2637,10 +2585,6 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) {
         ASSERT(is_header(hdr));
 
         switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-            case HAMT_SUBTAG_NODE_ARRAY:
-                sz   = 16;
-                ptr += 1;
-                break;
             case HAMT_SUBTAG_NODE_BITMAP:
                 sz   = hashmap_bitcount(MAP_HEADER_VAL(hdr));
                 ptr += 1;
@@ -2703,14 +2647,6 @@ static Eterm hashmap_info(Process *p, Eterm node) {
 		hdr = *ptr;
 		ASSERT(is_header(hdr));
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		    case HAMT_SUBTAG_NODE_ARRAY:
-			narray++;
-			sz = 16;
-			while(sz--) {
-			    ESTACK_PUSH(stack, clvl + 1);
-			    ESTACK_PUSH(stack, ptr[sz+1]);
-			}
-			break;
 		    case HAMT_SUBTAG_NODE_BITMAP:
 			nbitmap++;
 			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 1333a734a8..9fc1a72b68 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -178,21 +178,15 @@ typedef struct hashmap_head_s {
 #define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \
     MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp)
 
-#define MAP_HEADER_HAMT_NODE_ARRAY \
-    make_arityval(16)
-
 #define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \
     MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp)
 
 #define HAMT_HEAD_EMPTY_SZ     (2)
-#define HAMT_NODE_ARRAY_SZ     (17)
 #define HAMT_HEAD_ARRAY_SZ     (18)
 #define HAMT_NODE_BITMAP_SZ(n) (1 + n)
 #define HAMT_HEAD_BITMAP_SZ(n) (2 + n)
 
 #define _HEADER_MAP_SUBTAG_MASK    (0xfc) /* 2 bits maps tag + 4 bits subtag + 2 ignore bits */
-/* SUBTAG_NODE_ARRAY is in fact a tuple with 16 elements */
-#define HAMT_SUBTAG_NODE_ARRAY   (((16 << _HEADER_ARITY_OFFS) | ARITYVAL_SUBTAG) & _HEADER_MAP_SUBTAG_MASK)
 #define HAMT_SUBTAG_NODE_BITMAP  ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
 #define HAMT_SUBTAG_HEAD_ARRAY   ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
 #define HAMT_SUBTAG_HEAD_BITMAP  ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c99b60ed09..b0b232f185 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2633,8 +2633,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    *ep++ = MAP_EXT;
 		    ptr++;
 		    put_int32(*ptr, ep); ep += 4;
-		    /*fall through*/
-		case HAMT_SUBTAG_NODE_ARRAY:
 		    node_sz = 16;
 		    break;
 		case HAMT_SUBTAG_HEAD_BITMAP:
@@ -4172,8 +4170,8 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		hdr = *ptr;
 		ASSERT(is_header(hdr));
 		switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-		case HAMT_SUBTAG_HEAD_ARRAY: ptr++;
-		case HAMT_SUBTAG_NODE_ARRAY:
+		case HAMT_SUBTAG_HEAD_ARRAY:
+		    ptr++;
 		    node_sz = 16;
 		    break;
 		case HAMT_SUBTAG_HEAD_BITMAP: ptr++;
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 3549e18538..d98251addc 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1321,7 +1321,6 @@ make_hash2(Eterm term)
                 }
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
                 case HAMT_SUBTAG_HEAD_ARRAY:
-                case HAMT_SUBTAG_NODE_ARRAY:
                     i = 16;
                     break;
                 case HAMT_SUBTAG_HEAD_BITMAP:
@@ -1724,7 +1723,6 @@ make_internal_hash(Eterm term)
                 }
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
                 case HAMT_SUBTAG_HEAD_ARRAY:
-                case HAMT_SUBTAG_NODE_ARRAY:
                     i = 16;
                     break;
                 case HAMT_SUBTAG_HEAD_BITMAP:
@@ -2797,14 +2795,13 @@ tailrecur_ne:
 		    switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			aa++; bb++;
-		    case HAMT_SUBTAG_NODE_ARRAY:
 			sz = 16;
 			break;
 		    case HAMT_SUBTAG_HEAD_BITMAP:
 			aa++; bb++;
 		    case HAMT_SUBTAG_NODE_BITMAP:
 			sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-			ASSERT(sz > 0 && sz < 16);
+			ASSERT(sz > 0 && sz < 17);
 			break;
 		    default:
 			erl_exit(1, "Unknown hashmap subsubtag\n");
-- 
cgit v1.2.3


From 8d31ecea8b68ef6e16d7d77c0160e36f078b98de Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 24 Mar 2015 15:03:46 +0100
Subject: erts: Optimize hashmap_get

---
 erts/emulator/beam/erl_map.c | 96 +++++++++++++++++---------------------------
 1 file changed, 36 insertions(+), 60 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ef806c2cfa..3852366876 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1837,75 +1837,51 @@ erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base)
 erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
 #endif
 {
-    Eterm *ptr, hdr;
-    Uint ix,slot, lvl = 0;
+    Eterm *ptr, hdr, *res;
+    Uint ix, lvl = 0;
     Uint32 hval,bp;
     DeclareTmpHeapNoproc(th,2);
     UseTmpHeapNoproc(2);
 
+    ASSERT(is_boxed(node));
+    ptr = boxed_val(node);
+    hdr = *ptr;
+    ASSERT(is_header(hdr));
+    ASSERT(is_hashmap_header_head(hdr));
+    ptr++;
+
     for (;;) {
-        switch(primary_tag(node)) {
-            case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
-                ptr = list_val(node);
-                UnUseTmpHeapNoproc(2);
-
-                if (eq_rel(CAR(ptr), map_base, key, NULL)) {
-                    return &(CDR(ptr));
-                }
-                return NULL;
-            case TAG_PRIMARY_BOXED:
-                ptr = boxed_val(node);
-                hdr = *ptr;
-                ASSERT(is_header(hdr));
-
-                switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-                    case HAMT_SUBTAG_HEAD_ARRAY:
-                        ix   = hashmap_index(hx);
-                        hx   = hashmap_shift_hash(th,hx,lvl,key);
-                        node = ptr[ix+2];
-                        break;
-                    case HAMT_SUBTAG_NODE_BITMAP:
-                        hval = MAP_HEADER_VAL(hdr);
-                        ix   = hashmap_index(hx);
-                        bp   = 1 << ix;
-                        slot = hashmap_bitcount(hval & (bp - 1));
-
-                        /* occupied */
-                        if (bp & hval) {
-                            hx    = hashmap_shift_hash(th,hx,lvl,key);
-                            node  = ptr[slot+1];
-                            break;
-                        }
-                        /* not occupied */
-                        UnUseTmpHeapNoproc(2);
-                        return NULL;
-                    case HAMT_SUBTAG_HEAD_BITMAP:
-                        hval = MAP_HEADER_VAL(hdr);
-                        ix   = hashmap_index(hx);
-                        bp   = 1 << ix;
-                        slot = hashmap_bitcount(hval & (bp - 1));
-
-                        /* occupied */
-                        if (bp & hval) {
-                            hx    = hashmap_shift_hash(th,hx,lvl,key);
-                            node  = ptr[slot+2];
-                            break;
-                        }
-                        /* not occupied */
-                        UnUseTmpHeapNoproc(2);
-                        return NULL;
-                    default:
-                        erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
-                        break;
-                }
-                break;
-            default:
-                erl_exit(1, "bad primary tag %p\r\n", node);
+        hval = MAP_HEADER_VAL(hdr);
+        ix   = hashmap_index(hx);
+        if (hval != 0xffff) {
+            bp   = 1 << ix;
+            if (!(bp & hval)) {
+                /* not occupied */
+                res = NULL;
                 break;
+            }
+            ix = hashmap_bitcount(hval & (bp - 1));
+        }
+        node  = ptr[ix+1];
+
+        if (is_list(node)) { /* LEAF NODE [K|V] */
+            ptr = list_val(node);
+
+            res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL;
+            break;
         }
+
+        hx = hashmap_shift_hash(th,hx,lvl,key);
+
+        ASSERT(is_boxed(node));
+        ptr = boxed_val(node);
+        hdr = *ptr;
+        ASSERT(is_header(hdr));
+        ASSERT(!is_hashmap_header_head(hdr));
     }
+
     UnUseTmpHeapNoproc(2);
-    return NULL;
+    return res;
 }
 
 Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
-- 
cgit v1.2.3


From c7a07bf984739bcc679d800e5383c01e1d07ffa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 4 Mar 2015 17:08:31 +0100
Subject: erts: Enable command line argument for initial pd size

Use '+hpds size' to set initial process dictionary size for spawned processes.
---
 erts/emulator/beam/erl_init.c         | 13 +++++++++++++
 erts/emulator/beam/erl_process_dict.c |  2 +-
 erts/emulator/beam/erl_vm.h           |  1 +
 3 files changed, 15 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 61f8385efc..743fd235bc 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -194,6 +194,8 @@ int erts_disable_tolerant_timeofday; /* Time correction can be disabled it is
 
 int erts_atom_table_size = ATOM_LIMIT;	/* Maximum number of atoms */
 
+int erts_pd_initial_size = 10;
+
 int erts_modified_timing_level;
 
 int erts_no_crash_dump = 0;	/* Use -d to suppress crash dump. */
@@ -519,6 +521,8 @@ void erts_usage(void)
 	       H_DEFAULT_SIZE);
     erts_fprintf(stderr, "-hmbs size     set minimum binary virtual heap size in words (default %d)\n",
 	       VH_DEFAULT_SIZE);
+    erts_fprintf(stderr, "-hpds size     initial process dictionary size (default %d)\n",
+	       erts_pd_initial_size);
 
     /*    erts_fprintf(stderr, "-i module  set the boot module (default init)\n"); */
 
@@ -1408,6 +1412,7 @@ erl_start(int argc, char **argv)
 	     *
 	     * h|ms  - min_heap_size
 	     * h|mbs - min_bin_vheap_size
+	     * h|pds - erts_pd_initial_size
 	     *
 	     */
 	    if (has_prefix("mbs", sub_param)) {
@@ -1425,6 +1430,14 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 		VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE));
+	    } else if (has_prefix("pds", sub_param)) {
+		arg = get_arg(sub_param+3, argv[i+1], &i);
+		if ((erts_pd_initial_size = atoi(arg)) <= 0) {
+		    erts_fprintf(stderr, "bad initial process dictionary size %s\n", arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM, ("using initial process dictionary size %d\n",
+			    erts_pd_initial_size));
 	    } else {
 	        /* backward compatibility */
 		arg = get_arg(argv[i]+2, argv[i+1], &i);
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 23e5bf737f..af20b26b15 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -47,7 +47,7 @@
 
 /* Hash constant macros */
 #define MAX_HASH            1342177280UL
-#define INITIAL_SIZE        10
+#define INITIAL_SIZE        (erts_pd_initial_size)
 
 /* Hash utility macros  */
 #define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition)
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index b7de8208ad..6687b044ee 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -174,6 +174,7 @@ extern int H_MIN_SIZE;		/* minimum (heap + stack) */
 extern int BIN_VH_MIN_SIZE;	/* minimum virtual (bin) heap */
 
 extern int erts_atom_table_size;/* Atom table size */
+extern int erts_pd_initial_size;/* Initial Process dictionary table size */
 
 #define ORIG_CREATION 0
 #define INTERNAL_CREATION 255
-- 
cgit v1.2.3


From 59ec3041469226ef65894e5774ecac0fea1551cd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 25 Mar 2015 11:30:28 +0100
Subject: erts: Fix bug in list_to_integer for very large strings

list_to_integer(lists:duplicate(10000000,$0)).

crashed due to overflow when calculating nr heap words.
---
 erts/emulator/beam/bif.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 42dd160e38..3013ceff21 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2775,7 +2775,7 @@ static int do_list_to_integer(Process *p, Eterm orig_list,
      Uint ui = 0;
      int skip = 0;
      int neg = 0;
-     int n = 0;
+     Sint n = 0;
      int m;
      int lg2;
      Eterm res;
@@ -2855,7 +2855,9 @@ static int do_list_to_integer(Process *p, Eterm orig_list,
          else i = (Sint)ui;
 	 res = make_small(i);
      } else {
-	 lg2 =  (n+1)*230/69+1;
+	 /* Convert from log10 to log2 by multiplying with 1/log10(2)=3.3219
+	    which we round up to (3 + 1/3) */
+	 lg2 = (n+1)*3 + (n+1)/3 + 1;
 	 m  = (lg2+D_EXP-1)/D_EXP; /* number of digits */
 	 m  = BIG_NEED_SIZE(m);    /* number of words + thing */
 
-- 
cgit v1.2.3


From c157dce842bf78080c533472fcec74df01ed8fdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 20 Mar 2015 18:01:50 +0100
Subject: erts: Combine flat and hash maps under one unifying tag

---
 erts/emulator/beam/beam_emu.c        |  16 +--
 erts/emulator/beam/copy.c            |  57 ++++----
 erts/emulator/beam/erl_db_util.c     |  18 +--
 erts/emulator/beam/erl_gc.h          |   7 +-
 erts/emulator/beam/erl_map.c         |  42 +++---
 erts/emulator/beam/erl_map.h         |  50 ++++---
 erts/emulator/beam/erl_nif.c         |   4 +-
 erts/emulator/beam/erl_printf_term.c | 139 ++++++++++---------
 erts/emulator/beam/erl_term.c        |   1 -
 erts/emulator/beam/erl_term.h        |  99 +++++++-------
 erts/emulator/beam/external.c        |  22 +--
 erts/emulator/beam/io.c              |   6 +-
 erts/emulator/beam/utils.c           | 250 ++++++++++++++++++-----------------
 13 files changed, 349 insertions(+), 362 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 6526e87e4c..8fcdc72895 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6602,8 +6602,8 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     keys   = make_tuple(thp);
     *thp++ = make_arityval(n/2);
 
-    mp = (flatmap_t *)mhp; mhp += MAP_HEADER_SIZE;
-    mp->thing_word = MAP_HEADER;
+    mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ;
+    mp->thing_word = MAP_HEADER_FLATMAP;
     mp->size = n/2;
     mp->keys = keys;
 
@@ -6684,7 +6684,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      * update list are new).
      */
 
-    need = 2*(num_old+num_updates) + 1 + MAP_HEADER_SIZE;
+    need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ;
     if (HeapWordsLeft(p) < need) {
 	Uint live = Arg(3);
 	reg[live] = map;
@@ -6723,8 +6723,8 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     res = make_flatmap(hp);
     mp = (flatmap_t *)hp;
-    hp += MAP_HEADER_SIZE;
-    mp->thing_word = MAP_HEADER;
+    hp += MAP_HEADER_FLATMAP_SZ;
+    mp->thing_word = MAP_HEADER_FLATMAP;
     mp->keys = make_tuple(kp-1);
 
     old_vals = flatmap_get_values(old_mp);
@@ -6906,7 +6906,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      * Allocate the exact heap space needed.
      */
 
-    need = num_old + MAP_HEADER_SIZE;
+    need = num_old + MAP_HEADER_FLATMAP_SZ;
     if (HeapWordsLeft(p) < need) {
 	Uint live = Arg(3);
 	reg[live] = map;
@@ -6927,8 +6927,8 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     res = make_flatmap(hp);
     mp = (flatmap_t *)hp;
-    hp += MAP_HEADER_SIZE;
-    mp->thing_word = MAP_HEADER;
+    hp += MAP_HEADER_FLATMAP_SZ;
+    mp->thing_word = MAP_HEADER_FLATMAP;
     mp->size = num_old;
     mp->keys = old_mp->keys;
 
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 027b85b079..4d12dae787 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -127,8 +127,25 @@ Uint size_object(Eterm obj)
 			obj = *bptr;
 			break;
 		    }
-		case HASHMAP_SUBTAG:
+		case MAP_SUBTAG:
 		    switch (MAP_HEADER_TYPE(hdr)) {
+			case MAP_HEADER_TAG_FLATMAP_HEAD :
+                            {
+                                Uint n;
+                                flatmap_t *mp;
+                                mp  = (flatmap_t*)flatmap_val_rel(obj,base);
+                                ptr = (Eterm *)mp;
+                                n   = flatmap_get_size(mp) + 1;
+                                sum += n + 2;
+                                ptr += 2; /* hdr + size words */
+                                while (n--) {
+                                    obj = *ptr++;
+                                    if (!IS_CONST(obj)) {
+                                        ESTACK_PUSH(s, obj);
+                                    }
+                                }
+                                goto pop_next;
+                            }
 			case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
 			case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
 			case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
@@ -183,25 +200,7 @@ Uint size_object(Eterm obj)
 			goto pop_next;
 		    }
 		    break;
-		case MAP_SUBTAG:
-		    {
-			Uint n;
-			flatmap_t *mp;
-			mp  = (flatmap_t*)flatmap_val_rel(obj,base);
-			ptr = (Eterm *)mp;
-			n   = flatmap_get_size(mp) + 1;
-			sum += n + 2;
-			ptr += 2; /* hdr + size words */
-			while (n--) {
-			    obj = *ptr++;
-			    if (!IS_CONST(obj)) {
-				ESTACK_PUSH(s, obj);
-			    }
-			}
-			goto pop_next;
-		    }
-		    break;
-		case BIN_MATCHSTATE_SUBTAG:
+                case BIN_MATCHSTATE_SUBTAG:
 		    erl_exit(ERTS_ABORT_EXIT,
 			     "size_object: matchstate term not allowed");
 		default:
@@ -369,15 +368,6 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    }
 		}
 		break;
-	    case MAP_SUBTAG:
-		{
-		    i = flatmap_get_size(objp) + 3;
-		    *argp = make_flatmap_rel(htop, dst_base);
-		    while (i--) {
-			*htop++ = *objp++;
-		    }
-		}
-		break;
 	    case REFC_BINARY_SUBTAG:
 		{
 		    ProcBin* pb;
@@ -502,9 +492,16 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		  *argp = make_external_rel(tp, dst_base);
 		}
 		break;
-	    case HASHMAP_SUBTAG:
+	    case MAP_SUBTAG:
 		tp = htop;
 		switch (MAP_HEADER_TYPE(hdr)) {
+		    case MAP_HEADER_TAG_FLATMAP_HEAD :
+                        i = flatmap_get_size(objp) + 3;
+                        *argp = make_flatmap_rel(htop, dst_base);
+                        while (i--) {
+                            *htop++ = *objp++;
+                        }
+			break;
 		    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
 		    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
 			*htop++ = *objp++;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 9d699d4b22..0bf562d937 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2153,16 +2153,16 @@ restart:
 	    break;
         case matchMkFlatMap:
             n = *pc++;
-            ehp = HAllocX(build_proc, 1 + MAP_HEADER_SIZE + n, HEAP_XTRA);
+            ehp = HAllocX(build_proc, 1 + MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA);
             t = *ehp++ = *--esp;
             {
                 flatmap_t *m = (flatmap_t *)ehp;
-                m->thing_word = MAP_HEADER;
+                m->thing_word = MAP_HEADER_FLATMAP;
                 m->size = n;
                 m->keys = t;
             }
             t = make_flatmap(ehp);
-            ehp += MAP_HEADER_SIZE;
+            ehp += MAP_HEADER_FLATMAP_SZ;
             while (n--) {
                 *ehp++ = *--esp;
             }
@@ -3540,14 +3540,10 @@ static DMCRet dmc_one_term(DMCContext *context,
 	    DMC_PUSH(*stack, c);
 	    break;
         case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):
-            n = flatmap_get_size(flatmap_val(c));
-            DMC_PUSH(*text, matchPushM);
-            ++(context->stack_used);
-            DMC_PUSH(*text, n);
-            DMC_PUSH(*stack, c);
-            break;
-        case (_TAG_HEADER_HASHMAP >> _TAG_PRIMARY_SIZE):
-            n = hashmap_size(c);
+            if (is_flatmap(c))
+                n = flatmap_get_size(flatmap_val(c));
+            else
+                n = hashmap_size(c);
             DMC_PUSH(*text, matchPushM);
             ++(context->stack_used);
             DMC_PUSH(*text, n);
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 8afcb060a1..bd6dcc9078 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -55,9 +55,10 @@ do {                                                                    \
     nelts = header_arity(HDR);                                          \
     switch ((HDR) & _HEADER_SUBTAG_MASK) {                              \
     case SUB_BINARY_SUBTAG: nelts++; break;                             \
-    case MAP_SUBTAG: nelts+=flatmap_get_size(PTR) + 1; break;               \
-    case HASHMAP_SUBTAG: nelts=hashmap_bitcount(MAP_HEADER_VAL(HDR));   \
-	 nelts += is_hashmap_header_head(HDR) ? 1 : 0; break;           \
+    case MAP_SUBTAG:                                                    \
+        if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1;   \
+        else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR));            \
+    break;                                                              \
     case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break;   \
     }                                                                   \
     gval    = make_boxed(HTOP);                                         \
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 35446501d4..4af8e6f274 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -303,10 +303,10 @@ static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size) {
     hp   += size;
     mp    = (flatmap_t*)hp;
     res   = make_flatmap(mp);
-    hp   += MAP_HEADER_SIZE;
+    hp   += MAP_HEADER_FLATMAP_SZ;
     vs    = hp;
 
-    mp->thing_word = MAP_HEADER;
+    mp->thing_word = MAP_HEADER_FLATMAP;
     mp->size = size; /* set later, might shrink*/
     mp->keys = keys;
 
@@ -438,10 +438,10 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
 	ks    = hp;
 	hp   += n;
 	mp    = (flatmap_t*)hp;
-	hp   += MAP_HEADER_SIZE;
+	hp   += MAP_HEADER_FLATMAP_SZ;
 	vs    = hp;
 
-	mp->thing_word = MAP_HEADER;
+	mp->thing_word = MAP_HEADER_FLATMAP;
 	mp->size = n;
 	mp->keys = keys;
 
@@ -944,16 +944,16 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     n1   = flatmap_get_size(mp1);
     n2   = flatmap_get_size(mp2);
 
-    need = MAP_HEADER_SIZE + 1 + 2*(n1 + n2);
+    need = MAP_HEADER_FLATMAP_SZ + 1 + 2 * (n1 + n2);
 
     hp     = HAlloc(p, need);
     thp    = hp;
     tup    = make_tuple(thp);
     ks     = hp + 1; hp += 1 + n1 + n2;
-    mp_new = (flatmap_t*)hp; hp += MAP_HEADER_SIZE;
+    mp_new = (flatmap_t*)hp; hp += MAP_HEADER_FLATMAP_SZ;
     vs     = hp; hp += n1 + n2;
 
-    mp_new->thing_word = MAP_HEADER;
+    mp_new->thing_word = MAP_HEADER_FLATMAP;
     mp_new->size = 0;
     mp_new->keys = tup;
 
@@ -1344,12 +1344,12 @@ BIF_RETTYPE maps_new_0(BIF_ALIST_0) {
     Eterm tup;
     flatmap_t *mp;
 
-    hp    = HAlloc(BIF_P, (MAP_HEADER_SIZE + 1));
+    hp    = HAlloc(BIF_P, (MAP_HEADER_FLATMAP_SZ + 1));
     tup   = make_tuple(hp);
     *hp++ = make_arityval(0);
 
     mp    = (flatmap_t*)hp;
-    mp->thing_word = MAP_HEADER;
+    mp->thing_word = MAP_HEADER_FLATMAP;
     mp->size = 0;
     mp->keys = tup;
 
@@ -1401,7 +1401,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 	*thp++ = make_arityval(n - 1);
 
 	*res   = make_flatmap(mhp);
-	*mhp++ = MAP_HEADER;
+	*mhp++ = MAP_HEADER_FLATMAP;
 	*mhp++ = n - 1;
 	*mhp++ = tup;
 
@@ -1478,9 +1478,9 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res)
 	 * assume key-tuple will be intact
 	 */
 
-	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
+	hp  = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n);
 	shp = hp;
-	*hp++ = MAP_HEADER;
+	*hp++ = MAP_HEADER_FLATMAP;
 	*hp++ = n;
 	*hp++ = mp->keys;
 
@@ -1502,7 +1502,7 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res)
 	    }
 	}
 
-	HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
+	HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp);
 	return 0;
 
 found_key:
@@ -1536,12 +1536,12 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 	n = flatmap_get_size(mp);
 
 	if (n == 0) {
-	    hp    = HAlloc(p, MAP_HEADER_SIZE + 1 + 2);
+	    hp    = HAlloc(p, MAP_HEADER_FLATMAP_SZ + 1 + 2);
 	    tup   = make_tuple(hp);
 	    *hp++ = make_arityval(1);
 	    *hp++ = key;
 	    res   = make_flatmap(hp);
-	    *hp++ = MAP_HEADER;
+	    *hp++ = MAP_HEADER_FLATMAP;
 	    *hp++ = 1;
 	    *hp++ = tup;
 	    *hp++ = value;
@@ -1556,10 +1556,10 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 	 * assume key-tuple will be intact
 	 */
 
-	hp  = HAlloc(p, MAP_HEADER_SIZE + n);
+	hp  = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n);
 	shp = hp; /* save hp, used if optimistic update fails */
 	res = make_flatmap(hp);
-	*hp++ = MAP_HEADER;
+	*hp++ = MAP_HEADER_FLATMAP;
 	*hp++ = n;
 	*hp++ = mp->keys;
 
@@ -1591,7 +1591,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 	/* the map will grow */
 
 	if (n >= MAP_SMALL_MAP_LIMIT) {
-	    HRelease(p, shp + MAP_HEADER_SIZE + n, shp);
+	    HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp);
 	    ks = flatmap_get_keys(mp);
 	    vs = flatmap_get_values(mp);
 
@@ -1608,7 +1608,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 
 	hp    = HAlloc(p, 3 + n + 1);
 	res   = make_flatmap(hp);
-	*hp++ = MAP_HEADER;
+	*hp++ = MAP_HEADER_FLATMAP;
 	*hp++ = n + 1;
 	*hp++ = tup;
 
@@ -2258,10 +2258,10 @@ unroll:
 	ks    = hp;
 	hp   += n;
 	mp    = (flatmap_t*)hp;
-	hp   += MAP_HEADER_SIZE;
+	hp   += MAP_HEADER_FLATMAP_SZ;
 	vs    = hp;
 
-	mp->thing_word = MAP_HEADER;
+	mp->thing_word = MAP_HEADER_FLATMAP;
 	mp->size = n;
 	mp->keys = keys;
 
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 9fc1a72b68..2cc6768bfc 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -55,9 +55,9 @@ typedef struct flatmap_s {
 /* the head-node is a bitmap or array with an untagged size */
 
 
-#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
+#define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size)
 #define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
-#define hashmap_make_hash(Key) make_internal_hash(Key)
+#define hashmap_make_hash(Key)        make_internal_hash(Key)
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
     (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7)))
@@ -66,27 +66,15 @@ typedef struct flatmap_s {
 
 
 /* erl_term.h stuff */
-#define make_flatmap(x)		make_boxed((Eterm*)(x))
-#define make_flatmap_rel(x, BASE)   make_boxed_rel((Eterm*)(x),(BASE))
-#define is_flatmap(x)		(is_boxed((x)) && is_flatmap_header(*boxed_val((x))))
-#define is_flatmap_rel(RTERM,BASE)  is_flatmap(rterm2wterm(RTERM,BASE))
-#define is_not_flatmap(x)           (!is_flatmap((x)))
-#define is_flatmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
-#define header_is_flatmap(x)        ((((x) & (_HEADER_SUBTAG_MASK)) == MAP_SUBTAG))
-#define flatmap_val(x)		(_unchecked_boxed_val((x)))
-#define flatmap_val_rel(RTERM, BASE) flatmap_val(rterm2wterm(RTERM, BASE))
-
-#define flatmap_get_values(x)      (((Eterm *)(x)) + 3)
-#define flatmap_get_keys(x)        (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1)
-#define flatmap_get_size(x)        (((flatmap_t*)(x))->size)
+#define flatmap_get_values(x)        (((Eterm *)(x)) + 3)
+#define flatmap_get_keys(x)          (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1)
+#define flatmap_get_size(x)          (((flatmap_t*)(x))->size)
 
 #ifdef DEBUG
 #define MAP_SMALL_MAP_LIMIT    (3)
 #else
 #define MAP_SMALL_MAP_LIMIT    (32)
 #endif
-#define MAP_HEADER             _make_header(1,_TAG_HEADER_MAP)
-#define MAP_HEADER_SIZE        (sizeof(flatmap_t) / sizeof(Eterm))
 
 struct ErtsWStack_;
 struct ErtsEStack_;
@@ -170,7 +158,10 @@ typedef struct hashmap_head_s {
 #define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2)))
 
 #define MAKE_MAP_HEADER(Type,Arity,Val) \
-    (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_HASHMAP))
+    (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_MAP))
+
+#define MAP_HEADER_FLATMAP \
+    MAKE_MAP_HEADER(MAP_HEADER_TAG_FLATMAP_HEAD,0x1,0x0)
 
 #define MAP_HEADER_HAMT_HEAD_ARRAY \
     MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff)
@@ -181,15 +172,22 @@ typedef struct hashmap_head_s {
 #define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \
     MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp)
 
-#define HAMT_HEAD_EMPTY_SZ     (2)
-#define HAMT_HEAD_ARRAY_SZ     (18)
-#define HAMT_NODE_BITMAP_SZ(n) (1 + n)
-#define HAMT_HEAD_BITMAP_SZ(n) (2 + n)
+#define MAP_HEADER_FLATMAP_SZ  (sizeof(flatmap_t) / sizeof(Eterm))
+
+#define HAMT_NODE_ARRAY_SZ      (17)
+#define HAMT_HEAD_ARRAY_SZ      (18)
+#define HAMT_NODE_BITMAP_SZ(n)  (1 + n)
+#define HAMT_HEAD_BITMAP_SZ(n)  (2 + n)
+
+/* 2 bits maps tag + 4 bits subtag + 2 ignore bits */
+#define _HEADER_MAP_SUBTAG_MASK       (0xfc)
+/* 1 bit map tag + 1 ignore bit + 4 bits subtag + 2 ignore bits */
+#define _HEADER_MAP_HASHMAP_HEAD_MASK (0xbc)
 
-#define _HEADER_MAP_SUBTAG_MASK    (0xfc) /* 2 bits maps tag + 4 bits subtag + 2 ignore bits */
-#define HAMT_SUBTAG_NODE_BITMAP  ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
-#define HAMT_SUBTAG_HEAD_ARRAY   ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
-#define HAMT_SUBTAG_HEAD_BITMAP  ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | HASHMAP_SUBTAG)
+#define HAMT_SUBTAG_NODE_BITMAP  ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
+#define HAMT_SUBTAG_HEAD_ARRAY   ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY  << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
+#define HAMT_SUBTAG_HEAD_BITMAP  ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
+#define HAMT_SUBTAG_HEAD_FLATMAP ((MAP_HEADER_TAG_FLATMAP_HEAD << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
 
 #define hashmap_index(hash)      (((Uint32)hash) & 0xf)
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c7c8b3fee3..660f446a52 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1926,14 +1926,14 @@ int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)
 
 ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env)
 {
-    Eterm* hp = alloc_heap(env,MAP_HEADER_SIZE+1);
+    Eterm* hp = alloc_heap(env,MAP_HEADER_FLATMAP_SZ+1);
     Eterm tup;
     flatmap_t *mp;
 
     tup   = make_tuple(hp);
     *hp++ = make_arityval(0);
     mp    = (flatmap_t*)hp;
-    mp->thing_word = MAP_HEADER;
+    mp->thing_word = MAP_HEADER_FLATMAP;
     mp->size = 0;
     mp->keys = tup;
 
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index ac5b139f8d..e0dfbd31b8 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -565,77 +565,74 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    }
 	    break;
 	case MAP_DEF:
-	    {
-		Uint n;
-		Eterm *ks, *vs;
-		flatmap_t *mp = (flatmap_t *)flatmap_val(wobj);
-		n  = flatmap_get_size(mp);
-		ks = flatmap_get_keys(mp);
-		vs = flatmap_get_values(mp);
-
-		PRINT_CHAR(res, fn, arg, '#');
-		PRINT_CHAR(res, fn, arg, '{');
-		WSTACK_PUSH(s, PRT_CLOSE_TUPLE);
-		if (n > 0) {
-		    n--;
-		    WSTACK_PUSH5(s, vs[n], PRT_TERM, PRT_ASSOC, ks[n], PRT_TERM);
-		    while (n--) {
-			WSTACK_PUSH6(s, PRT_COMMA, vs[n], PRT_TERM, PRT_ASSOC,
-				     ks[n], PRT_TERM);
-		    }
-		}
-	    }
-	    break;
-	case HASHMAP_DEF:
-	    {
-		Uint n,mapval;
-		Eterm *head;
-		head = hashmap_val(wobj);
-		mapval = MAP_HEADER_VAL(*head);
-		switch (MAP_HEADER_TYPE(*head)) {
-		    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY:
-		    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP:
-			PRINT_STRING(res, fn, arg, "#<");
-			PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval);
-			PRINT_STRING(res, fn, arg, ">{");
-			WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
-			n = hashmap_bitcount(mapval);
-			ASSERT(n < 17);
-			head += 2;
-			if (n > 0) {
-			    n--;
-			    WSTACK_PUSH(s, head[n]);
-			    WSTACK_PUSH(s, PRT_TERM);
-			    while (n--) {
-				WSTACK_PUSH(s, PRT_COMMA);
-				WSTACK_PUSH(s, head[n]);
-				WSTACK_PUSH(s, PRT_TERM);
-			    }
-			}
-			break;
-		    case MAP_HEADER_TAG_HAMT_NODE_BITMAP:
-			n = hashmap_bitcount(mapval);
-			head++;
-			PRINT_CHAR(res, fn, arg, '<');
-			PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval);
-			PRINT_STRING(res, fn, arg, ">{");
-			WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
-			ASSERT(n < 17);
-			if (n > 0) {
-			    n--;
-			    WSTACK_PUSH(s, head[n]);
-			    WSTACK_PUSH(s, PRT_TERM);
-			    while (n--) {
-				WSTACK_PUSH(s, PRT_COMMA);
-				WSTACK_PUSH(s, head[n]);
-				WSTACK_PUSH(s, PRT_TERM);
-			    }
-			}
-			break;
-		}
-	    }
-	    break;
-	default:
+            if (is_flatmap(wobj)) {
+                Uint n;
+                Eterm *ks, *vs;
+                flatmap_t *mp = (flatmap_t *)flatmap_val(wobj);
+                n  = flatmap_get_size(mp);
+                ks = flatmap_get_keys(mp);
+                vs = flatmap_get_values(mp);
+
+                PRINT_CHAR(res, fn, arg, '#');
+                PRINT_CHAR(res, fn, arg, '{');
+                WSTACK_PUSH(s, PRT_CLOSE_TUPLE);
+                if (n > 0) {
+                    n--;
+                    WSTACK_PUSH5(s, vs[n], PRT_TERM, PRT_ASSOC, ks[n], PRT_TERM);
+                    while (n--) {
+                        WSTACK_PUSH6(s, PRT_COMMA, vs[n], PRT_TERM, PRT_ASSOC,
+                                ks[n], PRT_TERM);
+                    }
+                }
+            } else {
+                Uint n, mapval;
+                Eterm *head;
+                head = hashmap_val(wobj);
+                mapval = MAP_HEADER_VAL(*head);
+                switch (MAP_HEADER_TYPE(*head)) {
+                case MAP_HEADER_TAG_HAMT_HEAD_ARRAY:
+                case MAP_HEADER_TAG_HAMT_HEAD_BITMAP:
+                    PRINT_STRING(res, fn, arg, "#<");
+                    PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval);
+                    PRINT_STRING(res, fn, arg, ">{");
+                    WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
+                    n = hashmap_bitcount(mapval);
+                    ASSERT(n < 17);
+                    head += 2;
+                    if (n > 0) {
+                        n--;
+                        WSTACK_PUSH(s, head[n]);
+                        WSTACK_PUSH(s, PRT_TERM);
+                        while (n--) {
+                            WSTACK_PUSH(s, PRT_COMMA);
+                            WSTACK_PUSH(s, head[n]);
+                            WSTACK_PUSH(s, PRT_TERM);
+                        }
+                    }
+                    break;
+                case MAP_HEADER_TAG_HAMT_NODE_BITMAP:
+                    n = hashmap_bitcount(mapval);
+                    head++;
+                    PRINT_CHAR(res, fn, arg, '<');
+                    PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval);
+                    PRINT_STRING(res, fn, arg, ">{");
+                    WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
+                    ASSERT(n < 17);
+                    if (n > 0) {
+                        n--;
+                        WSTACK_PUSH(s, head[n]);
+                        WSTACK_PUSH(s, PRT_TERM);
+                        while (n--) {
+                            WSTACK_PUSH(s, PRT_COMMA);
+                            WSTACK_PUSH(s, head[n]);
+                            WSTACK_PUSH(s, PRT_TERM);
+                        }
+                    }
+                    break;
+                }
+            }
+            break;
+        default:
 	    PRINT_STRING(res, fn, arg, "<unknown:");
 	    PRINT_POINTER(res, fn, arg, wobj);
 	    PRINT_CHAR(res, fn, arg, '>');
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index d6fb88ea61..bc04d7b78e 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -90,7 +90,6 @@ unsigned tag_val_def(Wterm x)
 	    case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
 	    case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
 	    case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
-	    case (_TAG_HEADER_HASHMAP >> _TAG_PRIMARY_SIZE):	return HASHMAP_DEF;
 	  }
  
 	  break;
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 1625a4ec15..095aa54ddd 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -141,29 +141,27 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define HEAP_BINARY_SUBTAG	(0x9 << _TAG_PRIMARY_SIZE) /* BINARY */
 #define SUB_BINARY_SUBTAG	(0xA << _TAG_PRIMARY_SIZE) /* BINARY */
 /*   _BINARY_XXX_MASK depends on 0xB being unused */
-#define HASHMAP_SUBTAG		(0xB << _TAG_PRIMARY_SIZE) /* HASHMAP */
 #define EXTERNAL_PID_SUBTAG	(0xC << _TAG_PRIMARY_SIZE) /* EXTERNAL_PID */
 #define EXTERNAL_PORT_SUBTAG	(0xD << _TAG_PRIMARY_SIZE) /* EXTERNAL_PORT */
 #define EXTERNAL_REF_SUBTAG	(0xE << _TAG_PRIMARY_SIZE) /* EXTERNAL_REF */
 #define MAP_SUBTAG		(0xF << _TAG_PRIMARY_SIZE) /* MAP */
 
 
-#define _TAG_HEADER_ARITYVAL	(TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG)
-#define _TAG_HEADER_FUN		(TAG_PRIMARY_HEADER|FUN_SUBTAG)
-#define _TAG_HEADER_POS_BIG	(TAG_PRIMARY_HEADER|POS_BIG_SUBTAG)
-#define _TAG_HEADER_NEG_BIG	(TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG)
-#define _TAG_HEADER_FLOAT	(TAG_PRIMARY_HEADER|FLOAT_SUBTAG)
-#define _TAG_HEADER_EXPORT	(TAG_PRIMARY_HEADER|EXPORT_SUBTAG)
-#define _TAG_HEADER_REF		(TAG_PRIMARY_HEADER|REF_SUBTAG)
-#define _TAG_HEADER_REFC_BIN	(TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG)
-#define _TAG_HEADER_HEAP_BIN	(TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG)
-#define _TAG_HEADER_SUB_BIN	(TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG)
-#define _TAG_HEADER_EXTERNAL_PID  (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG)
-#define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG)
-#define _TAG_HEADER_EXTERNAL_REF  (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG)
+#define _TAG_HEADER_ARITYVAL       (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG)
+#define _TAG_HEADER_FUN	           (TAG_PRIMARY_HEADER|FUN_SUBTAG)
+#define _TAG_HEADER_POS_BIG        (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG)
+#define _TAG_HEADER_NEG_BIG        (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG)
+#define _TAG_HEADER_FLOAT          (TAG_PRIMARY_HEADER|FLOAT_SUBTAG)
+#define _TAG_HEADER_EXPORT         (TAG_PRIMARY_HEADER|EXPORT_SUBTAG)
+#define _TAG_HEADER_REF            (TAG_PRIMARY_HEADER|REF_SUBTAG)
+#define _TAG_HEADER_REFC_BIN       (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG)
+#define _TAG_HEADER_HEAP_BIN       (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG)
+#define _TAG_HEADER_SUB_BIN        (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG)
+#define _TAG_HEADER_EXTERNAL_PID   (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG)
+#define _TAG_HEADER_EXTERNAL_PORT  (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG)
+#define _TAG_HEADER_EXTERNAL_REF   (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG)
 #define _TAG_HEADER_BIN_MATCHSTATE (TAG_PRIMARY_HEADER|BIN_MATCHSTATE_SUBTAG)
-#define _TAG_HEADER_MAP	 	(TAG_PRIMARY_HEADER|MAP_SUBTAG)
-#define _TAG_HEADER_HASHMAP	(TAG_PRIMARY_HEADER|HASHMAP_SUBTAG)
+#define _TAG_HEADER_MAP	           (TAG_PRIMARY_HEADER|MAP_SUBTAG)
 
 
 #define _TAG_HEADER_MASK	0x3F
@@ -302,7 +300,7 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm)
 #define is_header(x)	(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER)
 //#define _unchecked_header_arity(x)	((x) >> _HEADER_ARITY_OFFS)
 #define _unchecked_header_arity(x) \
-    (is_hashmap_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS))
+    (is_map_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS))
 _ET_DECLARE_CHECKED(Uint,header_arity,Eterm)
 #define header_arity(x)	_ET_APPLY(header_arity,(x))
 
@@ -1001,7 +999,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #define MAP_HEADER_ARITY_SZ               (8)
 #define MAP_HEADER_VAL_SZ                 (16)
 
-#define MAP_HEADER_TAG_FLAT               (0x0)
+#define MAP_HEADER_TAG_FLATMAP_HEAD       (0x0)
 #define MAP_HEADER_TAG_HAMT_NODE_BITMAP   (0x1)
 #define MAP_HEADER_TAG_HAMT_HEAD_ARRAY    (0x2)
 #define MAP_HEADER_TAG_HAMT_HEAD_BITMAP   (0x3)
@@ -1010,17 +1008,27 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff))
 #define MAP_HEADER_VAL(Hdr)   (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff))
 
-#define make_hashmap(x)		make_boxed((Eterm*)(x))
-#define make_hashmap_rel 	make_boxed_rel
-#define is_hashmap(x)		(is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
-#define is_not_hashmap(x)       (!is_hashmap(x))
-#define is_hashmap_rel(RTERM,BASE)  is_hashmap(rterm2wterm(RTERM,BASE))
-#define is_hashmap_header(x)	(((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HASHMAP)
-#define hashmap_val(x)		_unchecked_boxed_val((x))
-#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE))
-
-#define is_map(x)            (is_flatmap(x) || is_hashmap(x))
-#define is_map_rel(x,BASE)   (is_flatmap_rel(x,BASE) || is_hashmap_rel(x,BASE))
+#define make_hashmap(x)		      make_boxed((Eterm*)(x))
+#define make_hashmap_rel 	      make_boxed_rel
+#define is_hashmap(x)		      (is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
+#define is_not_hashmap(x)             (!is_hashmap(x))
+#define is_hashmap_rel(RTERM,BASE)    is_hashmap(rterm2wterm(RTERM,BASE))
+#define is_hashmap_header(x)	      (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY)
+#define hashmap_val(x)		      _unchecked_boxed_val((x))
+#define hashmap_val_rel(RTERM, BASE)  hashmap_val(rterm2wterm(RTERM, BASE))
+
+#define make_flatmap(x)               make_boxed((Eterm*)(x))
+#define make_flatmap_rel(x, BASE)     make_boxed_rel((Eterm*)(x),(BASE))
+#define is_flatmap(x)                 (is_boxed((x)) && is_flatmap_header(*boxed_val((x))))
+#define is_flatmap_rel(RTERM,BASE)    is_flatmap(rterm2wterm(RTERM,BASE))
+#define is_not_flatmap(x)             (!is_flatmap((x)))
+#define is_flatmap_header(x)          (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP)
+#define flatmap_val(x)                (_unchecked_boxed_val((x)))
+#define flatmap_val_rel(RTERM, BASE)  flatmap_val(rterm2wterm(RTERM, BASE))
+
+#define is_map_header(x)       (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
+#define is_map(x)              (is_boxed((x)) && is_map_header(*boxed_val(x)))
+#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE))
 
 /* number tests */
 
@@ -1113,23 +1121,22 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define BINARY_DEF		0x0
 #define LIST_DEF		0x1
 #define NIL_DEF			0x2
-#define HASHMAP_DEF		0x3
-#define MAP_DEF			0x4
-#define TUPLE_DEF		0x5
-#define PID_DEF			0x6
-#define EXTERNAL_PID_DEF	0x7
-#define PORT_DEF		0x8
-#define EXTERNAL_PORT_DEF	0x9
-#define EXPORT_DEF		0xa
-#define FUN_DEF			0xb
-#define REF_DEF			0xc
-#define EXTERNAL_REF_DEF	0xd
-#define ATOM_DEF		0xe
-#define FLOAT_DEF		0xf
-#define BIG_DEF			0x10
-#define SMALL_DEF		0x11
-
-#define FIRST_VACANT_TAG_DEF    0x12
+#define MAP_DEF			0x3
+#define TUPLE_DEF		0x4
+#define PID_DEF			0x5
+#define EXTERNAL_PID_DEF	0x6
+#define PORT_DEF		0x7
+#define EXTERNAL_PORT_DEF	0x8
+#define EXPORT_DEF		0x9
+#define FUN_DEF			0xa
+#define REF_DEF			0xb
+#define EXTERNAL_REF_DEF	0xc
+#define ATOM_DEF		0xd
+#define FLOAT_DEF		0xe
+#define BIG_DEF			0xf
+#define SMALL_DEF		0x10
+
+#define FIRST_VACANT_TAG_DEF    0x11
 
 #if ET_DEBUG
 extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index b0b232f185..2a9189b51e 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2604,7 +2604,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    break;
 
 	case MAP_DEF:
-	    {
+	    if (is_flatmap(obj)) {
 		flatmap_t *mp = (flatmap_t*)flatmap_val(obj);
 		Uint size = flatmap_get_size(mp);
 
@@ -2618,11 +2618,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr,
 				 ENC_MAP_PAIR, size);
 		}
-	    }
-	    break;
-
-	case HASHMAP_DEF:
-	    {
+	    } else {
 		Eterm hdr;
 		Uint node_sz;
 		ptr = boxed_val(obj);
@@ -2656,7 +2652,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		}
 	    }
 	    break;
-
 	case FLOAT_DEF:
 	    GET_DOUBLE(obj, f);
 	    if (dflags & DFLAG_NEW_FLOATS) {
@@ -3607,7 +3602,7 @@ dec_term_atom_common:
                     kptr = hp - 1;
 
                     mp    = (flatmap_t*)hp;
-                    hp   += MAP_HEADER_SIZE;
+                    hp   += MAP_HEADER_FLATMAP_SZ;
                     hp   += size;
                     vptr = hp - 1;
 
@@ -3893,7 +3888,7 @@ dec_term_atom_common:
 
     while (maps_list) {
 	next  = (Eterm *)(EXPAND_POINTER(*maps_list));
-	*maps_list = MAP_HEADER;
+	*maps_list = MAP_HEADER_FLATMAP;
 	if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list))
 	    goto error;
 	maps_list  = next;
@@ -4121,7 +4116,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    }
 	    break;
 	case MAP_DEF:
-	    {
+	    if (is_flatmap(obj)) {
 		flatmap_t *mp = (flatmap_t*)flatmap_val(obj);
 		Uint size = flatmap_get_size(mp);
 		Uint i;
@@ -4158,11 +4153,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		    ++ptr;
 		}
 		goto outer_loop;
-	    }
-	    break;
-
-	case HASHMAP_DEF:
-	    {
+	    } else {
 		Eterm *ptr;
 		Eterm hdr;
 		Uint node_sz;
@@ -4190,7 +4181,6 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		}
 		result += 1 + 4; /* tag + 4 bytes size */
 	    }
-
 	    break;
 	case FLOAT_DEF:
 	    if (dflags & DFLAG_NEW_FLOATS) {
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 46f0eba5e0..1db3a9fba7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5354,7 +5354,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
             if (ptr[0] > MAP_SMALL_MAP_LIMIT) {
                 need += hashmap_over_estimated_heap_size(ptr[0]);
             } else {
-                need += MAP_HEADER_SIZE + 1 + 2*ptr[0];
+                need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0];
             }
 	    depth -= 2*ptr[0];
 	    if (depth < 0) ERTS_DDT_FAIL;
@@ -5627,12 +5627,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
                 hp += 1 + size;
                 mp = (flatmap_t*)hp;
-                mp->thing_word = MAP_HEADER;
+                mp->thing_word = MAP_HEADER_FLATMAP;
                 mp->size = size;
                 mp->keys = make_tuple(tp);
                 mess = make_flatmap(mp);
 
-                hp += MAP_HEADER_SIZE + size;   /* advance "heap" pointer */
+                hp += MAP_HEADER_FLATMAP_SZ + size;
 
                 tp += size;    /* point at last key */
                 vp = hp - 1;   /* point at last value */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 6edb466a36..cb4ef2b376 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1025,11 +1025,8 @@ tail_recur:
 	    break;
 	}	
     case MAP_DEF:
-    case HASHMAP_DEF:
-	{
-	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
-	    break;
-	}
+        hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
+        break;
     case TUPLE_DEF: 
 	{
 	    Eterm* ptr = tuple_val(term);
@@ -1273,41 +1270,40 @@ make_hash2(Eterm term)
 		}
 	    }
 	    break;
-	    case MAP_SUBTAG:
-	    {
-		flatmap_t *mp = (flatmap_t *)flatmap_val(term);
-		int i;
-		int size  = flatmap_get_size(mp);
-		Eterm *ks = flatmap_get_keys(mp);
-		Eterm *vs = flatmap_get_values(mp);
-		UINT32_HASH(size, HCONST_16);
-		if (size == 0) {
-		    goto hash2_common;
-		}
-		/* We want a portable hash function that is *independent* of
-		 * the order in which keys and values are encountered.
-		 * We therefore calculate context independent hashes for all    				      .
-		 * key-value pairs and then xor them together.
-		 */
-		ESTACK_PUSH(s, hash_xor_pairs);
-		ESTACK_PUSH(s, hash);
-		ESTACK_PUSH(s, HASH_MAP_TAIL);
-		hash = 0;
-		hash_xor_pairs = 0;
-		for (i = size - 1; i >= 0; i--) {
-		    ESTACK_PUSH(s, HASH_MAP_PAIR);
-		    ESTACK_PUSH(s, vs[i]);
-                    ESTACK_PUSH(s, ks[i]);
-		}
-		goto hash2_common;
-	    }
-	    break;
-            case HASHMAP_SUBTAG:
+            case MAP_SUBTAG:
             {
                 Eterm* ptr = boxed_val(term) + 1;
                 Uint size;
                 int i;
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+                case HAMT_SUBTAG_HEAD_FLATMAP:
+                {
+                    flatmap_t *mp = (flatmap_t *)flatmap_val(term);
+                    Eterm *ks = flatmap_get_keys(mp);
+                    Eterm *vs = flatmap_get_values(mp);
+                    size      = flatmap_get_size(mp);
+                    UINT32_HASH(size, HCONST_16);
+                    if (size == 0)
+                        goto hash2_common;
+
+                    /* We want a portable hash function that is *independent* of
+                     * the order in which keys and values are encountered.
+                     * We therefore calculate context independent hashes for all    				      .
+                     * key-value pairs and then xor them together.
+                     */
+                    ESTACK_PUSH(s, hash_xor_pairs);
+                    ESTACK_PUSH(s, hash);
+                    ESTACK_PUSH(s, HASH_MAP_TAIL);
+                    hash = 0;
+                    hash_xor_pairs = 0;
+                    for (i = size - 1; i >= 0; i--) {
+                        ESTACK_PUSH(s, HASH_MAP_PAIR);
+                        ESTACK_PUSH(s, vs[i]);
+                        ESTACK_PUSH(s, ks[i]);
+                    }
+                    goto hash2_common;
+                }
+
                 case HAMT_SUBTAG_HEAD_ARRAY:
                 case HAMT_SUBTAG_HEAD_BITMAP:
                     size = *ptr++;
@@ -1675,42 +1671,40 @@ make_internal_hash(Eterm term)
 		}
 	    }
 	    break;
-	    case MAP_SUBTAG:
-	    {
-		flatmap_t *mp = (flatmap_t *)flatmap_val(term);
-		int i;
-		int size  = flatmap_get_size(mp);
-		Eterm *ks = flatmap_get_keys(mp);
-		Eterm *vs = flatmap_get_values(mp);
-		UINT32_HASH(size, HCONST_16);
-		if (size == 0) {
-		    goto pop_next;
-		}
-		/* We want a hash function that is *independent* of
-		 * the order in which keys and values are encountered.
-		 * We therefore calculate context independent hashes for all    				      .
-		 * key-value pairs and then xor them together.
-		 */
-		ESTACK_PUSH(s, hash_xor_pairs);
-		ESTACK_PUSH(s, hash);
-		ESTACK_PUSH(s, HASH_MAP_TAIL);
-		hash = 0;
-		hash_xor_pairs = 0;
-		for (i = size - 1; i >= 0; i--) {
-		    ESTACK_PUSH(s, HASH_MAP_PAIR);
-		    ESTACK_PUSH(s, vs[i]);
-                    ESTACK_PUSH(s, ks[i]);
-		}
-		goto pop_next;
-	    }
-	    break;
-            case HASHMAP_SUBTAG:
+
+            case MAP_SUBTAG:
             {
                 Eterm* ptr = boxed_val(term) + 1;
                 Uint size;
                 int i;
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
-                case HAMT_SUBTAG_HEAD_ARRAY:
+                case HAMT_SUBTAG_HEAD_FLATMAP:
+                {
+                    flatmap_t *mp = (flatmap_t *)flatmap_val(term);
+                    Eterm *ks = flatmap_get_keys(mp);
+                    Eterm *vs = flatmap_get_values(mp);
+                    size      = flatmap_get_size(mp);
+                    UINT32_HASH(size, HCONST_16);
+                    if (size == 0)
+                        goto pop_next;
+
+                    /* We want a hash function that is *independent* of
+                     * the order in which keys and values are encountered.
+                     * We therefore calculate context independent hashes for all    				      .
+                     * key-value pairs and then xor them together.
+                     */
+                    ESTACK_PUSH(s, hash_xor_pairs);
+                    ESTACK_PUSH(s, hash);
+                    ESTACK_PUSH(s, HASH_MAP_TAIL);
+                    hash = 0;
+                    hash_xor_pairs = 0;
+                    for (i = size - 1; i >= 0; i--) {
+                        ESTACK_PUSH(s, HASH_MAP_PAIR);
+                        ESTACK_PUSH(s, vs[i]);
+                        ESTACK_PUSH(s, ks[i]);
+                    }
+                    goto pop_next;
+                }
                 case HAMT_SUBTAG_HEAD_BITMAP:
                     size = *ptr++;
                     UINT32_HASH(size, HCONST_16);
@@ -2170,11 +2164,8 @@ tail_recur:
 	break;
 
     case MAP_DEF:
-    case HASHMAP_DEF:
-	{
-	    hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
-	    break;
-	}
+        hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
+        break;
     case TUPLE_DEF: 
 	{
 	    Eterm* ptr = tuple_val(term);
@@ -2579,22 +2570,6 @@ tailrecur_ne:
 		    ++bb;
 		    goto term_array;
 		}
-	    case MAP_SUBTAG:
-		{
-		    aa = flatmap_val_rel(a, a_base);
-		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
-			goto not_equal;
-		    bb = flatmap_val_rel(b,b_base);
-		    sz = flatmap_get_size((flatmap_t*)aa);
-
-		    if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal;
-		    if (sz == 0) goto pop_next;
-
-		    aa += 2;
-		    bb += 2;
-		    sz += 1; /* increment for tuple-keys */
-		    goto term_array;
-		}
 	    case REFC_BINARY_SUBTAG:
 	    case HEAP_BINARY_SUBTAG:
 	    case SUB_BINARY_SUBTAG:
@@ -2786,8 +2761,23 @@ tailrecur_ne:
 		    }
 		    break; /* not equal */
 		}
-	    case HASHMAP_SUBTAG:
-		{
+	    case MAP_SUBTAG:
+                if (is_flatmap_rel(a, a_base)) {
+		    aa = flatmap_val_rel(a, a_base);
+		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
+			goto not_equal;
+		    bb = flatmap_val_rel(b,b_base);
+		    sz = flatmap_get_size((flatmap_t*)aa);
+
+		    if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal;
+		    if (sz == 0) goto pop_next;
+
+		    aa += 2;
+		    bb += 2;
+		    sz += 1; /* increment for tuple-keys */
+		    goto term_array;
+
+                } else {
 		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != hdr)
 			goto not_equal;
 
@@ -3124,44 +3114,56 @@ tailrecur_ne:
 		++aa;
 		++bb;
 		goto term_array;
-	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) :
-		if (!is_flatmap_rel(b,b_base)) {
-		    a_tag = MAP_DEF;
-		    goto mixed_types;
-		}
-		aa = (Eterm *)flatmap_val_rel(a,a_base);
-		bb = (Eterm *)flatmap_val_rel(b,b_base);
-
-		i = flatmap_get_size((flatmap_t*)aa);
-		if (i != flatmap_get_size((flatmap_t*)bb)) {
-		    RETURN_NEQ((int)(i - flatmap_get_size((flatmap_t*)bb)));
-		}
-		if (i == 0) {
-		    goto pop_next;
-		}
-		aa += 2;
-		bb += 2;
-		if (exact) {
-		    i  += 1; /* increment for tuple-keys */
-		    goto term_array;
-		}
-		else {
-		    /* Value array */
-		    WSTACK_PUSH3(stack, (UWord)(bb+1), (UWord)(aa+1), TERM_ARRAY_OP_WORD(i));
-		    /* Switch back from 'exact' key compare */
-		    WSTACK_PUSH(stack, OP_WORD(SWITCH_EXACT_OFF_OP));
-		    /* Now do 'exact' compare of key tuples */
-		    a = *aa;
-		    b = *bb;
-		    exact = 1;
-		    goto bodyrecur;
-		}
-
-	    case (_TAG_HEADER_HASHMAP >> _TAG_PRIMARY_SIZE) :
+            case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) :
 		{
                     struct erts_cmp_hashmap_state* sp;
+                    if (is_flatmap_header(ahdr)) {
+                        if (!is_flatmap_rel(b,b_base)) {
+                            if (is_hashmap_rel(b,b_base)) {
+                                aa = (Eterm *)flatmap_val_rel(a,a_base);
+                                i = flatmap_get_size((flatmap_t*)aa) - hashmap_size_rel(b,b_base);
+                                ASSERT(i != 0);
+                                RETURN_NEQ(i);
+                            }
+                            a_tag = MAP_DEF;
+                            goto mixed_types;
+                        }
+                        aa = (Eterm *)flatmap_val_rel(a,a_base);
+                        bb = (Eterm *)flatmap_val_rel(b,b_base);
+
+                        i = flatmap_get_size((flatmap_t*)aa);
+                        if (i != flatmap_get_size((flatmap_t*)bb)) {
+                            RETURN_NEQ((int)(i - flatmap_get_size((flatmap_t*)bb)));
+                        }
+                        if (i == 0) {
+                            goto pop_next;
+                        }
+                        aa += 2;
+                        bb += 2;
+                        if (exact) {
+                            i  += 1; /* increment for tuple-keys */
+                            goto term_array;
+                        }
+                        else {
+                            /* Value array */
+                            WSTACK_PUSH3(stack,(UWord)(bb+1),(UWord)(aa+1),TERM_ARRAY_OP_WORD(i));
+                            /* Switch back from 'exact' key compare */
+                            WSTACK_PUSH(stack,OP_WORD(SWITCH_EXACT_OFF_OP));
+                            /* Now do 'exact' compare of key tuples */
+                            a = *aa;
+                            b = *bb;
+                            exact = 1;
+                            goto bodyrecur;
+                        }
+                    }
 		    if (!is_hashmap_rel(b,b_base)) {
-			a_tag = HASHMAP_DEF;
+                        if (is_flatmap_rel(b,b_base)) {
+                            bb = (Eterm *)flatmap_val_rel(b,b_base);
+                            i = hashmap_size_rel(a,a_base) - flatmap_get_size((flatmap_t*)bb);
+                            ASSERT(i != 0);
+                            RETURN_NEQ(i);
+                        }
+			a_tag = MAP_DEF;
 			goto mixed_types;
 		    }
 		    i = hashmap_size_rel(a,a_base) - hashmap_size_rel(b,b_base);
-- 
cgit v1.2.3


From 139edf3f40fcf746fe21318f03032c091b3c2fc1 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Mar 2015 19:43:08 +0100
Subject: Fixes and cleanup

---
 erts/emulator/beam/erl_time_sup.c | 48 ++++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 28 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index ef39f4b5f4..8203436c85 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2015. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -139,12 +139,12 @@ struct time_sup_read_only__ {
     int os_monotonic_time_locked;
     Uint64 os_monotonic_time_resolution;
     Uint64 os_monotonic_time_extended;
+#endif
     char *os_system_time_func;
     char *os_system_time_clock_id;
     int os_system_time_locked;
     Uint64 os_system_time_resolution;
     Uint64 os_system_time_extended;
-#endif
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
     ErtsMonotonicTime start;
     struct {
@@ -287,7 +287,7 @@ get_time_offset(void)
 #define ERTS_LONG_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(60)
 #define ERTS_SHORT_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(15)
 
-#define ERTS_TIME_DRIFT_MAX_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(100)
+#define ERTS_TIME_DRIFT_MAX_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(50)
 #define ERTS_TIME_DRIFT_MIN_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(5)
 
 static ERTS_INLINE ErtsMonotonicTime
@@ -353,21 +353,21 @@ print_correction(int change,
 
     if (!change)
 	fprintf(stderr,
-		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppm] : "
+		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppb] : "
 		"tmo = %lld msec\r\n",
 		(long long) usec_sdiff,
 		(long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
-		(long long) (1000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		(long long) (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
 		(long long) tmo);
     else
 	fprintf(stderr,
-		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppm] "
-		"-> [ec=%lld ppm, dc=%lld ppm] : tmo = %lld msec\r\n",
+		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppb] "
+		"-> [ec=%lld ppm, dc=%lld ppb] : tmo = %lld msec\r\n",
 		(long long) usec_sdiff,
 		(long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
-		(long long) (1000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		(long long) (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
 		(long long) (1000000*new_ecorr) / ERTS_TCORR_ERR_UNIT,
-		(long long) (1000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		(long long) (1000000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
 		(long long) tmo);
 
 }
@@ -412,8 +412,7 @@ check_time_correction(void *unused)
 
     ASSERT(time_sup.inf.c.finalized_offset);
 
-    os_mtime = erts_os_monotonic_time();
-    os_stime = erts_os_system_time();
+    erts_os_times(&os_mtime, &os_stime);
 
     cdata = time_sup.inf.c.parmon.cdata;
 
@@ -556,13 +555,6 @@ check_time_correction(void *unused)
 	    ddp->acc.mon = mtime_acc;
 	    ddp->acc.sys = stime_acc;
 
-	    /*
-	     * If calculated drift adjustment is if off by more than 20%
-	     * from the average drift we interpret this as a discontinous
-	     * leap in system time and ignore it. If it actually is a
-	     * change in drift we will later detect this when the average
-	     * drift change.
-	     */
 	    drift_adj_diff = avg_drift_adj - drift_adj;
 	    if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF
 		|| ERTS_TIME_DRIFT_MAX_ADJ_DIFF < drift_adj_diff) {
@@ -691,8 +683,7 @@ init_check_time_correction(void *unused)
     old_mtime = ddp->intervals[0].time.mon;
     old_stime = ddp->intervals[0].time.sys;
 
-    mtime = erts_os_monotonic_time();
-    stime = erts_os_system_time();
+    erts_os_times(&mtime, &stime);
 
     mtime_diff = mtime - old_mtime;
     stime_diff = stime - old_stime;
@@ -732,8 +723,7 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep)
 
     erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
-    os_mtime = erts_os_monotonic_time();
-    *stimep = erts_os_system_time();
+    erts_os_times(&os_mtime, stimep);
 
     cdata = time_sup.inf.c.parmon.cdata;
 
@@ -852,6 +842,7 @@ void erts_init_sys_time_sup(void)
 	= sys_init_time_res.os_monotonic_time_info.resolution;
     time_sup.r.o.os_monotonic_time_extended
 	= sys_init_time_res.os_monotonic_time_info.extended;
+#endif
     time_sup.r.o.os_system_time_func
 	= sys_init_time_res.os_system_time_info.func;
     time_sup.r.o.os_system_time_clock_id
@@ -860,7 +851,6 @@ void erts_init_sys_time_sup(void)
 	= sys_init_time_res.os_system_time_info.locked_use;
     time_sup.r.o.os_system_time_resolution
 	= sys_init_time_res.os_system_time_info.resolution;
-#endif
 }
 
 int 
@@ -935,9 +925,11 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
 #endif
 
-    resolution = time_sup.r.o.os_monotonic_time_resolution;
-    if (resolution > time_sup.r.o.os_system_time_resolution)
-	resolution = time_sup.r.o.os_system_time_resolution;
+    resolution = time_sup.r.o.os_system_time_resolution;
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    if (resolution > time_sup.r.o.os_monotonic_time_resolution)
+	resolution = time_sup.r.o.os_monotonic_time_resolution;
+#endif
 
     time_sup.r.o.adj.large_diff = erts_time_sup__.r.o.monotonic_time_unit;
     time_sup.r.o.adj.large_diff *= 50;
@@ -972,8 +964,8 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	ErtsMonotonicCorrectionData *cdatap;
 	erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	ErtsMonotonicTime offset;
-	time_sup.inf.c.minit = erts_os_monotonic_time();
-	time_sup.inf.c.sinit = erts_os_system_time();
+	erts_os_times(&time_sup.inf.c.minit,
+		      &time_sup.inf.c.sinit);
 	time_sup.r.o.moffset = -1*time_sup.inf.c.minit;
 	offset = time_sup.inf.c.sinit;
 	offset -= ERTS_MONOTONIC_TIME_UNIT;
-- 
cgit v1.2.3


From 73b1ce18d362e5d192c2d8e31514a9ca1337aa2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 25 Mar 2015 16:06:38 +0100
Subject: erts: GC needs the size even if the frag is not referenced

---
 erts/emulator/beam/erl_map.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4af8e6f274..5d965f3116 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1008,6 +1008,7 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 
     n = n1 + n2 - unused_size;
     *thp = make_arityval(n);
+    mp_new->size = n;
 
     /* Reshape map to a hashmap if the map exceeds the limit */
 
@@ -1043,8 +1044,6 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 	return res;
     }
 
-    mp_new->size = n;
-
     return make_flatmap(mp_new);
 }
 
-- 
cgit v1.2.3


From 15ce2d396e5c3e5fe7c775a18764db1f7f589e54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 25 Mar 2015 17:17:19 +0100
Subject: erts: Refactor Map - use multiple values ESTACK_PUSHN

---
 erts/emulator/beam/erl_map.c | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 5d965f3116..ab40f47919 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -710,8 +710,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
 	}
     }
 
-    ESTACK_PUSH(stack, res);
-    ESTACK_PUSH(stack, 1 << slot);
+    ESTACK_PUSH2(stack,res,1 << slot);
 
     /* all of the other nodes .. */
     elems = n - 2; /* remove first and last elements */
@@ -755,14 +754,12 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
 	    slot = maskval(v, d);
 	    bp   = 1 << slot;
 	    /* no more collisions */
-	    ESTACK_PUSH(stack,res);
-	    ESTACK_PUSH(stack,bp);
+            ESTACK_PUSH2(stack,res,bp);
 	} else if (d == dn) {
 	    /* no collisions at all */
 	    slot = maskval(v, d);
 	    bp   = 1 << slot;
-	    ESTACK_PUSH(stack,res);
-	    ESTACK_PUSH(stack,hdr | bp);
+            ESTACK_PUSH2(stack,res,hdr | bp);
 	} else {
 	    /* dn < n, we have a drop and we are done
 	     * build nodes and subtree */
@@ -786,8 +783,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
 		hdr = ESTACK_POP(stack);
 		d--;
 	    }
-	    ESTACK_PUSH(stack, res);
-	    ESTACK_PUSH(stack, hdr);
+            ESTACK_PUSH2(stack,res,hdr);
 	}
 
 	vp = v;
@@ -1945,8 +1941,7 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 			slot = hashmap_bitcount(hval & (bp - 1));
 			n    = hashmap_bitcount(hval);
 
-			ESTACK_PUSH(*sp, n);
-			ESTACK_PUSH3(*sp, bp, slot, node);
+			ESTACK_PUSH4(*sp, n, bp, slot, node);
 
 			/* occupied */
 			if (bp & hval) {
@@ -1969,8 +1964,7 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 			slot = hashmap_bitcount(hval & (bp - 1));
 			n    = hashmap_bitcount(hval);
 
-			ESTACK_PUSH(*sp, n);
-			ESTACK_PUSH3(*sp, bp, slot, node);
+			ESTACK_PUSH4(*sp, n, bp, slot, node);
 
 			/* occupied */
 			if (bp & hval) {
@@ -2004,8 +1998,7 @@ insert_subnodes:
     cix   = hashmap_index(chx);
 
     while (cix == ix) {
-	ESTACK_PUSH(*sp, 0);
-	ESTACK_PUSH3(*sp, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
+	ESTACK_PUSH4(*sp, 0, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0));
 	size += HAMT_NODE_BITMAP_SZ(1);
 	hx    = hashmap_shift_hash(th,hx,lvl,key);
 	chx   = hashmap_shift_hash(th,chx,clvl,ckey);
@@ -2193,8 +2186,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
 			slot = hashmap_bitcount(hval & (bp - 1));
 			n    = hashmap_bitcount(hval);
 
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
+			ESTACK_PUSH4(stack, n, bp, slot, node);
 
 			/* occupied */
 			if (bp & hval) {
@@ -2213,8 +2205,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
 			slot = hashmap_bitcount(hval & (bp - 1));
 			n    = hashmap_bitcount(hval);
 
-			ESTACK_PUSH(stack, n);
-			ESTACK_PUSH3(stack, bp, slot, node);
+			ESTACK_PUSH4(stack, n, bp, slot, node);
 
 			/* occupied */
 			if (bp & hval) {
-- 
cgit v1.2.3


From 5d5543fb1e1a30e4572e90bac2ec194a61ca4e30 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 25 Mar 2015 19:55:43 +0100
Subject: erts: Optimize term_to_binary size estimation

for tuples and maps containing ascii strings (lists).
---
 erts/emulator/beam/external.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index b0b232f185..4cd57d8aeb 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -4109,8 +4109,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		}
 		for (i = 1; i <= arity; ++i) {
 		    if (is_list(ptr[i])) {
-			if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) {
+			if ((m = is_string(ptr[i])) && (m < MAX_STRING_LEN)) {
 			    result += m + 2 + 1;
+			    continue;
 			} else {
 			    result += 5;
 			}
@@ -4131,31 +4132,29 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 
 		/* push values first */
 		ptr = flatmap_get_values(mp);
-		i   = size;
-		while(i--) {
+		for (i = size; i; i--, ptr++) {
 		    if (is_list(*ptr)) {
 			if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) {
 			    result += m + 2 + 1;
+			    continue;
 			} else {
 			    result += 5;
 			}
 		    }
 		    ESTACK_PUSH(s,*ptr);
-		    ++ptr;
 		}
 
 		ptr = flatmap_get_keys(mp);
-		i   = size;
-		while(i--) {
+		for (i = size; i; i--, ptr++) {
 		    if (is_list(*ptr)) {
 			if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) {
 			    result += m + 2 + 1;
+			    continue;
 			} else {
 			    result += 5;
 			}
 		    }
 		    ESTACK_PUSH(s,*ptr);
-		    ++ptr;
 		}
 		goto outer_loop;
 	    }
-- 
cgit v1.2.3


From d09ec9ee97816bd987e6322797b0e28c27f8590a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 25 Mar 2015 20:50:12 +0100
Subject: erts: Fix bug in term_to_binary size estimation for hamt

---
 erts/emulator/beam/external.c | 39 +++++++++++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 4cd57d8aeb..8f0e19d27d 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -4172,8 +4172,12 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		case HAMT_SUBTAG_HEAD_ARRAY:
 		    ptr++;
 		    node_sz = 16;
+		    result += 1 + 4; /* tag + 4 bytes size */
 		    break;
-		case HAMT_SUBTAG_HEAD_BITMAP: ptr++;
+		case HAMT_SUBTAG_HEAD_BITMAP:
+		    ptr++;
+		    result += 1 + 4; /* tag + 4 bytes size */
+		    /*fall through*/
 		case HAMT_SUBTAG_NODE_BITMAP:
 		    node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
 		    ASSERT(node_sz < 17);
@@ -4183,11 +4187,38 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		}
 
 		ptr++;
-		ESTACK_RESERVE(s, node_sz);
+		ESTACK_RESERVE(s, node_sz*2);
 		while(node_sz--) {
-		    ESTACK_FAST_PUSH(s, *ptr++);
+                    if (is_list(*ptr)) {
+			Eterm* leaf = list_val(*ptr);
+			if (is_not_list(CAR(leaf))) {
+			    ESTACK_FAST_PUSH(s, CAR(leaf));
+			}
+			else {
+			    if ((m = is_string(CAR(leaf))) && (m < MAX_STRING_LEN)) {
+				result += m + 2 + 1;
+			    } else {
+				result += 5;
+				ESTACK_FAST_PUSH(s, CAR(leaf));
+			    }
+			}
+			if (is_not_list(CDR(leaf))) {
+			    ESTACK_FAST_PUSH(s, CDR(leaf));
+			}
+			else {
+			    if ((m = is_string(CDR(leaf))) && (m < MAX_STRING_LEN)) {
+				result += m + 2 + 1;
+			    } else {
+				result += 5;
+				ESTACK_FAST_PUSH(s, CDR(leaf));
+			    }
+			}
+		    }
+		    else {
+			ESTACK_FAST_PUSH(s, *ptr);
+		    }
+		    ptr++;
 		}
-		result += 1 + 4; /* tag + 4 bytes size */
 	    }
 
 	    break;
-- 
cgit v1.2.3


From 2e016f6cde89bf471269c89f0fd2bb40422ce204 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 26 Mar 2015 12:07:00 +0100
Subject: Misc fixes

---
 erts/emulator/beam/erl_time_sup.c | 2 +-
 erts/emulator/beam/time.c         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 8203436c85..bbdedcc128 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1693,7 +1693,7 @@ static void
 send_time_offset_changed_notifications(void *new_offsetp)
 {
     ErtsMonotonicTime new_offset;
-    ErtsTimeOffsetMonitorInfo *to_mon_info;
+    ErtsTimeOffsetMonitorInfo *to_mon_info = NULL; /* Shut up faulty warning */
     Uint no_monitors;
     char *tmp = NULL;
 
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 3dfd3f79d4..2bdda6c8af 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -576,7 +576,7 @@ erts_cancel_timer(ErlTimer *p)
 {
     ErtsTimerWheel *tiw;
     ErlCancelProc cancel;
-    void *arg;
+    void *arg = NULL; /* Shut up faulty warning... */
 
     tiw = get_timer_wheel(p);
     if (!tiw)
-- 
cgit v1.2.3


From be6af527e25ed548a392c78ea54942555bed94c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 26 Mar 2015 15:20:44 +0100
Subject: erts: Fix missing case in make_internal_hash

---
 erts/emulator/beam/utils.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index cb4ef2b376..8fc8962e4f 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1705,6 +1705,7 @@ make_internal_hash(Eterm term)
                     }
                     goto pop_next;
                 }
+                case HAMT_SUBTAG_HEAD_ARRAY:
                 case HAMT_SUBTAG_HEAD_BITMAP:
                     size = *ptr++;
                     UINT32_HASH(size, HCONST_16);
-- 
cgit v1.2.3


From 740f19448e936d9f03ceb50d27ec1a76749dda6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 27 Mar 2015 17:27:28 +0100
Subject: erts: Fix make_internal_hash for 0.0 vs -0.0

The internal_hash should produce the same hash value for
identical terms, in this case 0.0 =:= -0.0.
---
 erts/emulator/beam/utils.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 8fc8962e4f..91f4accd30 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1894,10 +1894,12 @@ make_internal_hash(Eterm term)
 	    {
 		FloatDef ff;
 		GET_DOUBLE(term, ff);
+                if (ff.fd == 0.0) {
+                    ff.fd = 0.0; /* ensure pos. 0.0 */
+                }
 		UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
 		goto pop_next;
 	    }
-
 	    default:
 		erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
 	    }
-- 
cgit v1.2.3


From 9a36246ad24561619e182d488d4c8a6825011d34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 27 Mar 2015 18:46:23 +0100
Subject: erts: Eliminate potential heap fragments after Map creation

---
 erts/emulator/beam/beam_emu.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8fcdc72895..fdb84aae42 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6573,6 +6573,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     ptr = &Arg(4);
 
     if (n > 2*MAP_SMALL_MAP_LIMIT) {
+        Eterm res;
 	if (HeapWordsLeft(p) < n) {
 	    erts_garbage_collect(p, n, reg, Arg(2));
 	}
@@ -6589,7 +6590,15 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
 	p->htop = mhp;
 
         factory.p = p;
-	return erts_hashmap_from_array(&factory, thp, n/2, 0);
+        res = erts_hashmap_from_array(&factory, thp, n/2, 0);
+        if (p->mbuf) {
+            Uint live = Arg(2);
+            reg[live] = res;
+            erts_garbage_collect(p, 0, reg, live+1);
+            res       = reg[live];
+            E = p->stop;
+        }
+        return res;
     }
 
     if (HeapWordsLeft(p) < need) {
-- 
cgit v1.2.3


From da14897a6363a5c39180e8e3edf0c1c8336cfeb5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 30 Mar 2015 17:02:05 +0200
Subject: erts: Suppress valgrind for bif_SUITE:erlang_halt

which does a deliberate deref of null pointer which is caught by
a SEGV signal handler to resume crash dumping.
---
 erts/emulator/beam/erl_bif_info.c | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b2658a1fd6..fa7de23f00 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3747,6 +3747,20 @@ BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1)
 
 static erts_smp_atomic_t hipe_test_reschedule_flag;
 
+#if defined(VALGRIND) && defined(__GNUC__)
+/* Force noinline for valgrind suppression */
+static void broken_halt_test(Eterm bif_arg_2) __attribute__((noinline));
+#endif
+
+static void broken_halt_test(Eterm bif_arg_2)
+{
+    /* Ugly ugly code used by bif_SUITE:erlang_halt/1 */
+#if defined(ERTS_HAVE_TRY_CATCH)
+    erts_get_scheduler_data()->run_queue = NULL;
+#endif
+    erl_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2);
+}
+
 
 BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 {
@@ -4040,11 +4054,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    }
 	}
         else if (ERTS_IS_ATOM_STR("broken_halt", BIF_ARG_1)) {
-            /* Ugly ugly code used by bif_SUITE:erlang_halt/1 */
-#if defined(ERTS_HAVE_TRY_CATCH)
-	    erts_get_scheduler_data()->run_queue = NULL;
-#endif
-            erl_exit(ERTS_DUMP_EXIT, "%T", BIF_ARG_2);
+            broken_halt_test(BIF_ARG_2);
         }
 	else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) {
 	    int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2);
-- 
cgit v1.2.3


From e4da94bee94bdd7e0dbb2e5021ab4cc5f89f8256 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 30 Mar 2015 19:54:23 +0200
Subject: erts: Refactor encode_size_struct_int

to handle the "start of list" case in one place and not seven.

Note that this commit reverts (47d6fd3ccf35) back to using WSTACK
and pushing raw pointers. We disable GC while yielding, so this should not
be a problem.
---
 erts/emulator/beam/dist.c     |   4 +-
 erts/emulator/beam/dist.h     |   2 +-
 erts/emulator/beam/external.c | 187 ++++++++++++++++--------------------------
 3 files changed, 73 insertions(+), 120 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index bfecac1612..32f3cda4f5 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -712,7 +712,7 @@ void erts_dsend_context_dtor(Binary* ctx_bin)
     ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
     switch (ctx->dss.phase) {
     case ERTS_DSIG_SEND_PHASE_MSG_SIZE:
-	DESTROY_SAVED_ESTACK(&ctx->dss.u.sc.estack);
+	DESTROY_SAVED_WSTACK(&ctx->dss.u.sc.wstack);
 	break;
     case ERTS_DSIG_SEND_PHASE_MSG_ENCODE:
 	DESTROY_SAVED_WSTACK(&ctx->dss.u.ec.wstack);
@@ -1800,7 +1800,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size);
 
 	    if (is_value(ctx->msg)) {
-		ctx->u.sc.estack.start = NULL;
+		ctx->u.sc.wstack.wstart = NULL;
 		ctx->u.sc.flags = ctx->flags;
 		ctx->u.sc.level = 0;
 		ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 2a2ba0c83f..cd2cc0ef4a 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -282,7 +282,7 @@ typedef struct TTBSizeContext_ {
     int level;
     Uint result;
     Eterm obj;
-    ErtsEStack estack;
+    ErtsWStack wstack;
 } TTBSizeContext;
 
 typedef struct TTBEncodeContext_ {
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 2117dbec62..fe48298155 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1779,7 +1779,7 @@ static void ttb_context_destructor(Binary *context_bin)
 	context->alive = 0;
 	switch (context->state) {
 	case TTBSize:
-	    DESTROY_SAVED_ESTACK(&context->s.sc.estack);
+	    DESTROY_SAVED_WSTACK(&context->s.sc.wstack);
 	    break;
 	case TTBEncode:
 	    DESTROY_SAVED_WSTACK(&context->s.ec.wstack);
@@ -1847,7 +1847,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 	/* Setup enough to get started */
 	context->state = TTBSize;
 	context->alive = 1;
-	context->s.sc.estack.start = NULL;
+	context->s.sc.wstack.wstart = NULL;
 	context->s.sc.flags = flags;
 	context->s.sc.level = level;
     } else {
@@ -3962,51 +3962,35 @@ static int
 encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		       unsigned dflags, Sint *reds, Uint *res)
 {
-    DECLARE_ESTACK(s);
+    DECLARE_WSTACK(s);
     Uint m, i, arity;
     Uint result = 0;
     Sint r = 0;
 
     if (ctx) {
-	ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+	WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
 	r = *reds;
 
-	if (ctx->estack.start) { /* restore saved stack */
-	    ESTACK_RESTORE(s, &ctx->estack);
+	if (ctx->wstack.wstart) { /* restore saved stack */
+	    WSTACK_RESTORE(s, &ctx->wstack);
 	    result = ctx->result;
 	    obj = ctx->obj;
 	}
     }
 
-    goto L_jump_start;
+#define LIST_TAIL_OP ((0 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER)
+#define TERM_ARRAY_OP(N) (((N) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER)
+#define TERM_ARRAY_OP_DEC(OP) ((OP) - (1 << _TAG_PRIMARY_SIZE))
+
+
+    for (;;) {
+	ASSERT(!is_header(obj));
 
- outer_loop:
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-    handle_popped_obj:
-	if (is_list(obj)) {
-	    Eterm* cons = list_val(obj);
-	    Eterm tl;
-
-	    tl = CDR(cons);
-	    obj = CAR(cons);
-	    ESTACK_PUSH(s, tl);
-	} else if (is_nil(obj)) {
-	    result++;
-	    goto outer_loop;
-	} else {
-	    /*
-	     * Other term (in the tail of a non-proper list or
-	     * in a fun's environment).
-	     */
-	}
-    
-    L_jump_start:
 	if (ctx && --r == 0) {
 	    *reds = r;
 	    ctx->obj = obj;
 	    ctx->result = result;
-	    ESTACK_SAVE(s, &ctx->estack);
+	    WSTACK_SAVE(s, &ctx->wstack);
 	    return -1;
 	}
 	switch (tag_val_def(obj)) {
@@ -4089,69 +4073,43 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		result += m + 2 + 1;
 	    } else {
 		result += 5;
-		goto handle_popped_obj;
+		WSTACK_PUSH2(s, (UWord)CDR(list_val(obj)), (UWord)LIST_TAIL_OP);
+		obj = CAR(list_val(obj));
+		continue; /* big loop */
 	    }
 	    break;
 	case TUPLE_DEF:
 	    {
 		Eterm* ptr = tuple_val(obj);
-		Uint i;
 		arity = arityval(*ptr);
 		if (arity <= 0xff) {
 		    result += 1 + 1;
 		} else {
 		    result += 1 + 4;
 		}
-		for (i = 1; i <= arity; ++i) {
-		    if (is_list(ptr[i])) {
-			if ((m = is_string(ptr[i])) && (m < MAX_STRING_LEN)) {
-			    result += m + 2 + 1;
-			    continue;
-			} else {
-			    result += 5;
-			}
-		    }
-		    ESTACK_PUSH(s,ptr[i]);
+		if (arity > 1) {
+		    WSTACK_PUSH2(s, (UWord) (ptr + 2),
+				    (UWord) TERM_ARRAY_OP(arity-1));
 		}
-		goto outer_loop;
+                else if (arity == 0) {
+		    break;
+                }
+		obj = ptr[1];
+		continue; /* big loop */
 	    }
-	    break;
 	case MAP_DEF:
 	    if (is_flatmap(obj)) {
 		flatmap_t *mp = (flatmap_t*)flatmap_val(obj);
 		Uint size = flatmap_get_size(mp);
-		Uint i;
-		Eterm *ptr;
 
 		result += 1 + 4; /* tag + 4 bytes size */
 
-		/* push values first */
-		ptr = flatmap_get_values(mp);
-		for (i = size; i; i--, ptr++) {
-		    if (is_list(*ptr)) {
-			if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) {
-			    result += m + 2 + 1;
-			    continue;
-			} else {
-			    result += 5;
-			}
-		    }
-		    ESTACK_PUSH(s,*ptr);
+                if (size) {
+		    WSTACK_PUSH4(s, (UWord) flatmap_get_values(mp),
+				    (UWord) TERM_ARRAY_OP(size),
+		                    (UWord) flatmap_get_keys(mp),
+				    (UWord) TERM_ARRAY_OP(size));
 		}
-
-		ptr = flatmap_get_keys(mp);
-		for (i = size; i; i--, ptr++) {
-		    if (is_list(*ptr)) {
-			if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) {
-			    result += m + 2 + 1;
-			    continue;
-			} else {
-			    result += 5;
-			}
-		    }
-		    ESTACK_PUSH(s,*ptr);
-		}
-		goto outer_loop;
 	    } else {
 		Eterm *ptr;
 		Eterm hdr;
@@ -4178,35 +4136,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		}
 
 		ptr++;
-		ESTACK_RESERVE(s, node_sz*2);
+		WSTACK_RESERVE(s, node_sz*2);
 		while(node_sz--) {
                     if (is_list(*ptr)) {
-			Eterm* leaf = list_val(*ptr);
-			if (is_not_list(CAR(leaf))) {
-			    ESTACK_FAST_PUSH(s, CAR(leaf));
-			}
-			else {
-			    if ((m = is_string(CAR(leaf))) && (m < MAX_STRING_LEN)) {
-				result += m + 2 + 1;
-			    } else {
-				result += 5;
-				ESTACK_FAST_PUSH(s, CAR(leaf));
-			    }
-			}
-			if (is_not_list(CDR(leaf))) {
-			    ESTACK_FAST_PUSH(s, CDR(leaf));
-			}
-			else {
-			    if ((m = is_string(CDR(leaf))) && (m < MAX_STRING_LEN)) {
-				result += m + 2 + 1;
-			    } else {
-				result += 5;
-				ESTACK_FAST_PUSH(s, CDR(leaf));
-			    }
-			}
-		    }
-		    else {
-			ESTACK_FAST_PUSH(s, *ptr);
+			WSTACK_FAST_PUSH(s, CAR(list_val(*ptr)));
+			WSTACK_FAST_PUSH(s, CDR(list_val(*ptr)));
+                    } else {
+			WSTACK_FAST_PUSH(s, *ptr);
 		    }
 		    ptr++;
 		}
@@ -4262,25 +4198,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		    result += 2 * (1 + 4); /* Index + Uniq */
 		    result += 1 + (funp->num_free < 0x100 ? 1 : 4);
 		}
-		for (i = 1; i < funp->num_free; i++) {
-		    obj = funp->env[i];
-
-		    if (is_not_list(obj)) {
-			/* Push any non-list terms on the stack */
-			ESTACK_PUSH(s, obj);
-		    } else {
-			/* Lists must be handled specially. */
-			if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) {
-			    result += m + 2 + 1;
-			} else {
-			    result += 5;
-			    ESTACK_PUSH(s, obj);
-			}
-		    }
+		if (funp->num_free > 1) {
+		    WSTACK_PUSH2(s, (UWord) (funp->env + 1),
+				    (UWord) TERM_ARRAY_OP(funp->num_free-1));
 		}
 		if (funp->num_free != 0) {
 		    obj = funp->env[0];
-		    goto L_jump_start;
+		    continue; /* big loop */
 		}
 		break;
 	    }
@@ -4303,11 +4227,40 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n",
 		     obj);
 	}
+
+	if (WSTACK_ISEMPTY(s)) {
+	    break;
+	}
+	obj = (Eterm) WSTACK_POP(s);
+
+        if (is_header(obj)) {
+            switch (obj) {
+	    case LIST_TAIL_OP:
+		obj = (Eterm) WSTACK_POP(s);
+		if (is_list(obj)) {
+		    Eterm* cons = list_val(obj);
+
+		    WSTACK_PUSH2(s, (UWord)CDR(cons), (UWord)LIST_TAIL_OP);
+		    obj = CAR(cons);
+		}
+		break;
+
+	    case TERM_ARRAY_OP(1):
+		obj = *(Eterm*)WSTACK_POP(s);
+		break;
+	    default: { /* TERM_ARRAY_OP(N) when N > 1 */
+		Eterm* ptr = (Eterm*) WSTACK_POP(s);
+		WSTACK_PUSH2(s, (UWord) (ptr+1),
+			        (UWord) TERM_ARRAY_OP_DEC(obj));
+		obj = *ptr;
+	    }
+	    }
+	}
     }
 
-    DESTROY_ESTACK(s);
+    WSTACK_DESTROY(s);
     if (ctx) {
-	ASSERT(ctx->estack.start == NULL);
+	ASSERT(ctx->wstack.wstart == NULL);
 	*reds = r;
     }
     *res = result;
-- 
cgit v1.2.3


From 450e5b610ac2de4817606c2966d9fb5a9850887a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 30 Mar 2015 21:45:42 +0200
Subject: erts: Optimize == and /= for unequal big maps

Bail out as soon as we find a diff between maps
if we are not interested in term order.
---
 erts/emulator/beam/erl_bif_lists.c |  2 +-
 erts/emulator/beam/erl_db_tree.c   |  8 ++++----
 erts/emulator/beam/erl_utils.h     | 28 +++++++++++++++-------------
 erts/emulator/beam/utils.c         |  9 +++++----
 4 files changed, 25 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 820ed2385d..e006d57124 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -390,7 +390,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List)
 		Eterm *tuple_ptr = tuple_val(term);
 		if (pos <= arityval(*tuple_ptr)) {
 		    Eterm element = tuple_ptr[pos];
-		    if (CMP(Key, element) == 0) {
+		    if (CMP_EQ(Key, element)) {
 			return term;
 		    }
 		}
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 577da35b75..d90af46659 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1116,7 +1116,7 @@ static int db_select_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	CMP(mpi.least,mpi.most) == 0) {
+	CMP_EQ(mpi.least,mpi.most)) {
 	doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */);
 	RET_TO_BIF(sc.accum,DB_ERROR_NONE);
     }
@@ -1324,7 +1324,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	CMP(mpi.least,mpi.most) == 0) {
+	CMP_EQ(mpi.least,mpi.most)) {
 	doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */);
 	RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE);
     }
@@ -1429,7 +1429,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	CMP(mpi.least,mpi.most) == 0) {
+	CMP_EQ(mpi.least,mpi.most)) {
 	doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */);
 	if (sc.accum != NIL) {
 	    hp=HAlloc(p, 3);
@@ -1673,7 +1673,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     sc.mp = mpi.mp;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	CMP(mpi.least,mpi.most) == 0) {
+	CMP_EQ(mpi.least,mpi.most)) {
 	doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't 
 						      matter */);
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 7cb8972e29..6a28105cb9 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -167,24 +167,26 @@ int eq(Eterm, Eterm);
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
 #if HALFWORD_HEAP
-Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int);
-#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0)
-#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1)
-#define CMP(A,B)                         erts_cmp_rel_opt(A,NULL,B,NULL,0)
-#define CMP_TERM(A,B)                    erts_cmp_rel_opt(A,NULL,B,NULL,1)
+Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int, int);
+#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0,0)
+#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1,0)
+#define CMP(A,B)                         erts_cmp_rel_opt(A,NULL,B,NULL,0,0)
+#define CMP_TERM(A,B)                    erts_cmp_rel_opt(A,NULL,B,NULL,1,0)
+#define CMP_EQ_ONLY(A,B)                 erts_cmp_rel_opt(A,NULL,B,NULL,0,1)
 #else
-Sint cmp(Eterm, Eterm);
-Sint erts_cmp(Eterm, Eterm, int);
-#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp(A,B,0)
-#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp(A,B,1)
-#define CMP(A,B)                         erts_cmp(A,B,0)
-#define CMP_TERM(A,B)                    erts_cmp(A,B,1)
+Sint erts_cmp(Eterm, Eterm, int, int);
+Sint cmp(Eterm a, Eterm b);
+#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp(A,B,0,0)
+#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp(A,B,1,0)
+#define CMP(A,B)                         erts_cmp(A,B,0,0)
+#define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
+#define CMP_EQ_ONLY(A,B)                 erts_cmp(A,B,0,1)
 #endif
 
 #define cmp_lt(a,b)          (CMP((a),(b)) <  0)
 #define cmp_le(a,b)          (CMP((a),(b)) <= 0)
-#define cmp_eq(a,b)          (CMP((a),(b)) == 0)
-#define cmp_ne(a,b)          (CMP((a),(b)) != 0)
+#define cmp_eq(a,b)          (CMP_EQ_ONLY((a),(b)) == 0)
+#define cmp_ne(a,b)          (CMP_EQ_ONLY((a),(b)) != 0)
 #define cmp_ge(a,b)          (CMP((a),(b)) >= 0)
 #define cmp_gt(a,b)          (CMP((a),(b)) >  0)
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 8fc8962e4f..889c217b0b 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2911,7 +2911,7 @@ static int cmp_atoms(Eterm a, Eterm b)
  */
 Sint cmp(Eterm a, Eterm b)
 {
-    return erts_cmp(a, b, 0);
+    return erts_cmp(a, b, 0, 0);
 }
 #endif
 
@@ -2920,9 +2920,10 @@ Sint cmp(Eterm a, Eterm b)
  * exact = 0 -> arith-based compare
  */
 #if HALFWORD_HEAP
-Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact)
+Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base,
+                      int exact, int eq_only)
 #else
-Sint erts_cmp(Eterm a, Eterm b, int exact)
+Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
 #endif
 {
 #define PSTACK_TYPE struct erts_cmp_hashmap_state
@@ -3742,7 +3743,7 @@ pop_next:
     return 0;
 
 not_equal:
-    if (!PSTACK_IS_EMPTY(hmap_stack)) {
+    if (!PSTACK_IS_EMPTY(hmap_stack) && !eq_only) {
         WSTACK_ROLLBACK(stack, PSTACK_TOP(hmap_stack)->wstack_rollback);
         goto pop_next;
     }
-- 
cgit v1.2.3


From 8f7246a7c02a50561c171f3d91170a2af96eddbf Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 31 Mar 2015 12:13:57 +0200
Subject: erts: Optimize insert and delete for big maps

Do fast path without bit count for full internal nodes.
---
 erts/emulator/beam/erl_map.c | 71 +++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 31 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ab40f47919..5a70407e42 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1937,26 +1937,31 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 		    case HAMT_SUBTAG_NODE_BITMAP:
 			hval = MAP_HEADER_VAL(hdr);
 			ix   = hashmap_index(hx);
-			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-			n    = hashmap_bitcount(hval);
+                        bp   = 1 << ix;
+                        if (hval == 0xffff) {
+                            slot = ix;
+                            n = 16;
+                        } else {
+                            slot = hashmap_bitcount(hval & (bp - 1));
+                            n    = hashmap_bitcount(hval);
+                        }
+
+                        ESTACK_PUSH4(*sp, n, bp, slot, node);
+
+                        if (!(bp & hval)) { /* not occupied */
+                            if (is_update) {
+                                return 0;
+                            }
+                            size += HAMT_NODE_BITMAP_SZ(n+1);
+                            goto unroll;
+                        }
+
+                        hx    = hashmap_shift_hash(th,hx,lvl,key);
+                        node  = ptr[slot+1];
+                        ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
+                        size += HAMT_NODE_BITMAP_SZ(n);
+                        break;
 
-			ESTACK_PUSH4(*sp, n, bp, slot, node);
-
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+1];
-			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
-			    size += HAMT_NODE_BITMAP_SZ(n);
-			    break;
-			}
-			/* not occupied */
-			if (is_update) {
-			    return 0;
-			}
-			size += HAMT_NODE_BITMAP_SZ(n+1);
-			goto unroll;
 		    case HAMT_SUBTAG_HEAD_BITMAP:
 			hval = MAP_HEADER_VAL(hdr);
 			ix   = hashmap_index(hx);
@@ -2183,21 +2188,25 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
 			hval = MAP_HEADER_VAL(hdr);
 			ix   = hashmap_index(hx);
 			bp   = 1 << ix;
-			slot = hashmap_bitcount(hval & (bp - 1));
-			n    = hashmap_bitcount(hval);
+                        if (hval == 0xffff) {
+                            slot = ix;
+                            n = 16;
+                        } else if (bp & hval) {
+                            slot = hashmap_bitcount(hval & (bp - 1));
+                            n    = hashmap_bitcount(hval);
+                        } else {
+                            /* not occupied */
+                            goto not_found;
+                        }
 
 			ESTACK_PUSH4(stack, n, bp, slot, node);
 
-			/* occupied */
-			if (bp & hval) {
-			    hx    = hashmap_shift_hash(th,hx,lvl,key);
-			    node  = ptr[slot+1];
-			    ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
-			    size += HAMT_NODE_BITMAP_SZ(n);
-			    break;
-			}
-			/* not occupied */
-			goto not_found;
+                        hx    = hashmap_shift_hash(th,hx,lvl,key);
+                        node  = ptr[slot+1];
+                        ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17);
+                        size += HAMT_NODE_BITMAP_SZ(n);
+                        break;
+
 		    case HAMT_SUBTAG_HEAD_BITMAP:
 			hval = MAP_HEADER_VAL(hdr);
 			ix   = hashmap_index(hx);
-- 
cgit v1.2.3


From dbad08513df1ea3f3de47375a50d16551df6f208 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 31 Mar 2015 14:49:02 +0200
Subject: erts: Fix size bug in maps:from_list/1 BIF

The wrong size was imprinted on maps with deep hash key collisions.
---
 erts/emulator/beam/erl_map.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ab40f47919..a8e8ad346b 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -89,7 +89,7 @@ static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys);
 static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root);
-static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root);
+static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, Uint size, int is_root);
 static Eterm hashmap_info(Process *p, Eterm node);
 static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 static int hxnodecmp(hxnode_t* a, hxnode_t* b);
@@ -661,7 +661,7 @@ static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory,
 	ix++;
     }
 
-    res = hashmap_from_chunked_array(factory, hxns, elems, !lvl);
+    res = hashmap_from_chunked_array(factory, hxns, elems, n, !lvl);
 
     ERTS_FACTORY_HOLE_CHECK(factory);
 
@@ -669,8 +669,8 @@ static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory,
 }
 
 #define HALLOC_EXTRA 200
-static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
-                                        hxnode_t *hxns, Uint n, int is_root) {
+static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns, Uint n,
+                                        Uint size, int is_root) {
     Uint ix, d, dn, dc, slot, elems;
     Uint32 v, vp, vn, hdr;
     Uint bp, sz;
@@ -840,7 +840,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory,
 
     if (is_root) {
 	*hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr);
-	*hp++ = n;
+	*hp++ = size;
     } else {
 	*hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr);
     }
-- 
cgit v1.2.3


From 71a2a9e102fdcc904cf81dc6369663d79bbfc54a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 31 Mar 2015 18:09:51 +0200
Subject: erts: Remove unused tmp heap in make_internal_hash

---
 erts/emulator/beam/utils.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 91f4accd30..2dd81e3ca3 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1614,7 +1614,6 @@ make_internal_hash(Eterm term)
     Eterm tmp;
     DECLARE_ESTACK(s);
 
-    UseTmpHeapNoproc(2);
     hash = 0;
     for (;;) {
 	switch (primary_tag(term)) {
@@ -1919,7 +1918,6 @@ make_internal_hash(Eterm term)
 	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
-		UnUseTmpHeapNoproc(2);
 		return hash;
 	    }
 
-- 
cgit v1.2.3


From 7556ab9ee0949c94e13d1ccfc6c7755637730be4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 31 Mar 2015 18:20:31 +0200
Subject: erts: Use halfword secure tmp heap

---
 erts/emulator/beam/erl_map.c | 38 +++++++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ab40f47919..2e31718629 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -392,12 +392,11 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     Eterm item = list;
     Eterm *hp;
     Eterm *kv, res;
-    Eterm tmp[2];
     Uint32 sw, hx;
     Uint ix = 0;
     hxnode_t *hxns;
     ErtsHeapFactory factory;
-
+    DeclareTmpHeap(tmp,2,p);
     ASSERT(size > 0);
 
     hp = HAlloc(p, (2 * size));
@@ -405,6 +404,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     /* create tmp hx values and leaf ptrs */
     hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, size * sizeof(hxnode_t));
 
+    UseTmpHeap(2,p);
     while(is_list(item)) {
 	res = CAR(list_val(item));
 	kv  = tuple_val(res);
@@ -417,6 +417,7 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
 	ix++;
 	item = CDR(list_val(item));
     }
+    UnUseTmpHeap(2,p);
 
     factory.p = p;
     res = hashmap_from_unsorted_array(&factory, hxns, size, 0);
@@ -625,9 +626,9 @@ static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory,
     Uint i,ix,jx,elems;
     Uint32 sw, hx;
     Eterm val;
-    Eterm th[2];
     hxnode_t *tmp;
-
+    DeclareTmpHeapNoproc(th,2);
+    UseTmpHeapNoproc(2);
     ASSERT(lvl < 32);
     ix = 0;
     elems = 1;
@@ -665,6 +666,7 @@ static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory,
 
     ERTS_FACTORY_HOLE_CHECK(factory);
 
+    UnUseTmpHeapNoproc(2);
     return res;
 }
 
@@ -1099,12 +1101,13 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     struct HashmapMergePStackType* sp = PSTACK_PUSH(s);
     Eterm *hp, *nhp;
     Eterm hdrA, hdrB;
-    Eterm th[2];
     Uint32 ahx, bhx;
     Uint size;  /* total key-value counter */
     int keepA = 0;
-    unsigned lvl = 0;
+    unsigned int lvl = 0;
+    DeclareTmpHeap(th,2,p);
     Eterm res = THE_NON_VALUE;
+    UseTmpHeap(2,p);
 
     /*
      * Strategy: Do depth-first traversal of both trees (at the same time)
@@ -1297,6 +1300,7 @@ recurse:
 	res = make_boxed(nhp);
     }
     PSTACK_DESTROY(s);
+    UnUseTmpHeap(2,p);
     return res;
 }
 
@@ -1315,8 +1319,9 @@ static int hash_cmp(Uint32 ha, Uint32 hb)
 
 int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
 {
-    Eterm th[2];
-    unsigned lvl = 0;
+    unsigned int lvl = 0;
+    DeclareTmpHeapNoproc(th,2);
+    UseTmpHeapNoproc(2);
 
     if (ap && bp) {
 	ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0);
@@ -1324,11 +1329,14 @@ int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
 	    Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap));
 	    Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp));
 	    int cmp = hash_cmp(ha, hb);
-	    if (cmp)
+	    if (cmp) {
+                UnUseTmpHeapNoproc(2);
 		return cmp;
+            }
 	    lvl += 8;
 	}
     }
+    UnUseTmpHeapNoproc(2);
     return ap ? -1 : 1;
 }
 
@@ -1901,13 +1909,14 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 			     Uint *update_size, ErtsEStack *sp, int is_update) {
     Eterm *ptr;
     Eterm hdr, ckey;
-    Eterm th[2];
     Uint32 ix, cix, bp, hval, chx;
     Uint slot, lvl = 0, clvl;
     Uint size = 0, n = 0;
+    DeclareTmpHeapNoproc(th,2);
 
     *update_size = 1;
 
+    UseTmpHeapNoproc(2);
     for (;;) {
 	switch(primary_tag(node)) {
 	    case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */
@@ -1918,6 +1927,7 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 		    goto unroll;
 		}
 		if (is_update) {
+                    UnUseTmpHeapNoproc(2);
 		    return 0;
 		}
 		goto insert_subnodes;
@@ -1953,6 +1963,7 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 			}
 			/* not occupied */
 			if (is_update) {
+                            UnUseTmpHeapNoproc(2);
 			    return 0;
 			}
 			size += HAMT_NODE_BITMAP_SZ(n+1);
@@ -1976,6 +1987,7 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 			}
 			/* not occupied */
 			if (is_update) {
+                            UnUseTmpHeapNoproc(2);
 			    return 0;
 			}
 			size += HAMT_HEAD_BITMAP_SZ(n+1);
@@ -2009,6 +2021,7 @@ insert_subnodes:
 
 unroll:
     *sz = size + /* res cons */ 2;
+    UnUseTmpHeapNoproc(2);
     return 1;
 }
 
@@ -2151,13 +2164,14 @@ static Eterm hashmap_values(Process* p, Eterm node) {
 
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
     Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
-    Eterm th[2];
     Eterm *ptr;
     Eterm hdr, res = map, node = map;
     Uint32 ix, bp, hval;
     Uint slot, lvl = 0;
     Uint size = 0, n = 0;
     DECLARE_ESTACK(stack);
+    DeclareTmpHeapNoproc(th,2);
+    UseTmpHeapNoproc(2);
 
     for (;;) {
 	switch(primary_tag(node)) {
@@ -2268,6 +2282,7 @@ unroll:
 	erts_validate_and_sort_flatmap(mp);
 
 	DESTROY_WSTACK(wstack);
+        UnUseTmpHeapNoproc(2);
 	return make_flatmap(mp);
     }
 
@@ -2408,6 +2423,7 @@ not_found:
     DESTROY_ESTACK(stack);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
     ERTS_HOLE_CHECK(p);
+    UnUseTmpHeapNoproc(2);
     return res;
 }
 
-- 
cgit v1.2.3


From 30ed5b1cc6aa0efe6ac099b66d33d46c7c0c6b47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 7 Apr 2015 12:58:15 +0200
Subject: erts: Fix deep colliding hash values in maps:from_list/1

Reported-by: Jesper Louis Andersen
---
 erts/emulator/beam/erl_map.c | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 20a17bcd24..3bb3622194 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -679,7 +679,35 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns
     DECLARE_ESTACK(stack);
     Eterm res = NIL, *hp = NULL, *nhp;
 
-    ASSERT(n > 1);
+
+    /* if we get here with only one element then
+     * we have eight levels of collisions
+     */
+
+    if (n == 1) {
+	res = hxns[0].val;
+	v   = hxns[0].hx;
+	for (d = 7; d > 0; d--) {
+	    slot  = maskval(v,d);
+	    hp    = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
+	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot);
+	    hp[1] = res;
+	    res   = make_hashmap(hp);
+	}
+
+	slot  = maskval(v,0);
+	hp    = erts_produce_heap(factory, (is_root ? 3 : 2), 0);
+
+	if (is_root) {
+	    hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << slot);
+	    hp[1] = size;
+	    hp[2] = res;
+	} else {
+	    hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot);
+	    hp[1] = res;
+	}
+	return make_hashmap(hp);
+    }
 
     /* push initial nodes on the stack,
      * this is the starting depth */
-- 
cgit v1.2.3


From 6492bf35902f476ef0eac972ef49c424fa6d8bc6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 7 Apr 2015 17:05:46 +0200
Subject: erts: Fix bug in binary_to_term for big maps with 32 bit hash-clash

binary_to_term threw badarg as the "reject_dupkey" case in
hashmap_from_unsored_array was always triggered when hash-clash was
found as the first round in the loop compared the key with itself.
---
 erts/emulator/beam/erl_map.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 20a17bcd24..4acf830655 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -568,14 +568,14 @@ static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory,
 
 	    while(ix < jx) {
 		lx = ix;
-		while(ix < jx && EQ(CAR(list_val(hxns[ix].val)), CAR(list_val(hxns[lx].val)))) {
+		while(++ix < jx && EQ(CAR(list_val(hxns[ix].val)),
+				      CAR(list_val(hxns[lx].val)))) {
                     if (reject_dupkeys)
                         return THE_NON_VALUE;
 
                     if (hxns[ix].i > hxns[lx].i) {
 			lx = ix;
 		    }
-		    ix++;
 		}
 		hxns[cx].hx  = hxns[lx].hx;
 		hxns[cx].val = hxns[lx].val;
-- 
cgit v1.2.3


From e3f21d4911117b80e80ea4a07f3532ad39ede16b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 7 Apr 2015 21:28:05 +0200
Subject: erts: Fix bug in map_from_list when keys clash in both value and hash

Subtle bug in qsort callback. Cast from Sint to int does not retain sign.
---
 erts/emulator/beam/erl_map.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4acf830655..14a8afe3a6 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -860,7 +860,12 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns
 #undef HALLOC_EXTRA
 
 static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) {
-    return CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val)));
+    Sint c = CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val)));
+#if ERTS_SIZEOF_ETERM <= SIZEOF_INT
+    return c;
+#else
+    return c > 0 ? 1 : (c < 0 ? -1 : 0);
+#endif
 }
 
 static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
-- 
cgit v1.2.3


From a4d1a73370dffa6ac96ee70693fd1bd335bd70fe Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 8 Apr 2015 17:02:15 +0200
Subject: erts: Fix ets bug in debug VM

Symptom: ASSERT(segtab[seg_ix] == NULL) in alloc_seg() fails.

Remedy: Make sure we set segment pointer to NULL in free_seg()
even when we switch to smaller segtab.
---
 erts/emulator/beam/erl_db_hash.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 06dac8f161..b1d9eb84bc 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2402,10 +2402,10 @@ static int alloc_seg(DbTableHash *tb)
 */
 static int free_seg(DbTableHash *tb, int free_records)
 {
-    int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1;
+    const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1;
+    struct segment** const segtab = SEGTAB(tb);
+    struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix];
     int bytes;
-    struct segment** segtab = SEGTAB(tb);
-    struct ext_segment* top = (struct ext_segment*) segtab[seg_ix];
     int nrecords = 0;
 
     ASSERT(top != NULL); 
@@ -2468,7 +2468,7 @@ static int free_seg(DbTableHash *tb, int free_records)
 		 (void*)top, bytes);
 #ifdef DEBUG
     if (seg_ix > 0) {
-	if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL;
+        segtab[seg_ix] = NULL;
     } else {
 	SET_SEGTAB(tb, NULL);
     }
-- 
cgit v1.2.3


From fe150e807667cf3aa1ecdd865a1885bdc326b0f6 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Sat, 28 Mar 2015 14:53:07 -0400
Subject: Keep dirty schedulers from waking other schedulers

Prevent dirty schedulers from checking regular run queues and trying
to wake up regular schedulers.
---
 erts/emulator/beam/erl_process.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f74a2ee54c..5f7770e65f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -4960,9 +4960,11 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 				 + ERTS_WAKEUP_OTHER_FIXED_INC);
 	    if (rq->wakeup_other > wakeup_other.limit) {
 #ifdef ERTS_DIRTY_SCHEDULERS
-		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->waiting)
-		    wake_dirty_schedulers(rq, 1);
-		else
+		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
+		    if (rq->waiting) {
+			wake_dirty_schedulers(rq, 1);
+		    }
+		} else
 #endif
 		{
 		    int empty_rqs =
-- 
cgit v1.2.3


From 1a191c166c446b21f515429fc9987e5a7add5ae5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 10 Apr 2015 15:12:45 +0200
Subject: erts: Fix building of Map result from match_specs

A faulty "box-value" entered into the heap which could cause
a segmentation fault in the garbage collector if it was written
on a heap fragment.
---
 erts/emulator/beam/erl_db_util.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 0bf562d937..0fb1c397c9 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2153,8 +2153,8 @@ restart:
 	    break;
         case matchMkFlatMap:
             n = *pc++;
-            ehp = HAllocX(build_proc, 1 + MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA);
-            t = *ehp++ = *--esp;
+            ehp = HAllocX(build_proc, MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA);
+            t = *--esp;
             {
                 flatmap_t *m = (flatmap_t *)ehp;
                 m->thing_word = MAP_HEADER_FLATMAP;
-- 
cgit v1.2.3


From d4bf1ea63a3ada92394378ce176b3c4aeba246d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 10 Apr 2015 15:17:24 +0200
Subject: erts: Remove code that was commented out

---
 erts/emulator/beam/erl_term.h | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 095aa54ddd..cff012d5d1 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -298,7 +298,6 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm)
 /* header (arityval or thing) access methods */
 #define _make_header(sz,tag) ((Uint)(((Uint)(sz) << _HEADER_ARITY_OFFS) + (tag)))
 #define is_header(x)	(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER)
-//#define _unchecked_header_arity(x)	((x) >> _HEADER_ARITY_OFFS)
 #define _unchecked_header_arity(x) \
     (is_map_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS))
 _ET_DECLARE_CHECKED(Uint,header_arity,Eterm)
-- 
cgit v1.2.3


From 2a0ff8b50a68d51e44fddf36867f390b64a4f2bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 1 Apr 2015 15:30:12 +0200
Subject: erts: Ensure hashing of zero is consistent

Erlang treats positive and negative zero as equal, meaning,

    true = 0.0 =:= 0.0/-1

However, Erlangs hash functions: hash, phash and phash2 did not
reflect this behaviour. Meaning, the hash values produced by
the different hash functions would not be identical for positive
and negative zero.

This commit ensures that hash value of positive zero is always
produced regardless of the signedness of the zero float, i.e.

    true = erlang:phash2(0.0) =:= erlang:phash2(0.0/-1)
---
 erts/emulator/beam/utils.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 2dd81e3ca3..2018518bf3 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -955,12 +955,14 @@ tail_recur:
 	UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10);
     case FLOAT_DEF: 
 	{
-	    FloatDef ff;
-	    GET_DOUBLE(term, ff);
-	    hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
-	    break;
+            FloatDef ff;
+            GET_DOUBLE(term, ff);
+            if (ff.fd == 0.0f) {
+                ff.fd = 0.0f; /* ensure pos. 0.0 */
+            }
+            hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
+            break;
 	}
-
     case MAKE_HASH_CDR_PRE_OP:
 	term = (Eterm) WSTACK_POP(stack);
 	if (is_not_list(term)) {
@@ -1474,6 +1476,9 @@ make_hash2(Eterm term)
 	    {
 		FloatDef ff;
 		GET_DOUBLE(term, ff);
+                if (ff.fd == 0.0f) {
+                    ff.fd = 0.0f; /* ensure pos. 0.0 */
+                }
 #if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
 		UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
 #else
@@ -1893,8 +1898,8 @@ make_internal_hash(Eterm term)
 	    {
 		FloatDef ff;
 		GET_DOUBLE(term, ff);
-                if (ff.fd == 0.0) {
-                    ff.fd = 0.0; /* ensure pos. 0.0 */
+                if (ff.fd == 0.0f) {
+                    ff.fd = 0.0f; /* ensure pos. 0.0 */
                 }
 		UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
 		goto pop_next;
@@ -2079,12 +2084,14 @@ tail_recur:
 	break;
     case FLOAT_DEF: 
 	{
-	    FloatDef ff;
-	    GET_DOUBLE(term, ff);
-	    hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
+            FloatDef ff;
+            GET_DOUBLE(term, ff);
+            if (ff.fd == 0.0f) {
+                ff.fd = 0.0f; /* ensure pos. 0.0 */
+            }
+            hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
 	}
 	break;
-
     case MAKE_HASH_CDR_PRE_OP:
 	term = (Eterm) WSTACK_POP(stack);
 	if (is_not_list(term)) {
-- 
cgit v1.2.3


From 9929a1fd151a78247f1d536f2a21843f310a4e3b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 10 Apr 2015 19:12:48 +0200
Subject: erts: Add etp_the_non_value

for a correct (non)value regardless of build type.
---
 erts/emulator/beam/erl_init.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 86d3416423..1346892e29 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -118,6 +118,8 @@ const int etp_big_endian = 1;
 #else
 const int etp_big_endian = 0;
 #endif
+const Eterm etp_the_non_value = THE_NON_VALUE;
+
 /*
  * Note about VxWorks: All variables must be initialized by executable code,
  * not by an initializer. Otherwise a new instance of the emulator will
-- 
cgit v1.2.3


From 82b0a889fd2534af81e21c277657adf58699d73c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 20 Mar 2015 12:20:52 +0100
Subject: Remove support for put_map_exact without a source map

Using the exact operator (':=') is only allowed when an existing map
is being updated. Thus the following causes a compilation error:

  #{k:=v}

Therefore there is no need to support the put_map_exact instruction
without a source map.
---
 erts/emulator/beam/ops.tab | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index d3649080dc..ae3b7d08b8 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1478,7 +1478,7 @@ put_map_assoc F Src=s Dst Live Size Rest=* => \
 	       update_map_assoc F Src Dst Live Size Rest
 put_map_assoc F Src Dst Live Size Rest=* => \
 	       move Src x | update_map_assoc F x Dst Live Size Rest
-put_map_exact F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
+
 put_map_exact F Src=s Dst Live Size Rest=* => \
 	      update_map_exact F Src Dst Live Size Rest
 put_map_exact F Src Dst Live Size Rest=* => \
-- 
cgit v1.2.3


From 8c5a577ed55a607841989763d78b3950eebd5b5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 20 Mar 2015 14:45:31 +0100
Subject: Correct transformation of put_map_assoc to new_map

A put_map_assoc instruction with an empty source map should be
converted to a simpler new_map instruction. The transformation
didn't happen because an empty source map is no longer represented
as a NIL term (as it was in the beginning before map literals
were implemented).
---
 erts/emulator/beam/beam_load.c | 17 +++++++++++++++++
 erts/emulator/beam/ops.tab     |  3 ++-
 2 files changed, 19 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 02689e5b19..f140bb54cc 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -36,6 +36,7 @@
 #include "beam_catches.h"
 #include "erl_binary.h"
 #include "erl_zlib.h"
+#include "erl_map.h"
 
 #ifdef HIPE
 #include "hipe_bif0.h"
@@ -4050,6 +4051,22 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
     return op;
 }
 
+/*
+ * Predicate to test whether the given literal is an empty map.
+ */
+
+static int
+is_empty_map(LoaderState* stp, GenOpArg Lit)
+{
+    Eterm term;
+
+    if (Lit.type != TAG_q) {
+	return 0;
+    }
+    term = stp->literals[Lit.val].term;
+    return is_flatmap(term) && flatmap_get_size(flatmap_val(term)) == 0;
+}
+
 /*
  * Replace a get_map_elements with one key to an instruction with one
  * element
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index ae3b7d08b8..2c4458ae74 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1473,7 +1473,8 @@ apply_last I P
 # Map instructions in R17.
 #
 
-put_map_assoc F n Dst Live Size Rest=* => new_map F Dst Live Size Rest
+put_map_assoc F Map Dst Live Size Rest=* | is_empty_map(Map) => \
+   new_map F Dst Live Size Rest
 put_map_assoc F Src=s Dst Live Size Rest=* => \
 	       update_map_assoc F Src Dst Live Size Rest
 put_map_assoc F Src Dst Live Size Rest=* => \
-- 
cgit v1.2.3


From ee91c4291212035b8e054072407c74afdd66f1d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 20 Mar 2015 15:07:24 +0100
Subject: Remove the fail label operand of the new_map instruction

The new_map instruction cannot fail, and thus needs no fail label.
---
 erts/emulator/beam/beam_debug.c | 2 +-
 erts/emulator/beam/beam_emu.c   | 8 ++++----
 erts/emulator/beam/ops.tab      | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 6bb987985d..38e54e9d1a 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -661,7 +661,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_put_tuple_rI:
     case op_i_put_tuple_xI:
     case op_i_put_tuple_yI:
-    case op_new_map_jdII:
+    case op_new_map_dII:
     case op_update_map_assoc_jsdII:
     case op_update_map_exact_jsdII:
     case op_i_has_map_fields_fsI:
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index fdb84aae42..a264669e50 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2379,16 +2379,16 @@ void process_main(void)
      Goto(*I);
  }
 
- OpCase(new_map_jdII): {
+ OpCase(new_map_dII): {
      Eterm res;
 
      x(0) = r(0);
      SWAPOUT;
-     res = new_map(c_p, reg, I);
+     res = new_map(c_p, reg, I-1);
      SWAPIN;
      r(0) = x(0);
-     StoreResult(res, Arg(1));
-     Next(4+Arg(3));
+     StoreResult(res, Arg(0));
+     Next(3+Arg(2));
  }
 
  OpCase(i_has_map_fields_fsI): {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 2c4458ae74..7993dd9e25 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1473,8 +1473,8 @@ apply_last I P
 # Map instructions in R17.
 #
 
-put_map_assoc F Map Dst Live Size Rest=* | is_empty_map(Map) => \
-   new_map F Dst Live Size Rest
+put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \
+   new_map Dst Live Size Rest
 put_map_assoc F Src=s Dst Live Size Rest=* => \
 	       update_map_assoc F Src Dst Live Size Rest
 put_map_assoc F Src Dst Live Size Rest=* => \
@@ -1485,7 +1485,7 @@ put_map_exact F Src=s Dst Live Size Rest=* => \
 put_map_exact F Src Dst Live Size Rest=* => \
 	      move Src x | update_map_exact F x Dst Live Size Rest
 
-new_map j d I I
+new_map d I I
 update_map_assoc j s d I I
 update_map_exact j s d I I
 
-- 
cgit v1.2.3


From 85afb5e3441e78cd0d572dc809d94cfc810d71ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 23 Mar 2015 10:24:56 +0100
Subject: Fully evaluate is_map/1 for literals at load-time

The compiler will only emit is_map/1 instructions with literal
argument if optimization is turned off. Therefore, the only
reason for this commit is cleanliness.
---
 erts/emulator/beam/beam_load.c | 14 ++++++++++++++
 erts/emulator/beam/ops.tab     |  4 ++--
 2 files changed, 16 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index f140bb54cc..60f4ab5280 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4051,6 +4051,20 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
     return op;
 }
 
+/*
+ * Predicate to test whether the given literal is a map.
+ */
+
+static int
+literal_is_map(LoaderState* stp, GenOpArg Lit)
+{
+    Eterm term;
+
+    ASSERT(Lit.type == TAG_q);
+    term = stp->literals[Lit.val].term;
+    return is_map(term);
+}
+
 /*
  * Predicate to test whether the given literal is an empty map.
  */
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 7993dd9e25..a5a89b3990 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1489,8 +1489,8 @@ new_map d I I
 update_map_assoc j s d I I
 update_map_exact j s d I I
 
-is_map Fail Literal=q => move Literal x | is_map Fail x
-is_map Fail c => jump Fail
+is_map Fail Lit=q | literal_is_map(Lit) =>
+is_map Fail cq => jump Fail
 
 %macro: is_map IsMap -fail_action
 is_map f r
-- 
cgit v1.2.3


From c92cf260bb888c004fb02651670d19989dbc2b74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 26 Mar 2015 13:16:45 +0100
Subject: De-optimize the has_map_fields instructions

The has_map_fields instruction is infrequently used. Thus there
is no need to have the fastest possible implementation; it is
better to have an implementation that reduces the code size in
the already big process_main() function.

We can transform has_map_fields to a get_map_elements instruction,
targeting the same unused x[0] register for all keys. That
instruction will only be marginally slower than existing
implementation.
---
 erts/emulator/beam/beam_debug.c |  1 -
 erts/emulator/beam/beam_emu.c   | 96 -----------------------------------------
 erts/emulator/beam/beam_load.c  | 20 ++++++---
 erts/emulator/beam/ops.tab      | 29 ++-----------
 4 files changed, 19 insertions(+), 127 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 38e54e9d1a..0367ca8aba 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -664,7 +664,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_new_map_dII:
     case op_update_map_assoc_jsdII:
     case op_update_map_exact_jsdII:
-    case op_i_has_map_fields_fsI:
     case op_i_get_map_elements_fsI:
 	{
 	    int n = unpacked[-1];
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a264669e50..f25b9f594d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -701,8 +701,6 @@ void** beam_ops;
 
 #define IsMap(Src, Fail) if (!is_map(Src)) { Fail; }
 
-#define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; }
-
 #define GetMapElement(Src, Key, Dst, Fail)	\
   do {						\
      Eterm _res = get_map_element(Src, Key);	\
@@ -960,7 +958,6 @@ static Eterm update_map_assoc(Process* p, Eterm* reg,
 			      Eterm map, BeamInstr* I) NOINLINE;
 static Eterm update_map_exact(Process* p, Eterm* reg,
 			      Eterm map, BeamInstr* I) NOINLINE;
-static int has_not_map_field(Eterm map, Eterm key);
 static Eterm get_map_element(Eterm map, Eterm key);
 
 /*
@@ -2391,67 +2388,6 @@ void process_main(void)
      Next(3+Arg(2));
  }
 
- OpCase(i_has_map_fields_fsI): {
-    flatmap_t* mp;
-    Eterm map;
-    Eterm field;
-    Eterm *ks;
-    BeamInstr* fs;
-    Uint sz,n;
-
-    GetArg1(1, map);
-    n  = (Uint)Arg(2);
-    fs = &Arg(3); /* pattern fields */
-
-    /* get term from field? */
-    if (is_hashmap(map)) {
-	Uint32 hx;
-	while(n--) {
-	    field = *fs++;
-	    hx = hashmap_make_hash(field);
-	    if (!erts_hashmap_get(hx,field,map)) {
-		SET_I((BeamInstr *) Arg(0));
-		goto has_map_fields_fail;
-	    }
-	}
-	goto has_map_fields_ok;
-    }
-
-    ASSERT(is_flatmap(map));
-
-    mp = (flatmap_t *)flatmap_val(map);
-    sz = flatmap_get_size(mp);
-
-    if (sz == 0) {
-	SET_I((BeamInstr *) Arg(0));
-	goto has_map_fields_fail;
-    }
-
-    ks = flatmap_get_keys(mp);
-
-    ASSERT(n>0);
-
-    while(sz) {
-	field = (Eterm)*fs;
-	if (EQ(field,*ks)) {
-	    n--;
-	    fs++;
-	    if (n == 0) break;
-	}
-	ks++; sz--;
-    }
-
-    if (n) {
-	SET_I((BeamInstr *) Arg(0));
-	goto has_map_fields_fail;
-    }
-has_map_fields_ok:
-    I += 4 + Arg(2);
-has_map_fields_fail:
-    ASSERT(VALID_INSTR(*I));
-    Goto(*I);
- }
-
 #define PUT_TERM_REG(term, desc)				\
 do {								\
     switch ((desc) & _TAG_IMMED1_MASK) {			\
@@ -6470,38 +6406,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     return make_fun(funp);
 }
 
-static int has_not_map_field(Eterm map, Eterm key)
-{
-    Uint32 hx;
-    if (is_flatmap(map)) {
-	flatmap_t* mp;
-	Eterm* keys;
-	Uint i;
-	Uint n;
-
-	mp   = (flatmap_t *)flatmap_val(map);
-	keys = flatmap_get_keys(mp);
-	n    = flatmap_get_size(mp);
-	if (is_immed(key)) {
-	    for (i = 0; i < n; i++) {
-		if (keys[i] == key) {
-		    return 0;
-		}
-	    }
-	} else {
-	    for (i = 0; i <  n; i++) {
-		if (EQ(keys[i], key)) {
-		    return 0;
-		}
-	    }
-	}
-	return 1;
-    }
-    ASSERT(is_hashmap(map));
-    hx = hashmap_make_hash(key);
-    return erts_hashmap_get(hx,key,map) ? 0 : 1;
-}
-
 static Eterm get_map_element(Eterm map, Eterm key)
 {
     Uint32 hx;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 60f4ab5280..18e1e312ad 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4107,21 +4107,31 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
 }
 
 static GenOp*
-gen_has_map_field(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
-		    GenOpArg Size, GenOpArg* Rest)
+gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
+		   GenOpArg Size, GenOpArg* Rest)
 {
     GenOp* op;
+    Uint i;
+    Uint n;
 
     ASSERT(Size.type == TAG_u);
+    n = Size.val;
 
     NEW_GENOP(stp, op);
+    GENOP_ARITY(op, 3 + 2*n);
     op->next = NULL;
-    op->op = genop_has_map_field_3;
-    op->arity = 4;
+    op->op = genop_get_map_elements_3;
 
     op->a[0] = Fail;
     op->a[1] = Src;
-    op->a[2] = Rest[0];
+    op->a[2].type = TAG_u;
+    op->a[2].val = 2*n;
+
+    for (i = 0; i < n; i++) {
+	op->a[3+2*i] = Rest[i];
+	op->a[3+2*i+1].type = TAG_x;
+	op->a[3+2*i+1].val = 0;	/* x(0); normally not used */
+    }
     return op;
 }
 
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index a5a89b3990..abaa47217a 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1497,31 +1497,10 @@ is_map f r
 is_map f x
 is_map f y
 
-## Transform has_map_field(s) #{ K1 := _, K2 := _ }
-
-has_map_field/3
-
-has_map_fields Fail Src Size=u==1 Rest=* => gen_has_map_field(Fail,Src,Size,Rest)
-has_map_fields Fail Src Size Rest=* => i_has_map_fields Fail Src Size Rest
-
-i_has_map_fields f s I
-
-has_map_field Fail Src=rxy Key=arxy => i_has_map_field Fail Src Key
-has_map_field Fail Src Key => move Key x | i_has_map_field Fail Src x
-
-%macro: i_has_map_field HasMapField -fail_action
-i_has_map_field f r a
-i_has_map_field f x a
-i_has_map_field f y a
-i_has_map_field f r r
-i_has_map_field f x r
-i_has_map_field f y r
-i_has_map_field f r x
-i_has_map_field f x x
-i_has_map_field f y x
-i_has_map_field f r y
-i_has_map_field f x y
-i_has_map_field f y y
+## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements
+
+has_map_fields Fail Src Size Rest=* => \
+   gen_has_map_fields(Fail, Src, Size, Rest)
 
 ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
 
-- 
cgit v1.2.3


From 529af720ae7b663aa6a1a086b31ba9e3605ff21e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 23 Mar 2015 13:10:20 +0100
Subject: Sort maps keys in the loader

The map instructions require that the keys in the instructions
are sorted (for flatmaps). But that is an implementation detail
that should not exposed outside of the BEAM virtual machine.
Therefore, make the sorting of the keys the responsibility of
the loader and not the compiler.

Also note that the sort order for maps with numeric keys or keys
with numeric components has changed in OTP 18. That means that
code compiled for OTP 17 that operated on maps with map keys
might not work in OTP 18 without the sorting in the loader
(although it is unlikely to be an issue in practice).
---
 erts/emulator/beam/beam_load.c | 80 ++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/ops.tab     | 24 +++++++++----
 2 files changed, 97 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 18e1e312ad..1a6c6c16ad 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4081,6 +4081,86 @@ is_empty_map(LoaderState* stp, GenOpArg Lit)
     return is_flatmap(term) && flatmap_get_size(flatmap_val(term)) == 0;
 }
 
+/*
+ * Pseudo predicate map_key_sort that will sort the Rest operand for
+ * map instructions as a side effect.
+ */
+
+typedef struct SortGenOpArg {
+    Eterm term;			/* Term to use for comparing  */
+    GenOpArg arg;		/* Original data */
+} SortGenOpArg;
+
+static int
+genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b)
+{
+    return CMP_TERM(a->term, b->term);
+}
+
+static int
+map_key_sort(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
+{
+    SortGenOpArg* t;
+    unsigned size = Size.val;
+    unsigned i;
+
+    if (size == 2) {
+	return 1;		/* Already sorted. */
+    }
+
+
+    t = (SortGenOpArg *) erts_alloc(ERTS_ALC_T_TMP, size*sizeof(SortGenOpArg));
+
+    /*
+     * Copy original data and sort keys to a temporary array.
+     */
+    for (i = 0; i < size; i += 2) {
+	t[i].arg = Rest[i];
+	switch (Rest[i].type) {
+	case TAG_a:
+	    t[i].term = Rest[i].val;
+	    ASSERT(is_atom(t[i].term));
+	    break;
+	case TAG_i:
+	    t[i].term = make_small(Rest[i].val);
+	    break;
+	case TAG_n:
+	    t[i].term = NIL;
+	    break;
+	case TAG_q:
+	    t[i].term = stp->literals[Rest[i].val].term;
+	    break;
+	default:
+	    /*
+	     * Not a literal key. Not allowed. Only a single
+	     * variable key is allowed in each map instruction.
+	     */
+	    erts_free(ERTS_ALC_T_TMP, (void *) t);
+	    return 0;
+	}
+#ifdef DEBUG
+	t[i+1].term = THE_NON_VALUE;
+#endif
+	t[i+1].arg = Rest[i+1];
+    }
+
+    /*
+     * Sort the temporary array.
+     */
+    qsort((void *) t, size / 2, 2 * sizeof(SortGenOpArg),
+	  (int (*)(const void *, const void *)) genopargtermcompare);
+
+    /*
+     * Copy back the sorted, original data.
+     */
+    for (i = 0; i < size; i++) {
+	Rest[i] = t[i].arg;
+    }
+
+    erts_free(ERTS_ALC_T_TMP, (void *) t);
+    return 1;
+}
+
 /*
  * Replace a get_map_elements with one key to an instruction with one
  * element
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index abaa47217a..92d9ccb5eb 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1473,16 +1473,24 @@ apply_last I P
 # Map instructions in R17.
 #
 
-put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \
+sorted_put_map_assoc/5
+put_map_assoc F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
+  sorted_put_map_assoc F Map Dst Live Size Rest
+
+sorted_put_map_exact/5
+put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
+  sorted_put_map_exact F Map Dst Live Size Rest
+
+sorted_put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \
    new_map Dst Live Size Rest
-put_map_assoc F Src=s Dst Live Size Rest=* => \
+sorted_put_map_assoc F Src=s Dst Live Size Rest=* => \
 	       update_map_assoc F Src Dst Live Size Rest
-put_map_assoc F Src Dst Live Size Rest=* => \
+sorted_put_map_assoc F Src Dst Live Size Rest=* => \
 	       move Src x | update_map_assoc F x Dst Live Size Rest
 
-put_map_exact F Src=s Dst Live Size Rest=* => \
+sorted_put_map_exact F Src=s Dst Live Size Rest=* => \
 	      update_map_exact F Src Dst Live Size Rest
-put_map_exact F Src Dst Live Size Rest=* => \
+sorted_put_map_exact F Src Dst Live Size Rest=* => \
 	      move Src x | update_map_exact F x Dst Live Size Rest
 
 new_map d I I
@@ -1506,8 +1514,10 @@ has_map_fields Fail Src Size Rest=* => \
 
 get_map_element/4
 
-get_map_elements Fail Src=rxy Size=u==2 Rest=* => gen_get_map_element(Fail,Src,Size,Rest)
-get_map_elements Fail Src Size Rest=* => i_get_map_elements Fail Src Size Rest
+get_map_elements Fail Src=rxy Size=u==2 Rest=* => \
+    gen_get_map_element(Fail, Src, Size, Rest)
+get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
+   i_get_map_elements Fail Src Size Rest
 
 i_get_map_elements f s I
 
-- 
cgit v1.2.3


From 47e1ed4c0681a73c9d6bc8d24ece85dd77957034 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 24 Mar 2015 15:06:25 +0100
Subject: beam_emu: Slightly optimize update_map_{assoc,exact}

In the update loop for big maps, the E variable is restored for
each turn of the loop. It only needs to be restored if a garbage
collection has been performed.

Also add a new test case that attempts to force several garbage
collections while updating a map, to help us find bugs with
incorrect restoration of the E variable after a garbage collection.
---
 erts/emulator/beam/beam_emu.c | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index f25b9f594d..4e64dce95c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6572,10 +6572,9 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 		reg[live] = res;
 		erts_garbage_collect(p, 0, reg, live+1);
 		res       = reg[live];
+		E = p->stop;
 	    }
 
-	    E = p->stop;
-
 	    new_p += 2;
 	}
 	return res;
@@ -6781,7 +6780,6 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	res = map;
 	E = p->stop;
 	while(n--) {
-	    /* assoc can't fail */
 	    GET_TERM(new_p[0], new_key);
 	    GET_TERM(new_p[1], val);
 	    hx = hashmap_make_hash(new_key);
@@ -6795,10 +6793,9 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 		reg[live] = res;
 		erts_garbage_collect(p, 0, reg, live+1);
 		res       = reg[live];
+		E = p->stop;
 	    }
 
-	    E = p->stop;
-
 	    new_p += 2;
 	}
 	return res;
@@ -6808,7 +6805,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     num_old = flatmap_get_size(old_mp);
 
     /*
-     * If the old map is empty, create a new map.
+     * If the old map is empty, fail.
      */
 
     if (num_old == 0) {
-- 
cgit v1.2.3


From 5b8872c37f63b35e13464109e986ef3727588040 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 25 Mar 2015 09:47:42 +0100
Subject: Optimize use of i_get_map_element/4

In the i_get_map_element/4 instruction, for literal keys other than
atoms, the key would be put into x[0] instead of used directly in the
instruction. The reason is that the original implementation of maps
only supported atom keys.
---
 erts/emulator/beam/ops.tab | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 92d9ccb5eb..23f5b75b7a 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1521,21 +1521,21 @@ get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
 
 i_get_map_elements f s I
 
-get_map_element Fail Src=rxy Key=ax Dst => i_get_map_element Fail Src Key Dst
-get_map_element Fail Src=rxy Key=rycq Dst => \
+get_map_element Fail Src=rxy Key=cx Dst => i_get_map_element Fail Src Key Dst
+get_map_element Fail Src=rxy Key=ry Dst => \
     move Key x | i_get_map_element Fail Src x Dst
 get_map_element Fail Src Key Dst => jump Fail
 
 %macro: i_get_map_element GetMapElement -fail_action
-i_get_map_element f r a r
-i_get_map_element f x a r
-i_get_map_element f y a r
-i_get_map_element f r a x
-i_get_map_element f x a x
-i_get_map_element f y a x
-i_get_map_element f r a y
-i_get_map_element f x a y
-i_get_map_element f y a y
+i_get_map_element f r c r
+i_get_map_element f x c r
+i_get_map_element f y c r
+i_get_map_element f r c x
+i_get_map_element f x c x
+i_get_map_element f y c x
+i_get_map_element f r c y
+i_get_map_element f x c y
+i_get_map_element f y c y
 i_get_map_element f r x r
 i_get_map_element f x x r
 i_get_map_element f y x r
-- 
cgit v1.2.3


From 5f1e301dfec48fccbf865a8b54af5908bebb77c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 25 Mar 2015 11:57:44 +0100
Subject: Teach the loader to pre-compute the hash value for single-key lookups

Let the loader pre-compute the hash value when a single, literal key
is matched as in:

  #{<<"some_key">>:=V} = Map

In my measurements, this optimization resulted in a 30 percent
speedup for short binary keys.

Unfortunately, this optimizization makes no difference for small
maps with less than 32 keys, since the hash value is not used.
Still, there are the following use cases:

* A map used instead of a record with more than 32 entries. I have
seen some applications with huge records.

* Lookup in JSON dictionaries represented as maps.
The hash value will only be used when the map is a hash map
(currently, that means at least 32 entries).
---
 erts/emulator/beam/beam_emu.c  | 46 ++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/beam_load.c | 42 +++++++++++++++++++++++++++++++++-----
 erts/emulator/beam/ops.tab     | 26 +++++++++++-------------
 3 files changed, 95 insertions(+), 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4e64dce95c..5f49032e23 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -710,6 +710,15 @@ void** beam_ops;
      Dst = _res;				\
   } while (0)
 
+#define GetMapElementHash(Src, Key, Hx, Dst, Fail)	\
+  do {							\
+     Eterm _res = get_map_element_hash(Src, Key, Hx);	\
+     if (is_non_value(_res)) {				\
+        Fail;						\
+     }							\
+     Dst = _res;					\
+  } while (0)
+
 #define IsFunction(X, Action)			\
   do {						\
      if ( !(is_any_fun(X)) ) {			\
@@ -959,6 +968,7 @@ static Eterm update_map_assoc(Process* p, Eterm* reg,
 static Eterm update_map_exact(Process* p, Eterm* reg,
 			      Eterm map, BeamInstr* I) NOINLINE;
 static Eterm get_map_element(Eterm map, Eterm key);
+static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx);
 
 /*
  * Functions not directly called by process_main(). OK to inline.
@@ -6441,6 +6451,42 @@ static Eterm get_map_element(Eterm map, Eterm key)
     return vs ? *vs : THE_NON_VALUE;
 }
 
+static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx)
+{
+    const Eterm *vs;
+
+    if (is_flatmap(map)) {
+	flatmap_t *mp;
+	Eterm *ks;
+	Uint i;
+	Uint n;
+
+	mp = (flatmap_t *)flatmap_val(map);
+	ks = flatmap_get_keys(mp);
+	vs = flatmap_get_values(mp);
+	n  = flatmap_get_size(mp);
+	if (is_immed(key)) {
+	    for (i = 0; i < n; i++) {
+		if (ks[i] == key) {
+		    return vs[i];
+		}
+	    }
+	} else {
+	    for (i = 0; i < n; i++) {
+		if (EQ(ks[i], key)) {
+		    return vs[i];
+		}
+	    }
+	}
+	return THE_NON_VALUE;
+    }
+
+    ASSERT(is_hashmap(map));
+    ASSERT(hx == hashmap_make_hash(key));
+    vs = erts_hashmap_get(hx, key, map);
+    return vs ? *vs : THE_NON_VALUE;
+}
+
 #define GET_TERM(term, dest)					\
 do {								\
     Eterm src = (Eterm)(term);					\
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 1a6c6c16ad..ee319d0a78 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4161,9 +4161,30 @@ map_key_sort(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
     return 1;
 }
 
+static int
+hash_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx)
+{
+    switch (Key.type) {
+    case TAG_a:
+	*hx = hashmap_make_hash(Key.val);
+	return 1;
+    case TAG_i:
+	*hx = hashmap_make_hash(make_small(Key.val));
+	return 1;
+    case TAG_n:
+	*hx = hashmap_make_hash(NIL);
+	return 1;
+    case TAG_q:
+	*hx = hashmap_make_hash(stp->literals[Key.val].term);
+	return 1;
+    default:
+	return 0;
+    }
+}
+
 /*
  * Replace a get_map_elements with one key to an instruction with one
- * element
+ * element.
  */
 
 static GenOp*
@@ -4171,18 +4192,29 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
 		    GenOpArg Size, GenOpArg* Rest)
 {
     GenOp* op;
+    GenOpArg Key;
+    Uint32 hx = 0;
 
     ASSERT(Size.type == TAG_u);
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    op->op = genop_get_map_element_4;
-    op->arity = 4;
-
     op->a[0] = Fail;
     op->a[1] = Src;
     op->a[2] = Rest[0];
-    op->a[3] = Rest[1];
+
+    Key = Rest[0];
+    if (hash_genop_arg(stp, Key, &hx)) {
+	op->arity = 5;
+	op->op = genop_i_get_map_element_hash_5;
+	op->a[3].type = TAG_u;
+	op->a[3].val = (BeamInstr) hx;
+	op->a[4] = Rest[1];
+    } else {
+	op->arity = 4;
+	op->op = genop_i_get_map_element_4;
+	op->a[3] = Rest[1];
+    }
     return op;
 }
 
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 23f5b75b7a..456f879ab5 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1512,8 +1512,6 @@ has_map_fields Fail Src Size Rest=* => \
 
 ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
 
-get_map_element/4
-
 get_map_elements Fail Src=rxy Size=u==2 Rest=* => \
     gen_get_map_element(Fail, Src, Size, Rest)
 get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
@@ -1521,21 +1519,21 @@ get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
 
 i_get_map_elements f s I
 
-get_map_element Fail Src=rxy Key=cx Dst => i_get_map_element Fail Src Key Dst
-get_map_element Fail Src=rxy Key=ry Dst => \
+i_get_map_element Fail Src=rxy Key=ry Dst => \
     move Key x | i_get_map_element Fail Src x Dst
-get_map_element Fail Src Key Dst => jump Fail
+
+%macro: i_get_map_element_hash GetMapElementHash -fail_action
+i_get_map_element_hash f r c I r
+i_get_map_element_hash f x c I r
+i_get_map_element_hash f y c I r
+i_get_map_element_hash f r c I x
+i_get_map_element_hash f x c I x
+i_get_map_element_hash f y c I x
+i_get_map_element_hash f r c I y
+i_get_map_element_hash f x c I y
+i_get_map_element_hash f y c I y
 
 %macro: i_get_map_element GetMapElement -fail_action
-i_get_map_element f r c r
-i_get_map_element f x c r
-i_get_map_element f y c r
-i_get_map_element f r c x
-i_get_map_element f x c x
-i_get_map_element f y c x
-i_get_map_element f r c y
-i_get_map_element f x c y
-i_get_map_element f y c y
 i_get_map_element f r x r
 i_get_map_element f x x r
 i_get_map_element f y x r
-- 
cgit v1.2.3


From 9b940b12210500e615ea05b447b0e1be34e70d39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 25 Mar 2015 13:10:30 +0100
Subject: Pre-compute hash values for the general get_map_elements instruction

See the previous commit for justification and use cases.
---
 erts/emulator/beam/beam_emu.c  | 14 +++++++-------
 erts/emulator/beam/beam_load.c | 41 +++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/ops.tab     |  2 +-
 3 files changed, 49 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 5f49032e23..6fde14bc8d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2427,7 +2427,7 @@ do {								\
      * i.e. that it follows a test is_map if needed.
      */
 
-    n  = (Uint)Arg(2) / 2;
+    n  = (Uint)Arg(2) / 3;
     fs = &Arg(3); /* pattern fields and target registers */
 
     if (is_flatmap(map)) {
@@ -2450,12 +2450,11 @@ do {								\
 	    if (EQ((Eterm)*fs,*ks)) {
 		PUT_TERM_REG(*vs, fs[1]);
 		n--;
-		fs += 2;
+		fs += 3;
 		/* no more values to fetch, we are done */
 		if (n == 0) break;
 	    }
-	    ks++; sz--;
-	    vs++;
+	    ks++, sz--, vs++;
 	}
 
 	if (n) {
@@ -2467,13 +2466,14 @@ do {								\
 	Uint32 hx;
 	ASSERT(is_hashmap(map));
 	while(n--) {
-	    hx = hashmap_make_hash((Eterm)*fs);
-	    if ((v = erts_hashmap_get(hx,(Eterm)*fs, map)) == NULL) {
+	    hx = fs[2];
+	    ASSERT(hx == hashmap_make_hash((Eterm)fs[0]));
+	    if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) {
 		SET_I((BeamInstr *) Arg(0));
 		goto get_map_elements_fail;
 	    }
 	    PUT_TERM_REG(*v, fs[1]);
-	    fs += 2;
+	    fs += 3;
 	}
     }
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index ee319d0a78..8d7beb4eb4 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4218,6 +4218,47 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
     return op;
 }
 
+static GenOp*
+gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
+		     GenOpArg Size, GenOpArg* Rest)
+{
+    GenOp* op;
+    Uint32 hx;
+    Uint i;
+    GenOpArg* dst;
+#ifdef DEBUG
+    int good_hash;
+#endif
+
+    ASSERT(Size.type == TAG_u);
+
+    NEW_GENOP(stp, op);
+    op->op = genop_i_get_map_elements_3;
+    GENOP_ARITY(op, 3 + 3*(Size.val/2));
+    op->next = NULL;
+    op->a[0] = Fail;
+    op->a[1] = Src;
+    op->a[2].type = TAG_u;
+    op->a[2].val = 3*(Size.val/2);
+
+    dst = op->a+3;
+    for (i = 0; i < Size.val / 2; i++) {
+	dst[0] = Rest[2*i];
+	dst[1] = Rest[2*i+1];
+#ifdef DEBUG
+	good_hash =
+#endif
+	    hash_genop_arg(stp, dst[0], &hx);
+#ifdef DEBUG
+	ASSERT(good_hash);
+#endif
+	dst[2].type = TAG_u;
+	dst[2].val = (BeamInstr) hx;
+	dst += 3;
+    }
+    return op;
+}
+
 static GenOp*
 gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
 		   GenOpArg Size, GenOpArg* Rest)
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 456f879ab5..9bdc9cb88d 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1515,7 +1515,7 @@ has_map_fields Fail Src Size Rest=* => \
 get_map_elements Fail Src=rxy Size=u==2 Rest=* => \
     gen_get_map_element(Fail, Src, Size, Rest)
 get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
-   i_get_map_elements Fail Src Size Rest
+   gen_get_map_elements(Fail, Src, Size, Rest)
 
 i_get_map_elements f s I
 
-- 
cgit v1.2.3


From 61712dda57647804aec8d4f45da8ae1ccfad732c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 26 Mar 2015 10:59:16 +0100
Subject: Tigthen code for the i_get_map_elements/3 instruction

---
 erts/emulator/beam/beam_emu.c | 26 ++++++++++----------------
 1 file changed, 10 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 6fde14bc8d..7e242640ed 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2439,28 +2439,27 @@ do {								\
 	sz = flatmap_get_size(mp);
 
 	if (sz == 0) {
-	    SET_I((BeamInstr *) Arg(0));
-	    goto get_map_elements_fail;
+	    ClauseFail();
 	}
 
 	ks = flatmap_get_keys(mp);
 	vs = flatmap_get_values(mp);
 
 	while(sz) {
-	    if (EQ((Eterm)*fs,*ks)) {
+	    if (EQ((Eterm) fs[0], *ks)) {
 		PUT_TERM_REG(*vs, fs[1]);
 		n--;
 		fs += 3;
 		/* no more values to fetch, we are done */
-		if (n == 0) break;
+		if (n == 0) {
+		    I = fs;
+		    Next(-1);
+		}
 	    }
 	    ks++, sz--, vs++;
 	}
 
-	if (n) {
-	    SET_I((BeamInstr *) Arg(0));
-	    goto get_map_elements_fail;
-	}
+	ClauseFail();
     } else {
 	const Eterm *v;
 	Uint32 hx;
@@ -2469,19 +2468,14 @@ do {								\
 	    hx = fs[2];
 	    ASSERT(hx == hashmap_make_hash((Eterm)fs[0]));
 	    if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) {
-		SET_I((BeamInstr *) Arg(0));
-		goto get_map_elements_fail;
+		ClauseFail();
 	    }
 	    PUT_TERM_REG(*v, fs[1]);
 	    fs += 3;
 	}
+	I = fs;
+	Next(-1);
     }
-
-
-    I += 4 + Arg(2);
-get_map_elements_fail:
-    ASSERT(VALID_INSTR(*I));
-    Goto(*I);
  }
 #undef PUT_TERM_REG
 
-- 
cgit v1.2.3


From 44c5a955c8409cc98a5fbdec7898de46b9e5dfbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 31 Mar 2015 10:15:33 +0200
Subject: erl_term.h: Add is_not_map() macro

For consistency with other data types, add the is_not_map() macro.
---
 erts/emulator/beam/erl_term.h | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index cff012d5d1..602aab46dc 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1027,6 +1027,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 
 #define is_map_header(x)       (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
 #define is_map(x)              (is_boxed((x)) && is_map_header(*boxed_val(x)))
+#define is_not_map(x)          (!is_map(x))
 #define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE))
 
 /* number tests */
-- 
cgit v1.2.3


From 0a1e2b5e6c92cbea2d2e853facb8b4d6f058d541 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 14 Apr 2015 13:29:08 +0200
Subject: erts: Refactor erts_queue_message

---
 erts/emulator/beam/bif.c           |  6 +----
 erts/emulator/beam/dist.c          | 12 ++-------
 erts/emulator/beam/erl_alloc.c     |  6 +----
 erts/emulator/beam/erl_bif_ddll.c  |  6 +----
 erts/emulator/beam/erl_bif_timer.c |  6 +----
 erts/emulator/beam/erl_gc.c        |  6 +----
 erts/emulator/beam/erl_message.c   | 32 +++++++---------------
 erts/emulator/beam/erl_message.h   | 12 ++++++---
 erts/emulator/beam/erl_nif.c       |  6 +----
 erts/emulator/beam/erl_process.c   | 28 +++-----------------
 erts/emulator/beam/erl_time_sup.c  |  6 +----
 erts/emulator/beam/erl_trace.c     | 54 +++++++-------------------------------
 erts/emulator/beam/io.c            | 40 +++++-----------------------
 erts/emulator/beam/utils.c         |  6 +----
 14 files changed, 48 insertions(+), 178 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 022150da55..4f2958c664 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -610,11 +610,7 @@ erts_queue_monitor_message(Process *p,
     ref_copy    = copy_struct(ref, ref_size, &hp, ohp);
 
     tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy);
-    erts_queue_message(p, p_locksp, bp, tup, NIL
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(p, p_locksp, bp, tup, NIL);
 }
 
 static BIF_RETTYPE
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 32f3cda4f5..142fcb3c00 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -388,11 +388,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	    Eterm tup;
 	    Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks);
 	    tup = TUPLE2(hp, am_nodedown, name);
-	    erts_queue_message(rp, &rp_locks, bp, tup, NIL
-#ifdef USE_VM_PROBES
-			       , NIL
-#endif
-			       );
+	    erts_queue_message(rp, &rp_locks, bp, tup, NIL);
 	}
 	erts_smp_proc_unlock(rp, rp_locks);
     }
@@ -3325,11 +3321,7 @@ send_nodes_mon_msg(Process *rp,
     }
 
     ASSERT(hend == hp);
-    erts_queue_message(rp, rp_locksp, bp, msg, NIL
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(rp, rp_locksp, bp, msg, NIL);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index f2bceff4eb..e396395dde 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3180,11 +3180,7 @@ reply_alloc_info(void *vair)
 	HRelease(rp, hp_end, hp);	    
     }
 
-    erts_queue_message(rp, &rp_locks, bp, msg, NIL
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
 
     if (air->req_sched == sched_id)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index fc4f819f56..7b35edc9c4 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1731,11 +1731,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	hp += REF_THING_SIZE;
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
     }
-    erts_queue_message(proc, &rp_locks, bp, mess, am_undefined
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(proc, &rp_locks, bp, mess, am_undefined);
     erts_smp_proc_unlock(proc, rp_locks);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 }
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 0bd8d20c34..ac4a5644ac 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -373,11 +373,7 @@ bif_timer_timeout(ErtsBifTimer* btm)
 		message = TUPLE3(hp, am_timeout, ref, message);
 	    }
 
-	    erts_queue_message(rp, &rp_locks, bp, message, NIL
-#ifdef USE_VM_PROBES
-			       , NIL
-#endif
-			       );
+	    erts_queue_message(rp, &rp_locks, bp, message, NIL);
 	    erts_smp_proc_unlock(rp, rp_locks);
 	}
     }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 4a116c0740..0b18d2b9e8 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2657,11 +2657,7 @@ reply_gc_info(void *vgcirp)
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, bp, msg, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-		       );
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
 
     if (gcirp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 22cbae10d1..247ea10764 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -369,11 +369,7 @@ erts_queue_dist_message(Process *rcvr,
                         tok_label, tok_lastcnt, tok_serial);
             }
 #endif
-	    erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token
-#ifdef USE_VM_PROBES
-			       , NIL
-#endif
-			       );
+	    erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
     }
     else {
 	/* Enqueue message on external format */
@@ -563,15 +559,15 @@ queue_message(Process *c_p,
 }
 
 void
-erts_queue_message(Process* receiver,
-		   ErtsProcLocks *receiver_locks,
-		   ErlHeapFragment* bp,
-		   Eterm message,
-		   Eterm seq_trace_token
 #ifdef USE_VM_PROBES
-		   , Eterm dt_utag
+erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks,
+                         ErlHeapFragment* bp,
+                         Eterm message, Eterm seq_trace_token, Eterm dt_utag)
+#else
+erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks,
+                   ErlHeapFragment* bp,
+                   Eterm message, Eterm seq_trace_token)
 #endif
-    )
 {
     queue_message(NULL,
 		  receiver,
@@ -1117,11 +1113,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL);
 	temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap);
-	erts_queue_message(to, to_locksp, bp, save, temptoken
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   );
+	erts_queue_message(to, to_locksp, bp, save, temptoken);
     } else {
 	ErlOffHeap *ohp;
 	sz_reason = size_object(reason);
@@ -1138,11 +1130,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 		     ? from
 		     : copy_struct(from, sz_from, &hp, ohp));
 	save = TUPLE3(hp, am_EXIT, from_copy, mess);
-	erts_queue_message(to, to_locksp, bp, save, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   );
+	erts_queue_message(to, to_locksp, bp, save, NIL);
     }
 }
 
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 8713941769..8f9ea939e8 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -258,11 +258,17 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
 					    Eterm *, Uint);
 void free_message_buffer(ErlHeapFragment *);
 void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm);
-void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm
 #ifdef USE_VM_PROBES
-		   , Eterm dt_utag
+void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*,
+                              Eterm message, Eterm seq_trace_token, Eterm dt_utag);
+#define erts_queue_message(RP,RL,BP,Msg,SEQ) \
+    erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL)
+#else
+void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*,
+                        Eterm message, Eterm seq_trace_token);
+#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \
+    erts_queue_message((RP),(RL),(BP),(Msg),(SEQ))
 #endif
-);
 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
 Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 660f446a52..776bbf6719 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -357,11 +357,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     if (flush_me) {	
 	flush_env(env); /* Needed for ERTS_HOLE_CHECK */ 
     }
-    erts_queue_message(rp, &rp_locks, frags, msg, am_undefined
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(rp, &rp_locks, frags, msg, am_undefined);
     if (c_p == rp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
     if (rp_locks)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f74a2ee54c..6518314e30 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1023,11 +1023,7 @@ reply_sched_wall_time(void *vswtrp)
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, bp, msg, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-		       );
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
 
     if (swtrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -9649,15 +9645,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 	ASSERT(hp_start + hsz == hp);
 #endif
 
-	erts_queue_message(rp,
-			   &rp_locks,
-			   bp,
-			   msg,
-			   NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-	    );
+	erts_queue_message(rp, &rp_locks, bp, msg, NIL);
 
 	if (c_p == rp)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -11366,11 +11354,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 
 	hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp);
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
-	erts_queue_message(to, to_locksp, bp, mess, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   );
+	erts_queue_message(to, to_locksp, bp, mess, NIL);
     } else {
 	ErlHeapFragment* bp;
 	Eterm* hp;
@@ -11386,11 +11370,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL);
 	temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap);
-	erts_queue_message(to, to_locksp, bp, mess, temp_token
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   );
+	erts_queue_message(to, to_locksp, bp, mess, temp_token);
     }
 }
 
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index bbdedcc128..9a7466ff48 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1773,11 +1773,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 		    *patch_refp = ref;
 		    ASSERT(hsz == size_object(message_template));
 		    message = copy_struct(message_template, hsz, &hp, ohp);
-		    erts_queue_message(rp, &rp_locks, bp, message, NIL
-#ifdef USE_VM_PROBES
-				       , NIL
-#endif
-			);
+		    erts_queue_message(rp, &rp_locks, bp, message, NIL);
 		}
 		erts_smp_proc_unlock(rp, rp_locks);
 	    }
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 2f9969b0e7..aaecc5a02e 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -130,14 +130,9 @@ do { \
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \
 } while(0)
 #else
-#ifdef USE_VM_PROBES
-#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \
-    erts_queue_message((TPROC), NULL, (BP), (MSG), NIL, NIL)
-#else
 #define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \
     erts_queue_message((TPROC), NULL, (BP), (MSG), NIL)
 #endif
-#endif
 
 /*
  * NOTE that the ERTS_GET_TRACER_REF() returns from the function  (!!!)
@@ -636,11 +631,7 @@ profile_send(Eterm from, Eterm message) {
 	hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0);
 	msg = copy_struct(message, sz, &hp, &bp->off_heap);
 	
-    	erts_queue_message(profile_p, NULL, bp, msg, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   );
+        erts_queue_message(profile_p, NULL, bp, msg, NIL);
     }
 }
 
@@ -1240,11 +1231,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
 #else
-	erts_queue_message(tracer, NULL, bp, mess, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   ); /* trace_token must be NIL here */
+        /* trace_token must be NIL here */
+	erts_queue_message(tracer, NULL, bp, mess, NIL);
 #endif
     }
 }
@@ -2343,11 +2331,7 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-		       );
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
 #endif
 }
 void 
@@ -2408,11 +2392,7 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time)
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-		       );
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
 #endif
 }
 
@@ -2483,11 +2463,7 @@ monitor_long_gc(Process *p, Uint time) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-		       );
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
 #endif
 }
 
@@ -2558,11 +2534,7 @@ monitor_large_heap(Process *p) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
 #endif
 }
 
@@ -2590,11 +2562,7 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
 #endif
 
 }
@@ -3389,11 +3357,7 @@ sys_msg_dispatcher_func(void *unused)
 		}
 		else {
 		queue_proc_msg:
-		    erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL
-#ifdef USE_VM_PROBES
-				       , NIL
-#endif
-				       );
+		    erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL);
 #ifdef DEBUG_PRINTOUTS
 		    erts_fprintf(stderr, "delivered\n");
 #endif
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 1db3a9fba7..8cb185cb2b 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1430,15 +1430,7 @@ queue_port_sched_op_reply(Process *rp,
 	    bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1);
     }
 
-    erts_queue_message(rp,
-		       rp_locksp,
-		       bp,
-		       msg,
-		       NIL
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-	);
+    erts_queue_message(rp, rp_locksp, bp, msg, NIL);
 }
 
 static void
@@ -3086,11 +3078,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
 	hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks);
 	res = copy_struct(res, sz_res, &hp, ohp);
 	tuple = TUPLE2(hp, sender, res);
-	erts_queue_message(rp, &rp_locks, bp, tuple, NIL
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   );
+	erts_queue_message(rp, &rp_locks, bp, tuple, NIL);
 
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
@@ -3186,11 +3174,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-		       );
+    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
@@ -3357,11 +3341,7 @@ deliver_vec_message(Port* prt,			/* Port */
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
     erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
 	erts_smp_proc_dec_refc(rp);
@@ -5061,11 +5041,7 @@ void driver_report_exit(ErlDrvPort ix, int status)
    hp += 3;
    tuple = TUPLE2(hp, prt->common.id, tuple);
 
-   erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined
-#ifdef USE_VM_PROBES
-		      , NIL
-#endif
-		      );
+   erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
 
    erts_smp_proc_unlock(rp, rp_locks);
    if (!scheduler)
@@ -5665,11 +5641,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    HRelease(rp, hp_end, hp);	    
 	}
 	/* send message */
-	erts_queue_message(rp, &rp_locks, bp, mess, am_undefined
-#ifdef USE_VM_PROBES
-			   , NIL
-#endif
-			   );
+	erts_queue_message(rp, &rp_locks, bp, mess, am_undefined);
     }
     else {
 	if (b2t.ix > b2t.used)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index aecfe04a75..f253aa5cd0 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2305,11 +2305,7 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
 	erts_queue_error_logger_message(from, tuple3, bp);
     }
 #else
-    erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL
-#ifdef USE_VM_PROBES
-		       , NIL
-#endif
-		       );
+    erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL);
 #endif
     return 0;
 }
-- 
cgit v1.2.3


From c8a0eca92cdda27c3efdde261c9c32bd445a3794 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 14 Apr 2015 15:24:18 +0200
Subject: erts: Refactor dtrace call probes

---
 erts/emulator/beam/beam_emu.c | 82 ++++++++++++++++---------------------------
 1 file changed, 31 insertions(+), 51 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index fdb84aae42..4e01d94b92 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1077,16 +1077,32 @@ init_emulator(void)
         DTRACE2(nif_return, process_name, mfa);                 \
     }
 
-#else /* USE_VM_PROBES */
-
-#define DTRACE_LOCAL_CALL(p, m, f, a)  do {} while (0)
-#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0)
-#define DTRACE_RETURN(p, m, f, a)      do {} while (0)
-#define DTRACE_BIF_ENTRY(p, m, f, a)   do {} while (0)
-#define DTRACE_BIF_RETURN(p, m, f, a)  do {} while (0)
-#define DTRACE_NIF_ENTRY(p, m, f, a)   do {} while (0)
-#define DTRACE_NIF_RETURN(p, m, f, a)  do {} while (0)
+#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e)                                                    \
+    do {                                                                                       \
+        if (DTRACE_ENABLED(global_function_entry)) {                                           \
+            BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \
+            DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);                     \
+        }                                                                                      \
+    } while(0)
+
+#define DTRACE_RETURN_FROM_PC(p)                                                        \
+    do {                                                                                \
+        BeamInstr* fp;                                                                  \
+        if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \
+            DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]);                \
+        }                                                                               \
+    } while(0)
 
+#else /* USE_VM_PROBES */
+#define DTRACE_LOCAL_CALL(p, m, f, a)        do {} while (0)
+#define DTRACE_GLOBAL_CALL(p, m, f, a)       do {} while (0)
+#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0)
+#define DTRACE_RETURN(p, m, f, a)            do {} while (0)
+#define DTRACE_RETURN_FROM_PC(p)             do {} while (0)
+#define DTRACE_BIF_ENTRY(p, m, f, a)         do {} while (0)
+#define DTRACE_BIF_RETURN(p, m, f, a)        do {} while (0)
+#define DTRACE_NIF_ENTRY(p, m, f, a)         do {} while (0)
+#define DTRACE_NIF_RETURN(p, m, f, a)        do {} while (0)
 #endif /* USE_VM_PROBES */
 
 /*
@@ -1523,12 +1539,7 @@ void process_main(void)
      * is not loaded, it points to code which will invoke the error handler
      * (see lb_call_error_handler below).
      */
-#ifdef USE_VM_CALL_PROBES
-    if (DTRACE_ENABLED(global_function_entry)) {
-	BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]);
-	DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
-    }
-#endif
+    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
     Dispatchx();
 
  OpCase(i_move_call_ext_cre): {
@@ -1538,12 +1549,7 @@ void process_main(void)
  /* FALL THROUGH */
  OpCase(i_call_ext_e):
     SET_CP(c_p, I+2);
-#ifdef USE_VM_CALL_PROBES
-    if (DTRACE_ENABLED(global_function_entry)) {
-	BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]);
-	DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
-    }
-#endif
+    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
     Dispatchx();
 
  OpCase(i_move_call_ext_only_ecr): {
@@ -1551,12 +1557,7 @@ void process_main(void)
  }
  /* FALL THROUGH */
  OpCase(i_call_ext_only_e):
-#ifdef USE_VM_CALL_PROBES
-    if (DTRACE_ENABLED(global_function_entry)) {
-	BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]);
-	DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
-    }
-#endif
+    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
     Dispatchx();
 
  OpCase(init_y): {
@@ -1590,18 +1591,9 @@ void process_main(void)
 	Next(1);
     }
 
-
  OpCase(return): {
-#ifdef USE_VM_CALL_PROBES
-    BeamInstr* fptr;
-#endif
     SET_I(c_p->cp);
-
-#ifdef USE_VM_CALL_PROBES
-    if (DTRACE_ENABLED(function_return) && (fptr = find_function_from_pc(c_p->cp))) {
-        DTRACE_RETURN(c_p, (Eterm)fptr[0], (Eterm)fptr[1], (Uint)fptr[2]);
-    }
-#endif
+    DTRACE_RETURN_FROM_PC(c_p);
     /*
      * We must clear the CP to make sure that a stale value do not
      * create a false module dependcy preventing code upgrading.
@@ -6087,13 +6079,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
 	save_calls(p, ep);
     }
-
-#ifdef USE_VM_CALL_PROBES
-    if (DTRACE_ENABLED(global_function_entry)) {
-        BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()];
-	DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]);
-    }
-#endif
+    DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
     return ep->addressv[erts_active_code_ix()];
 }
 
@@ -6142,13 +6128,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
 	save_calls(p, ep);
     }
-
-#ifdef USE_VM_CALL_PROBES
-    if (DTRACE_ENABLED(global_function_entry)) {
-        BeamInstr *fptr = (BeamInstr *)  ep->addressv[erts_active_code_ix()];
-	DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]);
-    }
-#endif
+    DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
     return ep->addressv[erts_active_code_ix()];
 }
 
-- 
cgit v1.2.3


From 647c66952afa9036c060a7685071bb27e01c2151 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 14 Apr 2015 18:07:05 +0200
Subject: erts: Use make_small for size terms on flat maps

---
 erts/emulator/beam/erl_bif_guard.c | 27 +++++++++++++--------------
 erts/emulator/beam/erl_map.c       |  8 +-------
 2 files changed, 14 insertions(+), 21 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index e7d84ebda1..a5a0c06ad6 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -459,25 +459,24 @@ Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live)
 Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg = reg[live];
-    Eterm* hp;
-    Uint size;
     if (is_flatmap(arg)) {
 	flatmap_t *mp = (flatmap_t*)flatmap_val(arg);
-	size = flatmap_get_size(mp);
+        return make_small(flatmap_get_size(mp));
     } else if (is_hashmap(arg)) {
+        Eterm* hp;
+        Uint size;
 	size = hashmap_size(arg);
-    } else {
-	BIF_ERROR(p, BADARG);
-    }
-    if (IS_USMALL(0, size)) {
-	return make_small(size);
+        if (IS_USMALL(0, size)) {
+            return make_small(size);
+        }
+        if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
+            erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
+        }
+        hp = p->htop;
+        p->htop += BIG_UINT_HEAP_SIZE;
+        return uint_to_big(size, hp);
     }
-    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
-	erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
-    }
-    hp = p->htop;
-    p->htop += BIG_UINT_HEAP_SIZE;
-    return uint_to_big(size, hp);
+    BIF_ERROR(p, BADARG);
 }
 
 Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live)
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 98023bbb47..bbb7d5e5c6 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -102,14 +102,8 @@ static int hxnodecmpkey(hxnode_t* a, hxnode_t* b);
 
 BIF_RETTYPE map_size_1(BIF_ALIST_1) {
     if (is_flatmap(BIF_ARG_1)) {
-	Eterm *hp;
-	Uint hsz  = 0;
 	flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
-	Uint n    = flatmap_get_size(mp);
-
-	erts_bld_uint(NULL, &hsz, n);
-	hp = HAlloc(BIF_P, hsz);
-	BIF_RET(erts_bld_uint(&hp, NULL, n));
+	BIF_RET(make_small(flatmap_get_size(mp)));
     } else if (is_hashmap(BIF_ARG_1)) {
 	Eterm *head, *hp, res;
 	Uint size, hsz=0;
-- 
cgit v1.2.3


From 5bb8262bd4ae8dbcc7438e80de5cedac7ccee1af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 31 Mar 2015 08:47:57 +0200
Subject: Raise more descriptive error messages for failed map operations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

According to EEP-43 for maps, a 'badmap' exception should be
generated when an attempt is made to update non-map term such as:

   <<>>#{a=>42}

That was not implemented in the OTP 17.

José Valim suggested that we should take the opportunity to
improve the errors coming from map operations:

  http://erlang.org/pipermail/erlang-questions/2015-February/083588.html

This commit implement better errors from map operations similar
to his suggestion.

When a map update operation (Map#{...}) or a BIF that expects a map
is given a non-map term, the exception will be:

  {badmap,Term}

This kind of exception is similar to the {badfun,Term} exception
from operations that expect a fun.

When a map operation requires a key that is not present in a map,
the following exception will be raised:

  {badkey,Key}

José Valim suggested that the exception should be
{badkey,Key,Map}. We decided not to do that because the map
could potentially be huge and cause problems if the error
propagated through links to other processes.

For BIFs, it could be argued that the exceptions could be simply
'badmap' and 'badkey', because the bad map and bad key can be found in
the argument list for the BIF in the stack backtrace. However, for the
map update operation (Map#{...}), the bad map or bad key will not be
included in the stack backtrace, so that information must be included
in the exception reason itself. For consistency, the BIFs should raise
the same exceptions as update operation.

If more than one key is missing, it is undefined which of
keys that will be reported in the {badkey,Key} exception.
---
 erts/emulator/beam/atom.names      |  2 +-
 erts/emulator/beam/beam_emu.c      | 31 +++++++++++++---
 erts/emulator/beam/erl_bif_guard.c |  3 +-
 erts/emulator/beam/erl_map.c       | 75 +++++++++++++++++++++++++-------------
 erts/emulator/beam/error.h         | 10 ++++-
 5 files changed, 87 insertions(+), 34 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index ae3f30d82f..8fdcbb4058 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -104,7 +104,7 @@ atom await_sched_wall_time_modifications
 atom awaiting_load
 atom awaiting_unload
 atom backtrace backtrace_depth
-atom badarg badarith badarity badfile badmatch badsig badfun
+atom badarg badarith badarity badfile badfun badkey badmap badmatch badsig
 atom bag
 atom band
 atom big
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 7e242640ed..6a5128e7f8 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2493,7 +2493,13 @@ do {								\
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
      } else {
-	 goto badarg;
+	 /*
+	  * This can only happen if the code was compiled
+	  * with the compiler in OTP 17.
+	  */
+	 c_p->freason = BADMAP;
+	 c_p->fvalue = map;
+	 goto lb_Cl_error;
      }
  }
 
@@ -2511,7 +2517,7 @@ do {								\
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
      } else {
-	 goto badarg;
+	 goto lb_Cl_error;
      }
  }
 
@@ -5261,7 +5267,9 @@ Eterm error_atom[NUMBER_EXIT_CODES] = {
   am_notalive,		/* 14 */
   am_system_limit,	/* 15 */
   am_try_clause,	/* 16 */
-  am_notsup		/* 17 */
+  am_notsup,		/* 17 */
+  am_badmap,		/* 18 */
+  am_badkey,		/* 19 */
 };
 
 /*
@@ -5517,6 +5525,8 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
     case (GET_EXC_INDEX(EXC_TRY_CLAUSE)):
     case (GET_EXC_INDEX(EXC_BADFUN)):
     case (GET_EXC_INDEX(EXC_BADARITY)):
+    case (GET_EXC_INDEX(EXC_BADMAP)):
+    case (GET_EXC_INDEX(EXC_BADKEY)):
         /* Some common exceptions: value -> {atom, value} */
         ASSERT(is_value(Value));
 	hp = HAlloc(c_p, 3);
@@ -6814,8 +6824,11 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	/* apparently the compiler does not emit is_map instructions,
 	 * bad compiler */
 
-	if (is_not_hashmap(map))
+	if (is_not_hashmap(map)) {
+	    p->freason = BADMAP;
+	    p->fvalue = map;
 	    return THE_NON_VALUE;
+	}
 
 	res = map;
 	E = p->stop;
@@ -6825,8 +6838,11 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	    hx = hashmap_make_hash(new_key);
 
 	    res = erts_hashmap_insert(p, hx, new_key, val, res,  1);
-	    if (is_non_value(res))
+	    if (is_non_value(res)) {
+		p->fvalue = new_key;
+		p->freason = BADKEY;
 		return res;
+	    }
 
 	    if (p->mbuf) {
 		Uint live = Arg(3);
@@ -6849,6 +6865,9 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      */
 
     if (num_old == 0) {
+	E = p->stop;
+	p->freason = BADKEY;
+	GET_TERM(new_p[0], p->fvalue);
 	return THE_NON_VALUE;
     }
 
@@ -6918,6 +6937,8 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      * update list did not previously exist.
      */
     ASSERT(hp == p->htop + need);
+    p->freason = BADKEY;
+    p->fvalue = new_key;
     return THE_NON_VALUE;
 }
 #undef GET_TERM
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index e7d84ebda1..069327ee9d 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -467,7 +467,8 @@ Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live)
     } else if (is_hashmap(arg)) {
 	size = hashmap_size(arg);
     } else {
-	BIF_ERROR(p, BADARG);
+	p->fvalue = arg;
+	BIF_ERROR(p, BADMAP);
     }
     if (IS_USMALL(0, size)) {
 	return make_small(size);
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 98023bbb47..3aebbfdaa3 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -122,7 +122,8 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) {
 	BIF_RET(res);
     }
 
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /* maps:to_list/1 */
@@ -150,7 +151,8 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
 	return hashmap_to_list(BIF_P, BIF_ARG_1);
     }
 
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /* maps:find/2
@@ -217,34 +219,29 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
 	}
 	BIF_RET(am_error);
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_2;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /* maps:get/2
  * return value if key *matches* a key in the map
- * exception bad_key if none matches
+ * exception badkey if none matches
  */
 
 BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
-	Eterm *hp;
-        Eterm error;
         const Eterm *value;
-	char *s_error;
 
         value = erts_maps_get(BIF_ARG_1, BIF_ARG_2);
         if (value) {
             BIF_RET(*value);
 	}
 
-	s_error = "bad_key";
-	error = am_atom_put(s_error, sys_strlen(s_error));
-
-	hp = HAlloc(BIF_P, 3);
-	BIF_P->fvalue = TUPLE2(hp, error, BIF_ARG_1);
-	BIF_ERROR(BIF_P, EXC_ERROR_2);
+	BIF_P->fvalue = BIF_ARG_1;
+	BIF_ERROR(BIF_P, BADKEY);
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_2;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /* maps:from_list/1
@@ -911,7 +908,8 @@ BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
 	BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_2;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /* maps:keys/1 */
@@ -939,7 +937,8 @@ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
     } else if (is_hashmap(BIF_ARG_1)) {
 	BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1));
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 /* maps:merge/2 */
 
@@ -951,6 +950,7 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
 	    /* Will always become a tree */
 	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
 	}
+	BIF_P->fvalue = BIF_ARG_2;
     } else if (is_hashmap(BIF_ARG_1)) {
 	if (is_hashmap(BIF_ARG_2)) {
 	    BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
@@ -958,8 +958,11 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
 	    /* Will always become a tree */
 	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1));
 	}
+	BIF_P->fvalue = BIF_ARG_2;
+    } else {
+	BIF_P->fvalue = BIF_ARG_1;
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
@@ -1398,7 +1401,8 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
     if (is_map(BIF_ARG_3)) {
 	BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3));
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_3;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /* maps:remove/3 */
@@ -1492,7 +1496,8 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
 	    BIF_RET(res);
 	}
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_2;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
@@ -1688,13 +1693,17 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 /* maps:update/3 */
 
 BIF_RETTYPE maps_update_3(BIF_ALIST_3) {
-    if (is_map(BIF_ARG_3)) {
+    if (is_not_map(BIF_ARG_3)) {
+	BIF_P->fvalue = BIF_ARG_3;
+	BIF_ERROR(BIF_P, BADMAP);
+    } else {
 	Eterm res;
 	if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) {
 	    BIF_RET(res);
 	}
+	BIF_P->fvalue = BIF_ARG_1;
+	BIF_ERROR(BIF_P, BADKEY);
     }
-    BIF_ERROR(BIF_P, BADARG);
 }
 
 
@@ -1723,7 +1732,8 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
     } else if (is_hashmap(BIF_ARG_1)) {
 	BIF_RET(hashmap_values(BIF_P, BIF_ARG_1));
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 static Eterm hashmap_to_list(Process *p, Eterm node) {
@@ -2546,8 +2556,12 @@ Uint hashmap_over_estimated_heap_size(Uint k)
 BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) {
     if (is_hashmap(BIF_ARG_1)) {
 	BIF_RET(hashmap_info(BIF_P,BIF_ARG_1));
+    } else if (is_flatmap(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P, BADARG);
+    } else {
+	BIF_P->fvalue = BIF_ARG_1;
+	BIF_ERROR(BIF_P, BADMAP);
     }
-    BIF_ERROR(BIF_P, BADARG);
 }
 
 /*
@@ -2560,8 +2574,12 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
     if (is_flatmap(BIF_ARG_1)) {
 	flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
 	BIF_RET(mp->keys);
+    } else if (is_hashmap(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P, BADARG);
+    } else {
+	BIF_P->fvalue = BIF_ARG_1;
+	BIF_ERROR(BIF_P, BADMAP);
     }
-    BIF_ERROR(BIF_P, BADARG);
 }
 
 /*
@@ -2589,7 +2607,8 @@ BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
                 erl_exit(1, "bad header");
         }
     }
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /*
@@ -2629,8 +2648,12 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) {
         hp = HAlloc(BIF_P, 2*sz);
         while(sz--) { res = CONS(hp, *ptr++, res); hp += 2; }
         BIF_RET(res);
+    } else if (is_flatmap(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P, BADARG);
+    } else {
+	BIF_P->fvalue = BIF_ARG_1;
+	BIF_ERROR(BIF_P, BADMAP);
     }
-    BIF_ERROR(BIF_P, BADARG);
 }
 
 
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index ddc2c1396d..e63967adb6 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -140,7 +140,13 @@
 #define EXC_NOTSUP		((17 << 8) | EXC_ERROR)
 					/* Not supported */
 
-#define NUMBER_EXIT_CODES 18	/* The number of exit code indices */
+#define EXC_BADMAP		((18 << 8) | EXC_ERROR)
+					/* Bad map */
+
+#define EXC_BADKEY		((19 << 8) | EXC_ERROR)
+					/* Bad key in map */
+
+#define NUMBER_EXIT_CODES 20	/* The number of exit code indices */
 
 /*
  * Internal pseudo-error codes.
@@ -152,6 +158,8 @@
  */
 #define BADARG EXC_BADARG
 #define BADARITH EXC_BADARITH
+#define BADKEY EXC_BADKEY
+#define BADMAP EXC_BADMAP
 #define BADMATCH EXC_BADMATCH
 #define SYSTEM_LIMIT EXC_SYSTEM_LIMIT
 
-- 
cgit v1.2.3


From 88b79b875731199a789bbda43d104ea9c8a1f235 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 15 Apr 2015 15:43:51 +0200
Subject: erts: Optimize comparison operator for frequent immediates

* small integers
* atoms
---
 erts/emulator/beam/utils.c | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index aecfe04a75..4db2015eea 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2922,15 +2922,44 @@ Sint cmp(Eterm a, Eterm b)
 }
 #endif
 
+#if HALFWORD_HEAP
+static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base,
+                                     Eterm b, Eterm* b_base,
+                                     int exact, int eq_only);
+#else
+static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
+#endif
+
+#if HALFWORD_HEAP
+Sint erts_cmp(Eterm a, Eterm* a_base,
+              Eterm b, Eterm* b_base,
+              int exact, int eq_only)
+#else
+Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
+#endif
+{
+    if (is_atom(a) && is_atom(b)) {
+        return cmp_atoms(a, b);
+    } else if (is_both_small(a, b)) {
+        return (signed_val(a) - signed_val(b));
+    }
+#if HALFWORD_HEAP
+    return erts_cmp_compound(a,a_base,b,b_base,exact,eq_only);
+#else
+    return erts_cmp_compound(a,b,exact,eq_only);
+#endif
+}
+
+
 /* erts_cmp(Eterm a, Eterm b, int exact)
  * exact = 1 -> term-based compare
  * exact = 0 -> arith-based compare
  */
 #if HALFWORD_HEAP
-Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base,
+static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base,
                       int exact, int eq_only)
 #else
-Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
+static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
 #endif
 {
 #define PSTACK_TYPE struct erts_cmp_hashmap_state
-- 
cgit v1.2.3


From 8846021fe1b1bcf79ec3a01037a8a656e161766c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Apr 2015 16:55:55 +0200
Subject: erts: Fix isfinite for windows

Add macro erts_isfinite as an OS independent way to check
if a float is finite.
---
 erts/emulator/beam/erl_nif.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ec82ef251e..8b6cb880a3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -967,7 +967,7 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)
 ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 {
     Eterm* hp;
-    if (!isfinite(d))
+    if (!erts_isfinite(d))
         return enif_make_badarg(env);
     hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
     FloatDef f;
-- 
cgit v1.2.3


From 80683a598eda7bea94eb54f9f9557e020b2fdb2c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Apr 2015 13:33:45 +0200
Subject: erts: Fix compile warning in enif_make_double

---
 erts/emulator/beam/erl_nif.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8b6cb880a3..6fa8164787 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -967,10 +967,11 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)
 ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 {
     Eterm* hp;
+    FloatDef f;
+
     if (!erts_isfinite(d))
         return enif_make_badarg(env);
     hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
-    FloatDef f;
     f.fd = d;
     PUT_DOUBLE(f, hp);
     return make_float(hp);
-- 
cgit v1.2.3


From 80e15112a6e31e053ad0670096c23bda2fc341e4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Apr 2015 15:18:33 +0200
Subject: erts: Add enif_has_pending_exception

---
 erts/emulator/beam/erl_nif.c           | 5 +++++
 erts/emulator/beam/erl_nif.h           | 3 ++-
 erts/emulator/beam/erl_nif_api_funcs.h | 2 ++
 3 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 6fa8164787..0a05bdcf8b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -747,6 +747,11 @@ Eterm enif_make_badarg(ErlNifEnv* env)
     BIF_ERROR(env->proc, BADARG);
 }
 
+int enif_has_pending_exception(ErlNifEnv* env)
+{
+    return env->exception_thrown;
+}
+
 int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len,
 		  ErlNifCharEncoding encoding)
 {
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 849024453c..006547e1d0 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -46,9 +46,10 @@
 **           remove enif_schedule_dirty_nif, enif_schedule_dirty_nif_finalizer, enif_dirty_nif_finalizer
 **           add ErlNifEntry options
 **           add ErlNifFunc flags
+** 2.8: 18.0 add enif_has_pending_exception
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 7
+#define ERL_NIF_MINOR_VERSION 8
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 630cefae93..bdcbb32c46 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -156,6 +156,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIte
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter));
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
+ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -305,6 +306,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_map_iterator_prev ERL_NIF_API_FUNC_MACRO(enif_map_iterator_prev)
 #  define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair)
 #  define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif)
+#  define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From dd9ad8da73e15e89c8ab27efdd47a8bda8019957 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Apr 2015 17:30:00 +0200
Subject: erts: Reject non-finite float terms in erl_drv_output_term

---
 erts/emulator/beam/io.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 012a7d1a4b..07dc0db3b9 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5581,7 +5581,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	    mess = make_float(hp);
 	    f.fd = *((double *) ptr[0]);
-	    PUT_DOUBLE(f, hp);
+            if (!erts_isfinite(f.fd))
+                ERTS_DDT_FAIL;
+            PUT_DOUBLE(f, hp);
 	    hp += FLOAT_SIZE_OBJECT;
 	    ptr++;
 	    break;
-- 
cgit v1.2.3


From 69e0f3a87346a6238b65a00a08b1902821612b36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 16 Apr 2015 17:25:59 +0200
Subject: erts: Remove instruction_count command option

* We use compile directive icount instead
---
 erts/emulator/beam/erl_init.c | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 86d3416423..a4f1e0d7ee 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1955,11 +1955,6 @@ erl_start(int argc, char **argv)
 		goto time_correction_false;
 	    else if (sys_strcmp(argv[i]+2, "true") == 0)
 		goto time_correction_true;
-#ifdef ERTS_OPCODE_COUNTER_SUPPORT
-	    else if (argv[i][2] == 'i') { /* -ci: undcoumented option*/
-		count_instructions = 1;
-	    }
-#endif
 	    else if (argv[i][2] == '\0') {
 		if (i + 1 >= argc)
 		    goto time_correction_false;
-- 
cgit v1.2.3


From 544d962f3401e96d1bd9efd1209ab53ade9568b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 16 Apr 2015 17:34:00 +0200
Subject: erts: Assume counting opcodes are correctly generated

* Assertion is only removed because we are in icount mode.
---
 erts/emulator/beam/beam_emu.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 6a3415d9f4..bb7b799950 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5137,8 +5137,6 @@ do {								\
 
 #ifndef NO_JUMP_TABLE
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
-     /* Are tables correctly generated by beam_makeops? */
-     ERTS_CT_ASSERT(sizeof(counting_opcodes) == sizeof(opcodes));
 #ifdef DEBUG
      counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
 #endif
-- 
cgit v1.2.3


From b3e121949ad524be44878d656e923dd187780eda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 20 Apr 2015 10:01:06 +0200
Subject: erts: Fix halfword compare

---
 erts/emulator/beam/utils.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ae62a7a708..edefc5170f 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2920,16 +2920,16 @@ Sint cmp(Eterm a, Eterm b)
 
 #if HALFWORD_HEAP
 static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base,
-                                     Eterm b, Eterm* b_base,
-                                     int exact, int eq_only);
+                                      Eterm b, Eterm* b_base,
+                                      int exact, int eq_only);
 #else
 static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
 #endif
 
 #if HALFWORD_HEAP
-Sint erts_cmp(Eterm a, Eterm* a_base,
-              Eterm b, Eterm* b_base,
-              int exact, int eq_only)
+Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base,
+                      Eterm b, Eterm* b_base,
+                      int exact, int eq_only)
 #else
 Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
 #endif
@@ -2940,7 +2940,7 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
         return (signed_val(a) - signed_val(b));
     }
 #if HALFWORD_HEAP
-    return erts_cmp_compound(a,a_base,b,b_base,exact,eq_only);
+    return erts_cmp_compound_rel_opt(a,a_base,b,b_base,exact,eq_only);
 #else
     return erts_cmp_compound(a,b,exact,eq_only);
 #endif
@@ -2952,8 +2952,9 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
  * exact = 0 -> arith-based compare
  */
 #if HALFWORD_HEAP
-static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base,
-                      int exact, int eq_only)
+static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base,
+                                      Eterm b, Eterm* b_base,
+                                      int exact, int eq_only)
 #else
 static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
 #endif
-- 
cgit v1.2.3


From f38aa5a70ff1244e0e3a88c4980b1e87274c0741 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 20 Apr 2015 18:09:04 +0200
Subject: erts: Brute force float comparisons as well

Increases float comparison speed by ~120%
---
 erts/emulator/beam/utils.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index edefc5170f..8f6335d5dd 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2938,6 +2938,11 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
         return cmp_atoms(a, b);
     } else if (is_both_small(a, b)) {
         return (signed_val(a) - signed_val(b));
+    } else if (is_float_rel(a, a_base) && is_float_rel(b, b_base)) {
+        FloatDef af, bf;
+        GET_DOUBLE_REL(a, af, a_base);
+        GET_DOUBLE_REL(b, bf, b_base);
+        return float_comp(af.fd, bf.fd);
     }
 #if HALFWORD_HEAP
     return erts_cmp_compound_rel_opt(a,a_base,b,b_base,exact,eq_only);
-- 
cgit v1.2.3


From 177d20a7329d24aec0f4a0cbbbd3fc803bcbba4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 17 Apr 2015 11:04:09 +0200
Subject: erts: Specialize compare instructions

* i_is_lt for r, x registers and constants
* i_is_ge for x registers and constants
* i_is_exact_eq for r and x registers
---
 erts/emulator/beam/beam_emu.c |  3 +++
 erts/emulator/beam/ops.tab    | 49 +++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 50 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index bb7b799950..aad76e9a93 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -662,6 +662,9 @@ void** beam_ops;
 
 #define EqualImmed(X, Y, Action) if (X != Y) { Action; }
 #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
+#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; }
+#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; }
+#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; }
 
 #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
 
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 9bdc9cb88d..9330192d04 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -391,6 +391,51 @@ i_is_ne_exact_literal r f c
 i_is_ne_exact_literal x f c
 i_is_ne_exact_literal y f c
 
+#
+# Common Compare Specializations
+# We don't do all of them since we want
+# to keep the instruction set small-ish
+#
+
+is_eq_exact Lbl S1=xy S2=r => is_eq_exact Lbl S2 S1
+is_eq_exact Lbl S1=rx S2=xy => i_is_eq_exact_spec Lbl S1 S2
+%macro: i_is_eq_exact_spec EqualExact -fail_action
+
+i_is_eq_exact_spec f x x
+i_is_eq_exact_spec f x y
+i_is_eq_exact_spec f r x
+i_is_eq_exact_spec f r y
+%cold
+i_is_eq_exact_spec f r r
+%hot
+
+is_lt Lbl S1=rxc S2=rxc  => i_is_lt_spec Lbl S1 S2
+
+%macro: i_is_lt_spec IsLessThan -fail_action
+
+i_is_lt_spec f x x
+i_is_lt_spec f x r
+i_is_lt_spec f x c
+i_is_lt_spec f r x
+i_is_lt_spec f r c
+i_is_lt_spec f c x
+i_is_lt_spec f c r
+%cold
+i_is_lt_spec f r r
+i_is_lt_spec f c c
+%hot
+
+is_ge Lbl S1=xc S2=xc => i_is_ge_spec Lbl S1 S2
+
+%macro: i_is_ge_spec IsGreaterEqual -fail_action
+
+i_is_ge_spec f x x
+i_is_ge_spec f x c
+i_is_ge_spec f c x
+%cold
+i_is_ge_spec f c c
+%hot
+
 #
 # All other comparisons.
 #
@@ -398,8 +443,8 @@ i_is_ne_exact_literal y f c
 is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
 is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
 
-is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl
 is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl
+is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl
 is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl
 is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl
 
@@ -493,7 +538,6 @@ put_list s s d
 %hot
 
 %macro: i_fetch FetchArgs -pack
-i_fetch c c
 i_fetch c r
 i_fetch c x
 i_fetch c y
@@ -510,6 +554,7 @@ i_fetch y x
 i_fetch y y
 
 %cold
+i_fetch c c
 i_fetch s s
 %hot
 
-- 
cgit v1.2.3


From 63949dcd13e24bdc73336b0f5e88d4f06cce56c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 15 Sep 2014 17:58:33 +0200
Subject: erts: Add instruction move3 for xy and xx

---
 erts/emulator/beam/beam_emu.c | 3 ++-
 erts/emulator/beam/ops.tab    | 8 ++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index aad76e9a93..8b9e271068 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -562,7 +562,8 @@ void** beam_ops;
        Store(term, Dst);           \
    } while (0)
 
-#define Move2(src1, dst1, src2, dst2) dst1 = (src1); dst2 = (src2)
+#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2)
+#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)
 
 #define MoveGenDest(src, dstp) \
    if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; }
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 9330192d04..66339f8ace 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -302,6 +302,9 @@ move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
 move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
 
+move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3
+move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3
+
 move C=aiq X=x==1 => move_x1 C
 move C=aiq X=x==2 => move_x2 C
 
@@ -313,6 +316,11 @@ move2 x y x y
 move2 y x y x
 move2 x x x x
 
+%macro: move3 Move3
+move3 x y x y x y
+move3 y x y x y x
+move3 x x x x x x
+
 # The compiler almost never generates a "move Literal y(Y)" instruction,
 # so let's cheat if we encounter one.
 move S=n D=y => init D
-- 
cgit v1.2.3


From d1321eaf9bd1e417561e3d70fd85749fe589143b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 22 Apr 2015 20:44:43 +0200
Subject: erts: Add move window instruction

Move an entire region of x registers to the stack.
This reduces the dispatch pressure of move instructions.

Also introduce a move2 specialization for some common move patterns:

move r y | move x y -> move2 : As above, moving regions to the stack
move x r | move x y -> move2 : A seemingly common pattern
---
 erts/emulator/beam/beam_emu.c | 34 ++++++++++++++++++++++++++++++++++
 erts/emulator/beam/ops.tab    | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8b9e271068..b9b2094e6c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1503,6 +1503,40 @@ void process_main(void)
 	Next(2);
     }
 
+ OpCase(move_window3_xxxy): {
+     BeamInstr *next;
+     Eterm *y;
+     PreFetch(4, next);
+     y = (Eterm *)(((unsigned char *)E) + (Arg(3)));
+     y[0] = xb(Arg(0));
+     y[1] = xb(Arg(1));
+     y[2] = xb(Arg(2));
+     NextPF(4, next);
+ }
+ OpCase(move_window4_xxxxy): {
+     BeamInstr *next;
+     Eterm *y;
+     PreFetch(5, next);
+     y = (Eterm *)(((unsigned char *)E) + (Arg(4)));
+     y[0] = xb(Arg(0));
+     y[1] = xb(Arg(1));
+     y[2] = xb(Arg(2));
+     y[3] = xb(Arg(3));
+     NextPF(5, next);
+ }
+ OpCase(move_window5_xxxxxy): {
+     BeamInstr *next;
+     Eterm *y;
+     PreFetch(6, next);
+     y = (Eterm *)(((unsigned char *)E) + (Arg(5)));
+     y[0] = xb(Arg(0));
+     y[1] = xb(Arg(1));
+     y[2] = xb(Arg(2));
+     y[3] = xb(Arg(3));
+     y[4] = xb(Arg(4));
+     NextPF(6, next);
+ }
+
  OpCase(i_move_call_only_fcr): {
      r(0) = Arg(1);
  }
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 66339f8ace..3f7e69e07e 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -298,12 +298,44 @@ move_jump f c
 move_jump f x
 move_jump f y
 
+
+# Movement to and from the stack is common
+# Try to pack as much as we can into one instruction
+
+# Window move
+move_window/5
+move_window/6
+
+# x -> y
+
+move S1=r S2=y | move X1=x Y1=y => move2 S1 S2 X1 Y1
+
+move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \
+    move_window X1 X2 X3 Y1 Y3
+
+move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \
+    move_window X1 X2 X3 X4 Y1 Y4
+
+move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \
+    move_window5 X1 X2 X3 X4 X5 Y1
+
+move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1
+move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1
+
+move_window3 x x x y
+move_window4 x x x x y
+move_window5 x x x x x y
+
 move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
 move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
 
+move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
+move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
+
 move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3
 move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3
+move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6
 
 move C=aiq X=x==1 => move_x1 C
 move C=aiq X=x==2 => move_x2 C
@@ -316,6 +348,11 @@ move2 x y x y
 move2 y x y x
 move2 x x x x
 
+move2 x r x y
+move2 r y x y
+move2 y r x y
+
+
 %macro: move3 Move3
 move3 x y x y x y
 move3 y x y x y x
-- 
cgit v1.2.3


From 06a912c068f62e1e04ebb2aa2002d611b8cc31e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 23 Apr 2015 15:34:07 +0200
Subject: erts: Fix loader increment from minus instruction

A type error caused the optimization to never kick in.
---
 erts/emulator/beam/beam_load.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 8d7beb4eb4..282aa71109 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3172,7 +3172,11 @@ gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
 static int
 negation_is_small(LoaderState* stp, GenOpArg Int)
 {
-    return Int.type == TAG_i && IS_SSMALL(-Int.val);
+    /* Check for the rare case of overflow in BeamInstr (UWord) -> Sint
+     * Cast to the correct type before using IS_SSMALL (Sint) */
+    return Int.type == TAG_i &&
+           !(Int.val & ~((((BeamInstr)1) << ((sizeof(Sint)*8)-1))-1)) &&
+           IS_SSMALL(-((Sint)Int.val));
 }
 
 
-- 
cgit v1.2.3


From 38384c6c5f66cf52d24ae6f48cc71bbe52611aa9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 23 Apr 2015 17:24:18 +0200
Subject: erts: Batch loads and stores for move_window

May lessen load/store latency.
---
 erts/emulator/beam/beam_emu.c | 48 +++++++++++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b9b2094e6c..beec8967fc 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1505,35 +1505,47 @@ void process_main(void)
 
  OpCase(move_window3_xxxy): {
      BeamInstr *next;
-     Eterm *y;
+     Eterm xt0, xt1, xt2;
+     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3)));
      PreFetch(4, next);
-     y = (Eterm *)(((unsigned char *)E) + (Arg(3)));
-     y[0] = xb(Arg(0));
-     y[1] = xb(Arg(1));
-     y[2] = xb(Arg(2));
+     xt0  = xb(Arg(0));
+     xt1  = xb(Arg(1));
+     xt2  = xb(Arg(2));
+     y[0] = xt0;
+     y[1] = xt1;
+     y[2] = xt2;
      NextPF(4, next);
  }
  OpCase(move_window4_xxxxy): {
      BeamInstr *next;
-     Eterm *y;
+     Eterm xt0, xt1, xt2, xt3;
+     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4)));
      PreFetch(5, next);
-     y = (Eterm *)(((unsigned char *)E) + (Arg(4)));
-     y[0] = xb(Arg(0));
-     y[1] = xb(Arg(1));
-     y[2] = xb(Arg(2));
-     y[3] = xb(Arg(3));
+     xt0  = xb(Arg(0));
+     xt1  = xb(Arg(1));
+     xt2  = xb(Arg(2));
+     xt3  = xb(Arg(3));
+     y[0] = xt0;
+     y[1] = xt1;
+     y[2] = xt2;
+     y[3] = xt3;
      NextPF(5, next);
  }
  OpCase(move_window5_xxxxxy): {
      BeamInstr *next;
-     Eterm *y;
+     Eterm xt0, xt1, xt2, xt3, xt4;
+     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5)));
      PreFetch(6, next);
-     y = (Eterm *)(((unsigned char *)E) + (Arg(5)));
-     y[0] = xb(Arg(0));
-     y[1] = xb(Arg(1));
-     y[2] = xb(Arg(2));
-     y[3] = xb(Arg(3));
-     y[4] = xb(Arg(4));
+     xt0  = xb(Arg(0));
+     xt1  = xb(Arg(1));
+     xt2  = xb(Arg(2));
+     xt3  = xb(Arg(3));
+     xt4  = xb(Arg(4));
+     y[0] = xt0;
+     y[1] = xt1;
+     y[2] = xt2;
+     y[3] = xt3;
+     y[4] = xt4;
      NextPF(6, next);
  }
 
-- 
cgit v1.2.3


From 699ad918545ac7e716b77ea05a6a943434616020 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 23 Apr 2015 19:06:03 +0200
Subject: erts: Specialize band instruction for common case

* i_band specialization on x registers and constants
---
 erts/emulator/beam/beam_emu.c | 33 +++++++++++++++++++++++++++++++++
 erts/emulator/beam/ops.tab    |  2 ++
 2 files changed, 35 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index beec8967fc..aca340db6b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2885,6 +2885,23 @@ do {								\
      goto do_big_arith2;
  }
 
+#define DO_BIG_ARITH(Func,Arg1,Arg2)     \
+    do {                                 \
+        Uint live = Arg(1);              \
+        SWAPOUT;                         \
+        reg[0] = r(0);                   \
+        reg[live] = (Arg1);              \
+        reg[live+1] = (Arg2);            \
+        result = (Func)(c_p, reg, live); \
+        r(0) = reg[0];                   \
+        SWAPIN;                          \
+        ERTS_HOLE_CHECK(c_p);            \
+        if (is_value(result)) {          \
+            StoreBifResult(4,result);    \
+        }                                \
+        goto lb_Cl_error;                \
+    } while(0)
+
  OpCase(i_rem_jId):
  {
      Eterm result;
@@ -2900,6 +2917,22 @@ do {								\
      }
  }
 
+ OpCase(i_band_jIxcd):
+ {
+     Eterm result;
+
+     if (is_both_small(xb(Arg(2)), Arg(3))) {
+         /*
+          * No need to untag -- TAG & TAG == TAG.
+          */
+         result = xb(Arg(2)) & Arg(3);
+         StoreBifResult(4, result);
+     }
+     DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3));
+ }
+
+#undef DO_BIG_ARITH
+
  OpCase(i_band_jId):
  {
      Eterm result;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 3f7e69e07e..3436b664d9 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1663,6 +1663,7 @@ gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Ds
 gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst
 gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst
 
+gc_bif2 Fail I u$bif:erlang:band/2 S1=x S2=c Dst=d => i_band Fail I S1 S2 Dst
 gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst
 gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst
 gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst
@@ -1686,6 +1687,7 @@ i_rem j I d
 i_bsl j I d
 i_bsr j I d
 
+i_band j I x c d
 i_band j I d
 i_bor j I d
 i_bxor j I d
-- 
cgit v1.2.3


From b58b4718012cfb848dad3106b2ab22c08997c639 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 23 Apr 2015 19:21:34 +0200
Subject: erts: Specialize rem instruction for common case

* i_rem specialization on x registers
---
 erts/emulator/beam/beam_emu.c | 13 +++++++++++++
 erts/emulator/beam/ops.tab    |  2 ++
 2 files changed, 15 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index aca340db6b..f369d7b632 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2902,6 +2902,19 @@ do {								\
         goto lb_Cl_error;                \
     } while(0)
 
+ OpCase(i_rem_jIxxd):
+ {
+     Eterm result;
+
+     if (xb(Arg(3)) == SMALL_ZERO) {
+	 goto badarith;
+     } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
+	 result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3))));
+         StoreBifResult(4, result);
+     }
+     DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3)));
+ }
+
  OpCase(i_rem_jId):
  {
      Eterm result;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 3436b664d9..7f7b0baacc 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1658,6 +1658,7 @@ gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail
 gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst
 
 gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst
+gc_bif2 Fail I u$bif:erlang:rem/2 S1=x S2=x Dst=d => i_rem Fail I S1 S2 Dst
 gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst
 
 gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst
@@ -1682,6 +1683,7 @@ i_minus j I d
 i_times j I d
 i_m_div j I d
 i_int_div j I d
+i_rem j I x x d
 i_rem j I d
 
 i_bsl j I d
-- 
cgit v1.2.3


From dd4d7667f5ab60c76567a6f3d814b3b64583280a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 24 Apr 2015 16:43:04 +0200
Subject: erts: Add move2 specialization for common move patterns

Common pattern seen in SSL:
move y x | move r x -> move2
move r x | move y x -> move2

Common pattern seen in SSL and Compiler:
move x r | move x x -> move2
---
 erts/emulator/beam/ops.tab | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 7f7b0baacc..22c7f14f0c 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -330,9 +330,13 @@ move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
 move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
 
+move S1=x S2=r | move S3=x S4=x => move2 S1 S2 S3 S4
 move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
 move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
 
+move Y1=y X1=x | move S1=r D1=x => move2 Y1 X1 S1 D1
+move S1=r D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1
+
 move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3
 move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3
 move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6
@@ -348,10 +352,14 @@ move2 x y x y
 move2 y x y x
 move2 x x x x
 
+move2 x r x x
+
 move2 x r x y
 move2 r y x y
 move2 y r x y
 
+move2 r x y x
+move2 y x r x
 
 %macro: move3 Move3
 move3 x y x y x y
-- 
cgit v1.2.3


From 979d05fc326d0f6cf7c1c4a5182bb33942852dd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 24 Apr 2015 17:42:24 +0200
Subject: erts: Specialize minus and plus instruction

Seen on SSL application where substraction with x registers were prevalent:
* i_minus specialization on x registers
* i_plus specialization on x registers
---
 erts/emulator/beam/beam_emu.c | 71 ++++++++++++++++++++++++++++++-------------
 erts/emulator/beam/ops.tab    |  4 +++
 2 files changed, 54 insertions(+), 21 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index f369d7b632..7dd7250e2f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1393,7 +1393,39 @@ void process_main(void)
 	    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
 	    goto find_func_info;
 	}
-	    
+
+#define DO_BIG_ARITH(Func,Arg1,Arg2)     \
+    do {                                 \
+        Uint live = Arg(1);              \
+        SWAPOUT;                         \
+        reg[0] = r(0);                   \
+        reg[live] = (Arg1);              \
+        reg[live+1] = (Arg2);            \
+        result = (Func)(c_p, reg, live); \
+        r(0) = reg[0];                   \
+        SWAPIN;                          \
+        ERTS_HOLE_CHECK(c_p);            \
+        if (is_value(result)) {          \
+            StoreBifResult(4,result);    \
+        }                                \
+        goto lb_Cl_error;                \
+    } while(0)
+
+ OpCase(i_plus_jIxxd):
+ {
+     Eterm result;
+
+     if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
+	 Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3)));
+	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+	 if (MY_IS_SSMALL(i)) {
+	     result = make_small(i);
+             StoreBifResult(4, result);
+	 }
+     }
+     DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3)));
+ }
+
  OpCase(i_plus_jId):
  {
      Eterm result;
@@ -1405,12 +1437,26 @@ void process_main(void)
 	     result = make_small(i);
 	     STORE_ARITH_RESULT(result);
 	 }
-     
      }
      arith_func = ARITH_FUNC(mixed_plus);
      goto do_big_arith2;
  }
 
+ OpCase(i_minus_jIxxd):
+ {
+     Eterm result;
+
+     if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
+	 Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3)));
+	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+	 if (MY_IS_SSMALL(i)) {
+	     result = make_small(i);
+             StoreBifResult(4, result);
+	 }
+     }
+     DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3)));
+ }
+
  OpCase(i_minus_jId):
  {
      Eterm result;
@@ -2885,23 +2931,6 @@ do {								\
      goto do_big_arith2;
  }
 
-#define DO_BIG_ARITH(Func,Arg1,Arg2)     \
-    do {                                 \
-        Uint live = Arg(1);              \
-        SWAPOUT;                         \
-        reg[0] = r(0);                   \
-        reg[live] = (Arg1);              \
-        reg[live+1] = (Arg2);            \
-        result = (Func)(c_p, reg, live); \
-        r(0) = reg[0];                   \
-        SWAPIN;                          \
-        ERTS_HOLE_CHECK(c_p);            \
-        if (is_value(result)) {          \
-            StoreBifResult(4,result);    \
-        }                                \
-        goto lb_Cl_error;                \
-    } while(0)
-
  OpCase(i_rem_jIxxd):
  {
      Eterm result;
@@ -2944,8 +2973,6 @@ do {								\
      DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3));
  }
 
-#undef DO_BIG_ARITH
-
  OpCase(i_band_jId):
  {
      Eterm result;
@@ -2961,6 +2988,8 @@ do {								\
      goto do_big_arith2;
  }
 
+#undef DO_BIG_ARITH
+
  do_big_arith2:
  {
      Eterm result;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 22c7f14f0c..ece038131e 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1660,7 +1660,9 @@ gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \
 # GCing arithmetic instructions.
 #
 
+gc_bif2 Fail I u$bif:erlang:splus/2 S1=x S2=x Dst=d => i_plus Fail I S1 S2 Dst
 gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst
+gc_bif2 Fail I u$bif:erlang:sminus/2 S1=x S2=x Dst=d => i_minus Fail I S1 S2 Dst
 gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst
 gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst
 gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst
@@ -1686,7 +1688,9 @@ i_increment r I I d
 i_increment x I I d
 i_increment y I I d
 
+i_plus j I x x d
 i_plus j I d
+i_minus j I x x d
 i_minus j I d
 i_times j I d
 i_m_div j I d
-- 
cgit v1.2.3


From 2a3349420d33a298aa02b176100f385c0ab31c99 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 4 May 2015 15:05:41 +0200
Subject: erts: Add debug assertions for match state sanity

---
 erts/emulator/beam/erl_bits.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 53c21c40e1..f96cb02587 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -165,6 +165,26 @@ erts_bs_start_match_2(Process *p, Eterm Binary, Uint Max)
     return make_matchstate(ms);
 }
 
+#ifdef DEBUG
+# define CHECK_MATCH_BUFFER(MB) check_match_buffer(MB)
+
+static void check_match_buffer(ErlBinMatchBuffer* mb)
+{
+    Eterm realbin;
+    Uint byteoffs;
+    byte* bytes, bitoffs, bitsz;
+    ProcBin* pb;
+    ERTS_GET_REAL_BIN(mb->orig, realbin, byteoffs, bitoffs, bitsz);
+    bytes = binary_bytes(realbin) + byteoffs;
+    ERTS_ASSERT(mb->base >= bytes && mb->base <= (bytes + binary_size(mb->orig)));
+    pb = (ProcBin *) boxed_val(realbin);
+    if (pb->thing_word == HEADER_PROC_BIN)
+        ERTS_ASSERT(pb->flags == 0);
+}
+#else
+# define CHECK_MATCH_BUFFER(MB)
+#endif
+
 Eterm
 erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb)
 {
@@ -185,6 +205,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
 	return SMALL_ZERO;
     }
     
+    CHECK_MATCH_BUFFER(mb);
     if (mb->size - mb->offset < num_bits) {	/* Asked for too many bits.  */
 	return THE_NON_VALUE;
     }
@@ -425,6 +446,7 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe
 {
     ErlSubBin* sb;
 
+    CHECK_MATCH_BUFFER(mb);
     if (mb->size - mb->offset < num_bits) {	/* Asked for too many bits.  */
 	return THE_NON_VALUE;
     }
@@ -456,6 +478,7 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer
     byte* fptr;
     FloatDef f;
 
+    CHECK_MATCH_BUFFER(mb);
     if (num_bits == 0) {
 	f.fd = 0.0;
 	hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT);
@@ -509,6 +532,8 @@ erts_bs_get_binary_all_2(Process *p, ErlBinMatchBuffer* mb)
 {
     ErlSubBin* sb;
     Uint size;
+
+    CHECK_MATCH_BUFFER(mb);
     size =  mb->size-mb->offset;
     sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE);
     sb->thing_word = HEADER_SUB_BIN;
@@ -1605,6 +1630,7 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb)
     byte* LSB;
     byte* MSB;
 	
+    CHECK_MATCH_BUFFER(mb);
     ASSERT((mb->offset & 7) != 0);
     ASSERT(mb->size - mb->offset >= 32);
 
@@ -1664,6 +1690,8 @@ erts_bs_get_utf8(ErlBinMatchBuffer* mb)
 	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,9,9,9,9,9,9,9,9
     };
 
+    CHECK_MATCH_BUFFER(mb);
+
     if ((remaining_bits = mb->size - mb->offset) < 8) {
 	return THE_NON_VALUE;
     }
@@ -1748,6 +1776,7 @@ erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags)
 	return THE_NON_VALUE;
     }
 
+    CHECK_MATCH_BUFFER(mb);
     /*
      * Set up the pointer to the source bytes.
      */
-- 
cgit v1.2.3


From 6bef0d08142e961d562a7fce4d8b036a32b2b98b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 4 May 2015 19:12:42 +0200
Subject: erts: Use a lockable allocator on 'sys_write_buf'

sys_write_buf allocator type is used from async-threads and needs
to be lockable. In the SMP case the temporary allocator is lockable but
not in the Non-SMP case.

To remedy this the binary-allocator is used for the Non-SMP case,
which is lockable.
---
 erts/emulator/beam/erl_alloc.types | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index e2f8da38b9..074f864dee 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -419,7 +419,12 @@ type	ENVIRONMENT	TEMPORARY	SYSTEM		environment
 type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
 type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
 type	SYS_BLOCKING	STANDARD	SYSTEM		sys_blocking
+
++if smp
 type	SYS_WRITE_BUF	TEMPORARY	SYSTEM		sys_write_buf
++else
+type	SYS_WRITE_BUF	BINARY		SYSTEM		sys_write_buf
++endif
 
 +endif
 
-- 
cgit v1.2.3


From 889dbf81def0986e4569841d64a7f3882808ee07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 6 May 2015 15:17:18 +0200
Subject: erts: Don't let the compiler optimize pos. zero fix

---
 erts/emulator/beam/erl_arith.c |  5 +++++
 erts/emulator/beam/global.h    |  3 +++
 erts/emulator/beam/utils.c     | 12 ++++++++----
 3 files changed, 16 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index 5150a8a507..47d516534f 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -2048,3 +2048,8 @@ Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live)
     }
     return result;
 } 
+
+/* Needed to remove compiler optimization */
+double erts_get_positive_zero_float() {
+    return 0.0f;
+}
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 40b043d1cc..54fba9274f 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -867,6 +867,9 @@ void print_process_info(int, void *, Process*);
 void info(int, void *);
 void loaded(int, void *);
 
+/* erl_arith.c */
+double erts_get_positive_zero_float(void);
+
 /* config.c */
 
 __decl_noreturn void __noreturn erl_exit(int n, char*, ...);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 8f6335d5dd..51de3528be 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -958,7 +958,8 @@ tail_recur:
             FloatDef ff;
             GET_DOUBLE(term, ff);
             if (ff.fd == 0.0f) {
-                ff.fd = 0.0f; /* ensure pos. 0.0 */
+                /* ensure positive 0.0 */
+                ff.fd = erts_get_positive_zero_float();
             }
             hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
             break;
@@ -1477,7 +1478,8 @@ make_hash2(Eterm term)
 		FloatDef ff;
 		GET_DOUBLE(term, ff);
                 if (ff.fd == 0.0f) {
-                    ff.fd = 0.0f; /* ensure pos. 0.0 */
+                    /* ensure positive 0.0 */
+                    ff.fd = erts_get_positive_zero_float();
                 }
 #if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
 		UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
@@ -1899,7 +1901,8 @@ make_internal_hash(Eterm term)
 		FloatDef ff;
 		GET_DOUBLE(term, ff);
                 if (ff.fd == 0.0f) {
-                    ff.fd = 0.0f; /* ensure pos. 0.0 */
+                    /* ensure positive 0.0 */
+                    ff.fd = erts_get_positive_zero_float();
                 }
 		UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
 		goto pop_next;
@@ -2087,7 +2090,8 @@ tail_recur:
             FloatDef ff;
             GET_DOUBLE(term, ff);
             if (ff.fd == 0.0f) {
-                ff.fd = 0.0f; /* ensure pos. 0.0 */
+                /* ensure positive 0.0 */
+                ff.fd = erts_get_positive_zero_float();
             }
             hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
 	}
-- 
cgit v1.2.3


From edce22eb43c40359ccbd0924412cf13692744779 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 26 Mar 2015 22:46:29 +0100
Subject: Misc time improvements

- Possibility to chose different clock sources
- Improved mach clock usage
- Improved linux clock_gettime() usage
- ...
---
 erts/emulator/beam/erl_gc.c       |  21 +-
 erts/emulator/beam/erl_time.h     |   9 +-
 erts/emulator/beam/erl_time_sup.c | 507 +++++++++++++++++++++++++-------------
 erts/emulator/beam/sys.h          |  10 +-
 erts/emulator/beam/time.c         |  66 +++--
 5 files changed, 404 insertions(+), 209 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 4a116c0740..307f9c93e0 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -404,7 +404,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     Uint reclaimed_now = 0;
     int done = 0;
     int off_heap_msgs;
-    Uint ms1, s1, us1;
+    ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
     erts_aint32_t state;
     ErtsSchedulerData *esdp;
 #ifdef USE_VM_PROBES
@@ -424,9 +424,8 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 
     state = erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     off_heap_msgs = state & ERTS_PSFLG_OFF_HEAP_MSGS;
-    if (erts_system_monitor_long_gc != 0) {
-	get_now(&ms1, &s1, &us1);
-    }
+    if (erts_system_monitor_long_gc != 0)
+	start_time = erts_get_monotonic_time();
 
     ERTS_CHK_OFFHEAP(p);
 
@@ -474,16 +473,14 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     }
 
     if (erts_system_monitor_long_gc != 0) {
-	Uint ms2, s2, us2;
-	Sint t;
+	ErtsMonotonicTime end_time;
+	Uint gc_time;
 	if (erts_test_long_gc_sleep)
 	    while (0 != erts_milli_sleep(erts_test_long_gc_sleep));
-	get_now(&ms2, &s2, &us2);
-	t = ms2 - ms1;
-	t = t*1000000 + s2 - s1;
-	t = t*1000 + ((Sint) (us2 - us1))/1000;
-	if (t > 0 && (Uint)t > erts_system_monitor_long_gc) {
-	    monitor_long_gc(p, t);
+	end_time = erts_get_monotonic_time();
+	gc_time = (Uint) ERTS_MONOTONIC_TO_MSEC(end_time - start_time);
+	if (gc_time && gc_time > erts_system_monitor_long_gc) {
+	    monitor_long_gc(p, gc_time);
 	}
     }
     if (erts_system_monitor_large_heap != 0) {
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index cb7764addc..5af3c21d40 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -26,6 +26,12 @@
 #define ERTS_TIME_ASSERT(B) ((void) 1)
 #endif
 
+typedef enum {
+    ERTS_NO_TIME_WARP_MODE,
+    ERTS_SINGLE_TIME_WARP_MODE,
+    ERTS_MULTI_TIME_WARP_MODE
+} ErtsTimeWarpMode;
+
 typedef struct ErtsTimerWheel_ ErtsTimerWheel;
 typedef erts_atomic64_t * ErtsNextTimeoutRef;
 extern ErtsTimerWheel *erts_default_timer_wheel;
@@ -38,7 +44,6 @@ extern SysTimeval erts_first_emu_time;
 typedef struct erl_timer {
     struct erl_timer* next;	/* next entry tiw slot or chain */
     struct erl_timer* prev;	/* prev entry tiw slot or chain */
-    Uint slot;			/* slot in timer wheel */
     erts_smp_atomic_t wheel;
     ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
     /* called when timeout */
@@ -46,6 +51,7 @@ typedef struct erl_timer {
     /* called when cancel (may be NULL) */
     void (*cancel)(void*);
     void* arg;        /* argument to timeout/cancel procs */
+    int slot;			/* slot in timer wheel */
 } ErlTimer;
 
 typedef void (*ErlTimeoutProc)(void*);
@@ -77,6 +83,7 @@ void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer);
 void erts_monitor_time_offset(Eterm id, Eterm ref);
 int erts_demonitor_time_offset(Eterm ref);
 
+int erts_init_time_sup(int, ErtsTimeWarpMode);
 void erts_late_init_time_sup(void);
 
 /* timer-wheel api */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index bbdedcc128..9d572c0b0a 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -133,6 +133,7 @@ struct time_sup_read_only__ {
     ErtsTimeWarpMode warp_mode;
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
     ErtsMonotonicTime moffset;
+    int os_corrected_monotonic_time;
     int os_monotonic_time_disable;
     char *os_monotonic_time_func;
     char *os_monotonic_time_clock_id;
@@ -159,12 +160,16 @@ struct time_sup_read_only__ {
 	ErtsMonotonicTime large_diff;
 	ErtsMonotonicTime small_diff;
     } adj;
+    struct {
+	ErtsMonotonicTime error;
+	ErtsMonotonicTime resolution;
+	int intervals;
+	int use_avg;
+    } drift_adj;
 };
 
 typedef struct {
-#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
     ErtsMonotonicTime drift; /* Correction for os monotonic drift */
-#endif
     ErtsMonotonicTime error; /* Correction for error between system times */
 } ErtsMonotonicCorrection;
 
@@ -174,7 +179,7 @@ typedef struct {
     ErtsMonotonicCorrection correction;
 } ErtsMonotonicCorrectionInstance;
 
-#define ERTS_DRIFT_INTERVALS 5
+#define ERTS_MAX_DRIFT_INTERVALS 50
 typedef struct {
     struct {
 	struct {
@@ -185,7 +190,7 @@ typedef struct {
 	    ErtsMonotonicTime sys;
 	    ErtsMonotonicTime mon;
 	} time;
-    } intervals[ERTS_DRIFT_INTERVALS];
+    } intervals[ERTS_MAX_DRIFT_INTERVALS];
     struct {
 	ErtsMonotonicTime sys;
 	ErtsMonotonicTime mon;
@@ -197,9 +202,7 @@ typedef struct {
 typedef struct {
     ErtsMonotonicCorrectionInstance prev;
     ErtsMonotonicCorrectionInstance curr;    
-#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
     ErtsMonotonicDriftData drift;
-#endif
     ErtsMonotonicTime last_check;
     int short_check_interval;
 } ErtsMonotonicCorrectionData;
@@ -213,10 +216,11 @@ struct time_sup_infrequently_changed__ {
     } parmon;
     ErtsMonotonicTime minit;
 #endif
-    int finalized_offset;
     ErtsSystemTime sinit;
     ErtsMonotonicTime not_corrected_moffset;
-    erts_atomic64_t offset;
+    erts_smp_atomic64_t offset;
+    ErtsMonotonicTime shadow_offset;
+    erts_smp_atomic32_t preliminary_offset;
 };
 
 struct time_sup_frequently_changed__ {
@@ -254,19 +258,19 @@ erts_get_approx_time(void)
 static ERTS_INLINE void
 init_time_offset(ErtsMonotonicTime offset)
 {
-    erts_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset);
+    erts_smp_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset);
 }
 
 static ERTS_INLINE void
 set_time_offset(ErtsMonotonicTime offset)
 {
-    erts_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset);
+    erts_smp_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset);
 }
 
 static ERTS_INLINE ErtsMonotonicTime
 get_time_offset(void)
 {
-    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&time_sup.inf.c.offset);
+    return (ErtsMonotonicTime) erts_smp_atomic64_read_acqb(&time_sup.inf.c.offset);
 }
 
 
@@ -290,16 +294,38 @@ get_time_offset(void)
 #define ERTS_TIME_DRIFT_MAX_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(50)
 #define ERTS_TIME_DRIFT_MIN_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(5)
 
+/*
+ * Maximum drift of the OS monotonic clock expected.
+ *
+ * We use 1 milli second per second. If the monotonic
+ * clock drifts more than this we will fail to adjust for
+ * drift, and error correction will kick in instead.
+ * If it is larger than this, one could argue that the
+ * primitive is to poor to be used...
+ */
+#define ERTS_MAX_MONOTONIC_DRIFT ERTS_MSEC_TO_MONOTONIC(1)
+
+/*
+ * We assume that precision is 32 times worse than the
+ * resolution. This is a wild guess, but there are no
+ * practical way to determine actual precision.
+ */
+#define ERTS_ASSUMED_PRECISION_DROP 32
+
+#define ERTS_MIN_MONOTONIC_DRIFT_MEASUREMENT \
+    (ERTS_SHORT_TIME_CORRECTION_CHECK - 2*ERTS_MAX_MONOTONIC_DRIFT)
+
+
 static ERTS_INLINE ErtsMonotonicTime
 calc_corrected_erl_mtime(ErtsMonotonicTime os_mtime,
 			 ErtsMonotonicCorrectionInstance *cip,
-			 ErtsMonotonicTime *os_mdiff_p)
+			 ErtsMonotonicTime *os_mdiff_p,
+			 int os_drift_corrected)
 {
     ErtsMonotonicTime erl_mtime, diff = os_mtime - cip->os_mtime;
     ERTS_TIME_ASSERT(diff >= 0);
-#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
-    diff += (cip->correction.drift*diff)/ERTS_MONOTONIC_TIME_UNIT;
-#endif
+    if (!os_drift_corrected)
+	diff += (cip->correction.drift*diff)/ERTS_MONOTONIC_TIME_UNIT;
     erl_mtime = cip->erl_mtime;
     erl_mtime += diff;
     erl_mtime += cip->correction.error*(diff/ERTS_TCORR_ERR_UNIT);
@@ -308,7 +334,8 @@ calc_corrected_erl_mtime(ErtsMonotonicTime os_mtime,
     return erl_mtime;
 }
 
-static ErtsMonotonicTime get_corrected_time(void)
+static ERTS_INLINE ErtsMonotonicTime
+read_corrected_time(int os_drift_corrected)
 {
     ErtsMonotonicTime os_mtime;
     ErtsMonotonicCorrectionData cdata;
@@ -331,7 +358,18 @@ static ErtsMonotonicTime get_corrected_time(void)
 	cip = &cdata.prev;
     }
 
-    return calc_corrected_erl_mtime(os_mtime, cip, NULL);
+    return calc_corrected_erl_mtime(os_mtime, cip, NULL,
+				    os_drift_corrected);
+}
+
+static ErtsMonotonicTime get_os_drift_corrected_time(void)
+{
+    return read_corrected_time(!0);
+}
+
+static ErtsMonotonicTime get_corrected_time(void)
+{
+    return read_corrected_time(0);
 }
 
 #ifdef ERTS_TIME_CORRECTION_PRINT
@@ -352,66 +390,42 @@ print_correction(int change,
 	usec_sdiff = ERTS_MONOTONIC_TO_USEC(sdiff);
 
     if (!change)
-	fprintf(stderr,
-		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppb] : "
-		"tmo = %lld msec\r\n",
-		(long long) usec_sdiff,
-		(long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
-		(long long) (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
-		(long long) tmo);
+	erts_fprintf(stderr,
+		     "sdiff = %b64d usec : [ec=%b64d ppm, dc=%b64d ppb] : "
+		     "tmo = %bpu msec\r\n",
+		     usec_sdiff,
+		     (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
+		     (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		     tmo);
     else
-	fprintf(stderr,
-		"sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppb] "
-		"-> [ec=%lld ppm, dc=%lld ppb] : tmo = %lld msec\r\n",
-		(long long) usec_sdiff,
-		(long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
-		(long long) (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
-		(long long) (1000000*new_ecorr) / ERTS_TCORR_ERR_UNIT,
-		(long long) (1000000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
-		(long long) tmo);
-
+	erts_fprintf(stderr,
+		     "sdiff = %b64d usec : [ec=%b64d ppm, dc=%b64d ppb] "
+		     "-> [ec=%b64d ppm, dc=%b64d ppb] : tmo = %bpu msec\r\n",
+		     usec_sdiff,
+		     (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT,
+		     (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		     (1000000*new_ecorr) / ERTS_TCORR_ERR_UNIT,
+		     (1000000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT,
+		     tmo);
 }
 
 #endif
 
 static void
-check_time_correction(void *unused)
+check_time_correction(void *idap)
 {
-#ifndef ERTS_TIME_CORRECTION_PRINT
-#  define ERTS_PRINT_CORRECTION
-#else
-#  ifdef ERTS_HAVE_CORRECTED_OS_MONOTONIC
-#    define ERTS_PRINT_CORRECTION		\
-    print_correction(set_new_correction,	\
-		     sdiff,			\
-		     cip->correction.error,	\
-		     0,				\
-		     new_correction.error,	\
-		     0,				\
-		     timeout)
-#  else
-#    define ERTS_PRINT_CORRECTION		\
-    print_correction(set_new_correction,	\
-		     sdiff,			\
-		     cip->correction.error,	\
-		     cip->correction.drift,	\
-		     new_correction.error,	\
-		     new_correction.drift,	\
-		     timeout)
-#  endif
-#endif
+    UWord init_drift_adj = (UWord) idap;
     ErtsMonotonicCorrectionData cdata;
     ErtsMonotonicCorrection new_correction;
     ErtsMonotonicCorrectionInstance *cip;
     ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime,
 	erl_stime, time_offset;
     Uint timeout;
-    int set_new_correction, begin_short_intervals = 0;
+    int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time;
+    int set_new_correction = 0, begin_short_intervals = 0;
 
     erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
-    ASSERT(time_sup.inf.c.finalized_offset);
-
     erts_os_times(&os_mtime, &os_stime);
 
     cdata = time_sup.inf.c.parmon.cdata;
@@ -423,14 +437,23 @@ check_time_correction(void *unused)
 		 "OS monotonic time stepped backwards\n");
     cip = &cdata.curr;
 
-    erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, &mdiff);
+    erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, &mdiff,
+					 os_drift_corrected);
     time_offset = get_time_offset();
     erl_stime = erl_mtime + time_offset;
 
     sdiff = erl_stime - os_stime;
 
+    if (time_sup.inf.c.shadow_offset) {
+	ERTS_TIME_ASSERT(time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE);
+	if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
+	    sdiff += time_sup.inf.c.shadow_offset;
+	else
+	    time_sup.inf.c.shadow_offset = 0;
+    }
+
     new_correction = cip->correction;
-    
+
     if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE
 	&& (sdiff < -2*time_sup.r.o.adj.small_diff
 	    || 2*time_sup.r.o.adj.small_diff < sdiff)) {
@@ -440,9 +463,24 @@ check_time_correction(void *unused)
 	set_time_offset(time_offset);
 	schedule_send_time_offset_changed_notifications(time_offset);
 	begin_short_intervals = 1;
-	if (cdata.curr.correction.error == 0)
-	    set_new_correction = 0;
-	else {
+	if (cdata.curr.correction.error != 0) {
+	    set_new_correction = 1;
+	    new_correction.error = 0;
+	}
+    }
+    else if ((time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE
+	      && erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
+	     && (sdiff < -2*time_sup.r.o.adj.small_diff
+		 || 2*time_sup.r.o.adj.small_diff < sdiff)) {
+	/*
+	 * System time diff exeeded limits; change shadow offset
+	 * and let OS system time leap away from Erlang system
+	 * time.
+	 */
+	time_sup.inf.c.shadow_offset -= sdiff;
+	sdiff = 0;
+	begin_short_intervals = 1;
+	if (cdata.curr.correction.error != 0) {
 	    set_new_correction = 1;
 	    new_correction.error = 0;
 	}
@@ -462,16 +500,11 @@ check_time_correction(void *unused)
 	    else
 		new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ;
 	}
-	else {
-	    set_new_correction = 0;
-	}
     }
     else if (cdata.curr.correction.error > 0) {
 	if (sdiff < 0) {
-	    if (cdata.curr.correction.error == ERTS_TCORR_ERR_LARGE_ADJ
-		|| -time_sup.r.o.adj.large_diff <= sdiff)
-		set_new_correction = 0;
-	    else {
+	    if (cdata.curr.correction.error != ERTS_TCORR_ERR_LARGE_ADJ
+		&& sdiff < -time_sup.r.o.adj.large_diff) {
 		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
 		set_new_correction = 1;
 	    }
@@ -490,14 +523,11 @@ check_time_correction(void *unused)
     }
     else /* if (cdata.curr.correction.error < 0) */ { 
 	if (0 < sdiff) {
-	    if (cdata.curr.correction.error == -ERTS_TCORR_ERR_LARGE_ADJ
-		|| sdiff <= time_sup.r.o.adj.large_diff)
-		set_new_correction = 0;
-	    else {
+	    if (cdata.curr.correction.error != -ERTS_TCORR_ERR_LARGE_ADJ
+		&& time_sup.r.o.adj.large_diff < sdiff) {
 		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
 		set_new_correction = 1;
 	    }
-	    set_new_correction = 0;
 	}
 	else if (sdiff < -time_sup.r.o.adj.small_diff) {
 	    set_new_correction = 1;
@@ -512,8 +542,7 @@ check_time_correction(void *unused)
 	}
     }
 
-#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
-    {
+    if (!os_drift_corrected) {
 	ErtsMonotonicDriftData *ddp = &time_sup.inf.c.parmon.cdata.drift;
 	int ix = ddp->ix;
 	ErtsMonotonicTime mtime_diff, old_os_mtime;
@@ -521,25 +550,26 @@ check_time_correction(void *unused)
 	old_os_mtime = ddp->intervals[ix].time.mon;
 	mtime_diff = os_mtime - old_os_mtime;
 
-	if (mtime_diff >= ERTS_SEC_TO_MONOTONIC(10)) {
+	if ((mtime_diff >= ERTS_MIN_MONOTONIC_DRIFT_MEASUREMENT)
+	    | init_drift_adj) {
 	    ErtsMonotonicTime drift_adj, drift_adj_diff, old_os_stime,
-		stime_diff, mtime_acc, stime_acc, avg_drift_adj;
+		smtime_diff, stime_diff, mtime_acc, stime_acc,
+		avg_drift_adj, max_drift;
 
 	    old_os_stime = ddp->intervals[ix].time.sys;
 
 	    mtime_acc = ddp->acc.mon;
 	    stime_acc = ddp->acc.sys;
 
-	    avg_drift_adj = (((stime_acc - mtime_acc)*ERTS_MONOTONIC_TIME_UNIT)
+	    avg_drift_adj = (((stime_acc - mtime_acc)
+			      * ERTS_MONOTONIC_TIME_UNIT)
 			     / mtime_acc);
 
 	    mtime_diff = os_mtime - old_os_mtime;
 	    stime_diff = os_stime - old_os_stime;
-	    drift_adj = (((stime_diff - mtime_diff)*ERTS_MONOTONIC_TIME_UNIT)
-			 / mtime_diff);
-	    
+	    smtime_diff = stime_diff - mtime_diff;
 	    ix++;
-	    if (ix >= ERTS_DRIFT_INTERVALS)
+	    if (ix >= time_sup.r.o.drift_adj.intervals)
 		ix = 0;
 	    mtime_acc -= ddp->intervals[ix].diff.mon;
 	    mtime_acc += mtime_diff;
@@ -555,24 +585,50 @@ check_time_correction(void *unused)
 	    ddp->acc.mon = mtime_acc;
 	    ddp->acc.sys = stime_acc;
 
-	    drift_adj_diff = avg_drift_adj - drift_adj;
-	    if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF
-		|| ERTS_TIME_DRIFT_MAX_ADJ_DIFF < drift_adj_diff) {
-		ddp->dirty_counter = ERTS_DRIFT_INTERVALS;
+	    max_drift = ERTS_MAX_MONOTONIC_DRIFT;
+	    max_drift *= ERTS_MONOTONIC_TO_SEC(mtime_diff);
+
+	    if (smtime_diff > time_sup.r.o.drift_adj.error + max_drift
+		|| smtime_diff < -1*time_sup.r.o.drift_adj.error - max_drift) {
+	    dirty_intervals:
+		/*
+		 * We had a leap in system time. Mark array as
+		 * dirty to ensure that dirty values are rotated
+		 * out before we use it again...
+		 */
+		ddp->dirty_counter = time_sup.r.o.drift_adj.intervals;
 		begin_short_intervals = 1;
 	    }
-	    else {
-		if (ddp->dirty_counter <= 0) {
-		    drift_adj = ((stime_acc - mtime_acc)
-				 *ERTS_MONOTONIC_TIME_UNIT) / mtime_acc;
+	    else if (ddp->dirty_counter > 0) {
+		if (init_drift_adj) {
+		    new_correction.drift = ((smtime_diff
+					     * ERTS_MONOTONIC_TIME_UNIT)
+					    / mtime_diff);
+		    set_new_correction = 1;
 		}
-		if (ddp->dirty_counter >= 0) {
-		    if (ddp->dirty_counter == 0) {
-			/* Force set new drift correction... */
-			set_new_correction = 1;
-		    }
+		ddp->dirty_counter--;
+	    }
+	    else {
+		if (ddp->dirty_counter == 0) {
+		    /* Force set new drift correction... */
+		    set_new_correction = 1;
 		    ddp->dirty_counter--;
 		}
+
+		if (time_sup.r.o.drift_adj.use_avg)
+		    drift_adj = (((stime_acc - mtime_acc)
+				  * ERTS_MONOTONIC_TIME_UNIT)
+				 / mtime_acc);
+		else
+		    drift_adj = ((smtime_diff
+				  * ERTS_MONOTONIC_TIME_UNIT)
+				 / mtime_diff);
+
+		drift_adj_diff = avg_drift_adj - drift_adj;
+		if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF
+		    || ERTS_TIME_DRIFT_MAX_ADJ_DIFF < drift_adj_diff)
+		    goto dirty_intervals;
+
 		drift_adj_diff = drift_adj - new_correction.drift;
 		if (drift_adj_diff) {
 		    if (drift_adj_diff > ERTS_TIME_DRIFT_MAX_ADJ_DIFF)
@@ -580,7 +636,6 @@ check_time_correction(void *unused)
 		    else if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF)
 			drift_adj_diff = -ERTS_TIME_DRIFT_MAX_ADJ_DIFF;
 		    new_correction.drift += drift_adj_diff;
-
 		    if (drift_adj_diff < -ERTS_TIME_DRIFT_MIN_ADJ_DIFF
 			|| ERTS_TIME_DRIFT_MIN_ADJ_DIFF < drift_adj_diff) {
 			set_new_correction = 1;
@@ -589,7 +644,6 @@ check_time_correction(void *unused)
 	    }
 	}
     }
-#endif
 
     begin_short_intervals |= set_new_correction;
     
@@ -608,25 +662,34 @@ check_time_correction(void *unused)
 	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK);
     else {
 	ErtsMonotonicTime ecorr = new_correction.error;
-	if (sdiff < 0)
-	    sdiff = -1*sdiff;
+	ErtsMonotonicTime abs_sdiff;
+	abs_sdiff = (sdiff < 0) ? -1*sdiff : sdiff;
 	if (ecorr < 0)
 	    ecorr = -1*ecorr;
-	if (sdiff > ecorr*(ERTS_LONG_TIME_CORRECTION_CHECK/ERTS_TCORR_ERR_UNIT))
+	if (abs_sdiff > ecorr*(ERTS_LONG_TIME_CORRECTION_CHECK/ERTS_TCORR_ERR_UNIT))
 	    timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK);
 	else {
-	    timeout = ERTS_MONOTONIC_TO_MSEC((ERTS_TCORR_ERR_UNIT*sdiff)/ecorr);
+	    timeout = ERTS_MONOTONIC_TO_MSEC((ERTS_TCORR_ERR_UNIT*abs_sdiff)/ecorr);
 	    if (timeout < 10)
 		timeout = 10;
 	}
     }
 
     if (timeout > ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK)
-	&& time_sup.inf.c.parmon.cdata.short_check_interval) {
+	&& (time_sup.inf.c.parmon.cdata.short_check_interval
+	    || time_sup.inf.c.parmon.cdata.drift.dirty_counter >= 0)) {
 	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK);
     }
 
-    ERTS_PRINT_CORRECTION;
+#ifdef ERTS_TIME_CORRECTION_PRINT
+    print_correction(set_new_correction,
+		     sdiff,
+		     cip->correction.error,
+		     cip->correction.drift,
+		     new_correction.error,
+		     new_correction.drift,
+		     timeout);
+#endif
 
     if (set_new_correction) {
 	erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx);
@@ -638,16 +701,17 @@ check_time_correction(void *unused)
 
 	/*
 	 * Current correction instance begin when
-	 * OS monotonic time has increased one unit.
+	 * OS monotonic time has increased two units.
 	 */
-	os_mtime++;
+	os_mtime += 2;
 
 	/*
 	 * Erlang monotonic time corresponding to
 	 * next OS monotonic time using previous
 	 * correction.
 	 */
-	erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, NULL);
+	erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, NULL,
+					     os_drift_corrected);
 
 	/*
 	 * Save new current correction instance.
@@ -664,18 +728,60 @@ check_time_correction(void *unused)
 		   NULL,
 		   NULL,
 		   timeout);
+}
 
-#undef ERTS_PRINT_CORRECTION
+static ErtsMonotonicTime get_os_corrected_time(void)
+{
+    ASSERT(time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE);
+    return erts_os_monotonic_time() + time_sup.r.o.moffset;
 }
 
-#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
+static void
+check_time_offset(void *unused)
+{
+    ErtsMonotonicTime sdiff, os_mtime, erl_mtime, os_stime,
+	erl_stime, time_offset;
+
+    ASSERT(time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE);
+
+    erts_os_times(&os_mtime, &os_stime);
+
+    erl_mtime =  os_mtime + time_sup.r.o.moffset;
+    time_offset = get_time_offset();
+    erl_stime = erl_mtime + time_offset;
+
+    sdiff = erl_stime - os_stime;
+
+    if ((sdiff < -2*time_sup.r.o.adj.small_diff
+	 || 2*time_sup.r.o.adj.small_diff < sdiff)) {
+	/* System time diff exeeded limits; change time offset... */
+#ifdef ERTS_TIME_CORRECTION_PRINT
+	erts_fprintf(stderr, "sdiff = %b64d nsec -> 0 nsec\n",
+		     ERTS_MONOTONIC_TO_NSEC(sdiff));
+#endif
+	time_offset -= sdiff;
+	sdiff = 0;
+	set_time_offset(time_offset);
+	schedule_send_time_offset_changed_notifications(time_offset);
+    }
+#ifdef ERTS_TIME_CORRECTION_PRINT
+    else erts_fprintf(stderr, "sdiff = %b64d nsec\n",
+		      ERTS_MONOTONIC_TO_NSEC(sdiff));
+#endif
+
+    erts_set_timer(&time_sup.inf.c.parmon.timer,
+		   check_time_offset,
+		   NULL,
+		   NULL,
+		   ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK));
+}
 
 static void
-init_check_time_correction(void *unused)
+init_check_time_correction(void *quick_init_drift)
 {
     ErtsMonotonicDriftData *ddp;
     ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff,
-	stime_diff;
+	stime_diff, smtime_diff, max_drift;
     int ix;
 
     ddp = &time_sup.inf.c.parmon.cdata.drift;
@@ -687,7 +793,13 @@ init_check_time_correction(void *unused)
 
     mtime_diff = mtime - old_mtime;
     stime_diff = stime - old_stime;
-    if (100*stime_diff  < 80*mtime_diff || 120*mtime_diff < 100*stime_diff ) {
+    smtime_diff = stime_diff - mtime_diff;
+
+    max_drift = ERTS_MAX_MONOTONIC_DRIFT;
+    max_drift *= ERTS_MONOTONIC_TO_SEC(mtime_diff);
+
+    if (smtime_diff > time_sup.r.o.drift_adj.error + max_drift
+	|| smtime_diff < -1*time_sup.r.o.drift_adj.error - max_drift) {
 	/* Had a system time leap... pretend no drift... */
 	stime_diff = mtime_diff;
     }
@@ -697,29 +809,28 @@ init_check_time_correction(void *unused)
      * a drift adjustment, and repeat this interval
      * in all slots...
      */
-    for (ix = 0; ix < ERTS_DRIFT_INTERVALS; ix++) {
+    for (ix = 0; ix < time_sup.r.o.drift_adj.intervals; ix++) {
 	ddp->intervals[ix].diff.mon = mtime_diff;
 	ddp->intervals[ix].diff.sys = stime_diff;
 	ddp->intervals[ix].time.mon = old_mtime;
 	ddp->intervals[ix].time.sys = old_stime;
     }
 
-    ddp->acc.sys = stime_diff*ERTS_DRIFT_INTERVALS;
-    ddp->acc.mon = mtime_diff*ERTS_DRIFT_INTERVALS;
+    ddp->acc.sys = stime_diff*time_sup.r.o.drift_adj.intervals;
+    ddp->acc.mon = mtime_diff*time_sup.r.o.drift_adj.intervals;
     ddp->ix = 0;
-    ddp->dirty_counter = ERTS_DRIFT_INTERVALS;
+    ddp->dirty_counter = time_sup.r.o.drift_adj.intervals;
 
-    check_time_correction(NULL);
+    check_time_correction(quick_init_drift);
 }
 
-#endif
-
 static ErtsMonotonicTime
 finalize_corrected_time_offset(ErtsSystemTime *stimep)
 {
     ErtsMonotonicTime os_mtime;
     ErtsMonotonicCorrectionData cdata;
     ErtsMonotonicCorrectionInstance *cip;
+    int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time;
 
     erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
@@ -734,25 +845,38 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep)
 		 "OS monotonic time stepped backwards\n");
     cip = &cdata.curr;
 
-    return calc_corrected_erl_mtime(os_mtime, cip, NULL);
+    return calc_corrected_erl_mtime(os_mtime, cip, NULL,
+				    os_drift_corrected);
 }
 
 static void
 late_init_time_correction(void)
 {
-    if (time_sup.inf.c.finalized_offset) {
+    Uint timeout;
+    Uint quick_init_drift_adj;
+    void (*check_func)(void *);
 
-	erts_init_timer(&time_sup.inf.c.parmon.timer);
-	erts_set_timer(&time_sup.inf.c.parmon.timer,
-#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
-		       init_check_time_correction,
-#else
-		       check_time_correction,
-#endif
-		       NULL,
-		       NULL,
-		       ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK));
-    }
+    quick_init_drift_adj =
+	(Uint) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error) == 0;
+
+    if (quick_init_drift_adj)
+	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK/10);
+    else
+	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK);
+
+    if (!time_sup.r.o.os_corrected_monotonic_time)
+	check_func = init_check_time_correction;
+    else if (time_sup.r.o.get_time == get_os_corrected_time)
+	check_func = check_time_offset;
+    else
+	check_func = check_time_correction;
+
+    erts_init_timer(&time_sup.inf.c.parmon.timer);
+    erts_set_timer(&time_sup.inf.c.parmon.timer,
+		   check_func,
+		   NULL,
+		   (void *) quick_init_drift_adj,
+		   timeout);
 }
 
 #endif /* ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT */
@@ -832,6 +956,8 @@ void erts_init_sys_time_sup(void)
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
     time_sup.r.o.os_monotonic_time_disable
 	= !sys_init_time_res.have_os_monotonic_time;
+    time_sup.r.o.os_corrected_monotonic_time =
+	sys_init_time_res.have_corrected_os_monotonic_time;
     time_sup.r.o.os_monotonic_time_func
 	= sys_init_time_res.os_monotonic_time_info.func;
     time_sup.r.o.os_monotonic_time_clock_id
@@ -856,7 +982,7 @@ void erts_init_sys_time_sup(void)
 int 
 erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 {
-    ErtsMonotonicTime resolution;
+    ErtsMonotonicTime resolution, ilength, intervals, short_isecs;
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
     ErtsMonotonicTime abs_native_offset, native_offset;
 #endif
@@ -870,9 +996,10 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     time_sup.r.o.warp_mode = time_warp_mode;
  
     if (time_warp_mode == ERTS_SINGLE_TIME_WARP_MODE)
-	time_sup.inf.c.finalized_offset = 0;
+	erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 1);
     else
-	time_sup.inf.c.finalized_offset = ~0;
+	erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 0);
+    time_sup.inf.c.shadow_offset = 0;
 
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
 
@@ -882,7 +1009,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
     time_sup.r.o.start += ERTS_MONOTONIC_TIME_UNIT;
     native_offset = time_sup.r.o.start - ERTS_MONOTONIC_TIME_UNIT;
-    native_offset = native_offset;
+    abs_native_offset = native_offset;
 #else /* ARCH_64 */
     if (ERTS_MONOTONIC_TIME_UNIT <= 10*1000*1000) {
 	time_sup.r.o.start = 0;
@@ -938,17 +1065,66 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	time_sup.r.o.adj.large_diff = ERTS_USEC_TO_MONOTONIC(500);
     time_sup.r.o.adj.small_diff = time_sup.r.o.adj.large_diff/10;
 
+    time_sup.r.o.drift_adj.resolution = resolution;
+
+    if (time_sup.r.o.os_corrected_monotonic_time) {
+	time_sup.r.o.drift_adj.use_avg = 0;
+	time_sup.r.o.drift_adj.intervals = 0;
+	time_sup.r.o.drift_adj.error = 0;
+	time_sup.inf.c.parmon.cdata.drift.dirty_counter = -1;
+    }
+    else {
+	/*
+	 * Calculate length of the interval in seconds needed
+	 * in order to get an error that is at most 1 micro second.
+	 * If this interval is longer than the short time correction
+	 * check interval we use the average of all values instead
+	 * of the latest value.
+	 */
+	short_isecs = ERTS_MONOTONIC_TO_SEC(ERTS_SHORT_TIME_CORRECTION_CHECK);
+	ilength = ERTS_ASSUMED_PRECISION_DROP * ERTS_MONOTONIC_TIME_UNIT;
+	ilength /= (resolution * ERTS_USEC_TO_MONOTONIC(1));
+	time_sup.r.o.drift_adj.use_avg = ilength > short_isecs;
+
+	if (ilength == 0)
+	    intervals = 5;
+	else {
+	    intervals = ilength / short_isecs;
+	    if (intervals > ERTS_MAX_DRIFT_INTERVALS)
+		intervals = ERTS_MAX_DRIFT_INTERVALS;
+	    else if (intervals < 5)
+		intervals = 5;
+	}
+	time_sup.r.o.drift_adj.intervals = (int) intervals;
+
+	/*
+	 * drift_adj.error equals maximum assumed error
+	 * over a short time interval. We use this value also
+	 * when examining a large interval. In this case the
+	 * error will be smaller, but we do not want to
+	 * recalculate this over and over again.
+	 */
+
+	time_sup.r.o.drift_adj.error = ERTS_MONOTONIC_TIME_UNIT;
+	time_sup.r.o.drift_adj.error *= ERTS_ASSUMED_PRECISION_DROP;
+	time_sup.r.o.drift_adj.error /= resolution * short_isecs;
+    }
 #ifdef ERTS_TIME_CORRECTION_PRINT
-    fprintf(stderr, "start         = %lld\n\r", (long long) ERTS_MONOTONIC_TIME_START);
-    fprintf(stderr, "native offset = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_NATIVE);
-    fprintf(stderr, "nsec offset   = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_NSEC);
-    fprintf(stderr, "usec offset   = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_USEC);
-    fprintf(stderr, "msec offset   = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_MSEC);
-    fprintf(stderr, "sec offset    = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_SEC);
-    fprintf(stderr, "large diff    = %lld usec\r\n",
-	    (long long) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.large_diff));
-    fprintf(stderr, "small diff    = %lld usec\r\n",
-	    (long long) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.small_diff));
+    erts_fprintf(stderr, "resolution           = %b64d\n", resolution);
+    erts_fprintf(stderr, "adj large diff       = %b64d usec\n",
+		 ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.large_diff));
+    erts_fprintf(stderr, "adj small diff       = %b64d usec\n",
+		 ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.small_diff));
+    if (!time_sup.r.o.os_corrected_monotonic_time) {
+	erts_fprintf(stderr, "drift intervals      = %d\n",
+		     time_sup.r.o.drift_adj.intervals);
+	erts_fprintf(stderr, "drift adj error      = %b64d usec\n",
+		     ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error));
+	erts_fprintf(stderr, "drift adj max diff   = %b64d nsec\n",
+		     ERTS_MONOTONIC_TO_NSEC(ERTS_TIME_DRIFT_MAX_ADJ_DIFF));
+	erts_fprintf(stderr, "drift adj min diff   = %b64d nsec\n",
+		     ERTS_MONOTONIC_TO_NSEC(ERTS_TIME_DRIFT_MIN_ADJ_DIFF));
+    }
 #endif
 
     if (ERTS_MONOTONIC_TIME_UNIT < ERTS_CLKTCK_RESOLUTION)
@@ -967,6 +1143,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	erts_os_times(&time_sup.inf.c.minit,
 		      &time_sup.inf.c.sinit);
 	time_sup.r.o.moffset = -1*time_sup.inf.c.minit;
+	time_sup.r.o.moffset += ERTS_MONOTONIC_TIME_UNIT;
 	offset = time_sup.inf.c.sinit;
 	offset -= ERTS_MONOTONIC_TIME_UNIT;
 	init_time_offset(offset);
@@ -979,11 +1156,9 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
 	cdatap = &time_sup.inf.c.parmon.cdata;
     
-#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
 	cdatap->drift.intervals[0].time.sys = time_sup.inf.c.sinit;
 	cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit;
 	cdatap->curr.correction.drift = 0;
-#endif
 	cdatap->curr.correction.error = 0;
 	cdatap->curr.erl_mtime = ERTS_MONOTONIC_TIME_UNIT;
 	cdatap->curr.os_mtime = time_sup.inf.c.minit;
@@ -991,7 +1166,12 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER;
 	cdatap->prev = cdatap->curr;
 
-	time_sup.r.o.get_time = get_corrected_time;
+	if (!time_sup.r.o.os_corrected_monotonic_time)
+	    time_sup.r.o.get_time = get_corrected_time;
+	else if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE)
+	    time_sup.r.o.get_time = get_os_corrected_time;
+	else
+	    time_sup.r.o.get_time = get_os_drift_corrected_time;
     }
     else
 #endif
@@ -1020,8 +1200,8 @@ void
 erts_late_init_time_sup(void)
 {
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
-    /* Timer wheel must be initialized */
-    if (time_sup.r.o.get_time == get_corrected_time)
+    /* Timer wheel must have beeen initialized */
+    if (time_sup.r.o.get_time != get_not_corrected_time)
 	late_init_time_correction();
 #endif
     erts_late_sys_init_time();
@@ -1038,9 +1218,9 @@ ErtsTimeOffsetState erts_time_offset_state(void)
     case ERTS_NO_TIME_WARP_MODE:
 	return ERTS_TIME_OFFSET_FINAL;
     case ERTS_SINGLE_TIME_WARP_MODE:
-	if (time_sup.inf.c.finalized_offset)
-	    return ERTS_TIME_OFFSET_FINAL;
-	return ERTS_TIME_OFFSET_PRELIMINARY;
+	if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
+	    return ERTS_TIME_OFFSET_PRELIMINARY;
+	return ERTS_TIME_OFFSET_FINAL;
     case ERTS_MULTI_TIME_WARP_MODE:
 	return ERTS_TIME_OFFSET_VOLATILE;
     default:
@@ -1073,7 +1253,7 @@ erts_finalize_time_offset(void)
 
 	erts_smp_mtx_lock(&erts_get_time_mtx);
 
-	if (!time_sup.inf.c.finalized_offset) {
+	if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) {
 	    ErtsMonotonicTime mtime, new_offset;
 
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
@@ -1110,19 +1290,12 @@ erts_finalize_time_offset(void)
 	    set_time_offset(new_offset);
 	    schedule_send_time_offset_changed_notifications(new_offset);
 
-	    time_sup.inf.c.finalized_offset = ~0;
+	    erts_smp_atomic32_set_nob(&time_sup.inf.c.preliminary_offset, 0);
 	    res = ERTS_TIME_OFFSET_PRELIMINARY;
 	}
 
 	erts_smp_mtx_unlock(&erts_get_time_mtx);
 
-#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
-	if (res == ERTS_TIME_OFFSET_PRELIMINARY
-	    && time_sup.r.o.get_time == get_corrected_time) {
-	    late_init_time_correction();
-	}
-#endif
-
 	return res;
     }
     default:
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 251b39508f..f7a21406f3 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -703,14 +703,9 @@ extern char *erts_default_arg0;
 
 extern char os_type[];
 
-typedef enum {
-    ERTS_NO_TIME_WARP_MODE,
-    ERTS_SINGLE_TIME_WARP_MODE,
-    ERTS_MULTI_TIME_WARP_MODE
-} ErtsTimeWarpMode;
-
 typedef struct {
     int have_os_monotonic_time;
+    int have_corrected_os_monotonic_time;
     ErtsMonotonicTime os_monotonic_time_unit;
     ErtsMonotonicTime sys_clock_resolution;
     struct {
@@ -729,14 +724,13 @@ typedef struct {
 } ErtsSysInitTimeResult;
 
 #define ERTS_SYS_INIT_TIME_RESULT_INITER \
-    {0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}
+    {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}
 
 extern void erts_init_sys_time_sup(void);
 extern void sys_init_time(ErtsSysInitTimeResult *);
 extern void erts_late_sys_init_time(void);
 extern void erts_deliver_time(void);
 extern void erts_time_remaining(SysTimeval *);
-extern int erts_init_time_sup(int, ErtsTimeWarpMode);
 extern void erts_sys_init_float(void);
 extern void erts_thread_init_float(void);
 extern void erts_thread_disable_fpe(void);
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 2bdda6c8af..1fffb5f357 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -115,7 +115,7 @@ struct ErtsTimerWheel_ {
     Uint nto;
     struct {
 	ErlTimer *head;
-	ErlTimer **tail;
+	ErlTimer *tail;
 	Uint nto;
     } at_once;
     int true_next_timeout_time;
@@ -235,23 +235,39 @@ found_next:
 static void
 remove_timer(ErtsTimerWheel *tiw, ErlTimer *p)
 {
-    /* first */
-    if (!p->prev) {
-	tiw->w[p->slot] = p->next;
+    if (p->slot >= 0) {
+	/* Timer in wheel... */
+	ASSERT(p->slot < TIW_SIZE);
+	if (p->prev)
+	    p->prev->next = p->next;
+	else {
+	    ASSERT(tiw->w[p->slot] == p);
+	    tiw->w[p->slot] = p->next;
+	}
 	if(p->next)
-	    p->next->prev = NULL;
-    } else {
-	p->prev->next = p->next;
+	    p->next->prev = p->prev;
     }
-
-    /* last */
-    if (!p->next) {
+    else {
+	/* Timer in "at once" queue... */
+	ASSERT(p->slot == -1);
 	if (p->prev)
-	    p->prev->next = NULL;
-    } else {
-	p->next->prev = p->prev;
+	    p->prev->next = p->next;
+	else {
+	    ASSERT(tiw->at_once.head == p);
+	    tiw->at_once.head = p->next;
+	}
+	if (p->next)
+	    p->next->prev = p->prev;
+	else {
+	    ASSERT(tiw->at_once.tail == p);
+	    tiw->at_once.tail = p->prev;
+	}
+	ASSERT(tiw->at_once.nto > 0);
+	tiw->at_once.nto--;
     }
 
+    p->slot = -2;
+
     p->next = NULL;
     p->prev = NULL;
 
@@ -337,11 +353,17 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
     }
     else {
 	ASSERT(tiw->nto >= tiw->at_once.nto);
-	timeout_head = tiw->at_once.head;
-	timeout_tail = tiw->at_once.tail;
+	p = timeout_head = tiw->at_once.head;
+	while (1) {
+	    set_timer_wheel(p, NULL);
+	    if (!p->next) {
+		timeout_tail = &p->next;
+		break;
+	    }
+	}
 	tiw->nto -= tiw->at_once.nto;
 	tiw->at_once.head = NULL;
-	tiw->at_once.tail = &tiw->at_once.head;
+	tiw->at_once.tail = NULL;
 	tiw->at_once.nto = 0;
     }
 
@@ -458,7 +480,7 @@ erts_create_timer_wheel(int no)
     tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
     tiw->nto = 0;
     tiw->at_once.head = NULL;
-    tiw->at_once.tail = &tiw->at_once.head;
+    tiw->at_once.tail = NULL;
     tiw->at_once.nto = 0;
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
@@ -523,10 +545,13 @@ erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout,
 	timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 	tiw->nto++;
 	tiw->at_once.nto++;
-	*tiw->at_once.tail = p;
-	tiw->at_once.tail = &p->next;
 	p->next = NULL;
+	p->prev = tiw->at_once.tail;
+	tiw->at_once.tail = p;
+	if (!tiw->at_once.head)
+	    tiw->at_once.head = p;
 	p->timeout_pos = timeout_pos;
+	p->slot = -1;
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
     }
     else {
@@ -538,7 +563,7 @@ erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout,
 
 	/* calculate slot */
 	tm = (int) (timeout_pos & (TIW_SIZE-1));
-	p->slot = (Uint) tm;
+	p->slot = tm;
   
 	/* insert at head of list at slot */
 	p->next = tiw->w[tm];
@@ -587,7 +612,6 @@ erts_cancel_timer(ErlTimer *p)
 	cancel = NULL;
     else {
 	remove_timer(tiw, p);
-	p->slot = 0;
 
 	cancel = p->cancel;
 	arg = p->arg;
-- 
cgit v1.2.3


From 7ba91b64862e29bfd579b04c73e2bccacde6a003 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 5 May 2015 20:03:17 +0200
Subject: Reusable red-black tree implementation

---
 erts/emulator/beam/erl_rbtree.h | 1740 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 1740 insertions(+)
 create mode 100644 erts/emulator/beam/erl_rbtree.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h
new file mode 100644
index 0000000000..ea0a8976bb
--- /dev/null
+++ b/erts/emulator/beam/erl_rbtree.h
@@ -0,0 +1,1740 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2015. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	A Red-Black (binary search) Tree implementation. The search,
+ *              insert, and delete operations are all O(log n) operations
+ *              on a Red-Black Tree. Red-Black Trees are described in
+ *              "Introduction to Algorithms", by Thomas H. Cormen, Charles
+ *              E. Leiserson, and Ronald L. Riverest.
+ *
+ *              Use by defining mandatory defines as well as defines for
+ *              API functions wanted, and include this header.
+ *
+ * Author: 	Rickard Green
+ *
+ *
+ * Mandatory defines:
+ * - ERTS_RBT_PREFIX - Prefix to use on functions.
+ * - ERTS_RBT_T - Type of a tree node.
+ * - ERTS_RBT_KEY_T - Type of key for a tree node.
+ * - ERTS_RBT_FLAGS_T - Type of flags for a tree node.
+ * - ERTS_RBT_INIT_EMPTY_TNODE(T) -Initialize an empty tree node.
+ * - ERTS_RBT_IS_RED(T) - Is tree node red?
+ * - ERTS_RBT_SET_RED(T) - Set tree node red.
+ * - ERTS_RBT_IS_BLACK(T) - Is tree node back?
+ * - ERTS_RBT_SET_BLACK(T) - Set tree node black.
+ * - ERTS_RBT_GET_FLAGS(T) - Get flags of tree node (incl colors).
+ * - ERTS_RBT_SET_FLAGS(T, F) - Set flags of tree note.
+ * - ERTS_RBT_GET_PARENT(T) - Get parent node.
+ * - ERTS_RBT_SET_PARENT(T, P) - Set parent node.
+ * - ERTS_RBT_GET_RIGHT(T) - Get right child node.
+ * - ERTS_RBT_SET_RIGHT(T, R) - Set right child node.
+ * - ERTS_RBT_GET_LEFT(T) - Get left child node.
+ * - ERTS_RBT_SET_LEFT(T, L) - Set left child node.
+ * - ERTS_RBT_GET_KEY(T) - Get key of node.
+ * - ERTS_RBT_IS_LT(KX, KY) - Is key KX less than key KY?
+ * - ERTS_RBT_IS_EQ(KX, KY) - Is key KX equal to key KY?
+ *
+ * Optional defines:
+ *
+ * - ERTS_RBT_UNDEF - Undefine all user defined ERTS_RBT_*
+ *     defines after use.
+ *
+ * - ERTS_RBT_NO_API_INLINE - Do not inline API functions.
+ *
+ * Attached data management:
+ * - ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(L, OP, NP) - Called
+ *     when a rotate operation has been performed. If L (in int)
+ *     is a non zero, a left rotation was performed; otherwise,
+ *     a right rotation was performed. OR points to the old
+ *     parent node and NP points to the new parent node.
+ * - ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(F, T) - Called when
+ *     a delete operation modifies a tree node. A delete
+ *     modification is either a removal or replacement of a
+ *     node. F points to the parent of the tree node that was
+ *     modified. T points to the next ancestor that will be
+ *     modified. If T is NULL, no more removal and/or
+ *     replacements will be made. One typically wants to update
+ *     the attached data of each node between F and T. If T is
+ *     NULL all the way up to the root.
+ * - ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(OR, NR) - Called
+ *     when the root node changes. OR points to the old
+ *     root node and NP points to the new root node.
+ *
+ * Request implementation of API functions:
+ * - ERTS_RBT_WANT_DELETE
+ * - ERTS_RBT_WANT_INSERT
+ * - ERTS_RBT_WANT_LOOKUP_INSERT
+ * - ERTS_RBT_WANT_REPLACE
+ * - ERTS_RBT_WANT_LOOKUP
+ * - ERTS_RBT_WANT_SMALLEST
+ * - ERTS_RBT_WANT_LARGEST
+ * - ERTS_RBT_WANT_FOREACH
+ * - ERTS_RBT_WANT_FOREACH_DESTROY
+ * - ERTS_RBT_WANT_FOREACH_YIELDING
+ * - ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+ * - ERTS_RBT_WANT_FOREACH_SMALL
+ * - ERTS_RBT_WANT_FOREACH_LARGE
+ * - ERTS_RBT_WANT_FOREACH_SMALL_DESTROY
+ * - ERTS_RBT_WANT_FOREACH_LARGE_DESTROY
+ * - ERTS_RBT_WANT_FOREACH_SMALL_YIELDING
+ * - ERTS_RBT_WANT_FOREACH_LARGE_YIELDING
+ * - ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING
+ * - ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING
+ * - ERTS_RBT_WANT_DEBUG_PRINT
+ *
+ * The yield state data type will equal
+ * <ERTS_RBT_PREFIX>_rbt_yield_state_t.
+ *
+ * The yield state should be statically initialized by
+ * ERTS_RBT_YIELD_STAT_INITER.
+ *
+ *
+ * The following API functions are implemented if corresponding
+ * ERTS_RBT_WANT_<OPERATION> is defined:
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_delete(
+ *                         ERTS_RBT_T **tree,
+ *                         ERTS_RBT_T *element);
+ *         Delete element from tree.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_insert(
+ *                         ERTS_RBT_T **tree,
+ *                         ERTS_RBT_T *element);
+ *         Insert element into tree.
+ *
+ * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_lookup_insert(
+ *                         ERTS_RBT_T **tree,
+ *                         ERTS_RBT_T *element);
+ *         Look up an element in the tree that compares as equal to the
+ *         element passed as argument, and return the looked up element.
+ *         If no element compared as equal, insert the element passed as
+ *         argument into the tree, and return NULL.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_replace(
+ *                         ERTS_RBT_T **tree,
+ *                         ERTS_RBT_T *old_element,
+ *                         ERTS_RBT_T *new_element);
+ *         Replace old_element in the tree with new_element. Both elements
+ *         *should* compare as equal.
+ *
+ * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_lookup(
+ *                         ERTS_RBT_T *tree,
+ *                         ERTS_RBT_KEY_T key);
+ *         Look up an element with a key that compares as equal to
+ *         the key passed as argument.
+ *
+ * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_smallest(
+ *                         ERTS_RBT_T *tree);
+ *         Look up the element with the smallest key.
+ *
+ * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_largest(
+ *                         ERTS_RBT_T *tree);
+ *         Look up the element with the largest key.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_foreach(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg);
+ *         Operate by calling the operator 'op' on each element.
+ *         Order is undefined.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_foreach_destroy(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg);
+ *         Operate by calling the operator 'op' on each element.
+ *         Order is undefined. Each element should be destroyed
+ *         by 'op'.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - int <ERTS_RBT_PREFIX>_rbt_foreach_yielding(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg,
+ *                         <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate,
+ *                         Sint ylimit);
+ *         Operate by calling the operator 'op' on each element.
+ *         Order is undefined.
+ *
+ *         Yield when 'ylimit' elements has been processed. Zero is
+ *         returned when yielding, and a non-zero value is returned when
+ *         the whole tree has been processed. The tree should not be
+ *         modified until all of it has been processed.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - int <ERTS_RBT_PREFIX>_rbt_foreach_destroy_yielding(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg,
+ *                         <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate,
+ *                         Sint ylimit);
+ *         Operate by calling the operator 'op' on each element.
+ *         Order is undefined. Each element should be destroyed
+ *         by 'op'.
+ *
+ *         Yield when 'ylimit' elements has been processed. Zero is
+ *         returned when yielding, and a non-zero value is returned when
+ *         the whole tree has been processed.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_foreach_small(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg);
+ *         Operate by calling the operator 'op' on each element from
+ *         smallest towards larger elements.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_foreach_large(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg);
+ *         Operate by calling the operator 'op' on each element from
+ *         largest towards smaller elements.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_yielding(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg,
+ *                         <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate,
+ *                         Sint ylimit);
+ *         Operate by calling the operator 'op' on each element from
+ *         smallest towards larger elements.
+ *
+ *         Yield when 'ylimit' elements has been processed. Zero is
+ *         returned when yielding, and a non-zero value is returned when
+ *         the whole tree has been processed. The tree should not be
+ *         modified until all of it has been processed.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_yielding(
+ *                         ERTS_RBT_T *tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void *arg,
+ *                         <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate,
+ *                         Sint ylimit);
+ *         Operate by calling the operator 'op' on each element from
+ *         largest towards smaller elements.
+ *
+ *         Yield when 'ylimit' elements has been processed. Zero is
+ *         returned when yielding, and a non-zero value is returned when
+ *         the whole tree has been processed. The tree should not be
+ *         modified until all of it has been processed.
+ *
+ *         'arg' is passed as argument to 'op'.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy(
+ *                         ERTS_RBT_T **tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void (*destr)(ERTS_RBT_T *, void *),
+ *                         void *arg);
+ *         Operate by calling the operator 'op' on each element from
+ *         smallest towards larger elements.
+ *
+ *         Destroy elements by calling the destructor 'destr'. Elements
+ *         are destroyed when not needed by the tree structure anymore.
+ *         Note that elements are often *not* destroyed in another order
+ *         than the order that the elements are operated on.
+ *
+ *         'arg' is passed as argument to 'op' and 'destroy'.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy(
+ *                         ERTS_RBT_T **tree,
+ *                         void (*op)(ERTS_RBT_T *, void *),
+ *                         void (*destr)(ERTS_RBT_T *, void *),
+ *                         void *arg);
+ *         Operate by calling the operator 'op' on each element from
+ *         largest towards smaller elements.
+ *
+ *         Destroy elements by calling the destructor 'destr'. Elements
+ *         are destroyed when not needed by the tree structure anymore.
+ *         Note that elements are often destroyed in another order
+ *         than the order that the elements are operated on.
+ *
+ *         'arg' is passed as argument to 'op' and 'destroy'.
+ *
+ * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy_yielding(
+ *                        ERTS_RBT_T **tree,
+ *                        void (*op)(ERTS_RBT_T *, void *),
+ *                        void (*destr)(ERTS_RBT_T *, void *),
+ *                        void *arg,
+ *                        <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate,
+ *                        Sint ylimit);
+ *         Operate by calling the operator 'op' on each element from
+ *         smallest towards larger elements.
+ *
+ *         Destroy elements by calling the destructor 'destr'. Elements
+ *         are destroyed when not needed by the tree structure anymore.
+ *         Note that elements are often destroyed in another order
+ *         than the order that the elements are operated on.
+ *
+ *         Yield when 'ylimit' elements has been processed. Zero is
+ *         returned when yielding, and a non-zero value is returned when
+ *         the whole tree has been processed. The tree should not be
+ *         modified until all of it has been processed.
+ *
+ *         'arg' is passed as argument to 'op' and 'destroy'.
+ *
+ * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy_yielding(
+ *                        ERTS_RBT_T **tree,
+ *                        void (*op)(ERTS_RBT_T *, void *),
+ *                        void (*destr)(ERTS_RBT_T *, void *),
+ *                        void *arg,
+ *                        <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate,
+ *                        Sint ylimit);
+ *         Operate by calling the operator 'op' on each element from
+ *         largest towards smaller elements.
+ *
+ *         Destroy elements by calling the destructor 'destr'. Elements
+ *         are destroyed when not needed by the tree structure anymore.
+ *         Note that elements are often destroyed in another order
+ *         than the order that the elements are operated on.
+ *
+ *         Yield when 'ylimit' elements has been processed. Zero is
+ *         returned when yielding, and a non-zero value is returned when
+ *         the whole tree has been processed. The tree should not be
+ *         modified until all of it has been processed.
+ *
+ *         'arg' is passed as argument to 'op' and 'destroy'.
+ *
+ * - void <ERTS_RBT_PREFIX>_rbt_debug_print(
+ *                        FILE *filep,
+ *                        ERTS_RBT_T *x,
+ *                        int indent,
+ *                        (void)(*print_node)(ERTS_RBT_T *));
+ *         Prints the tree. Note that this function is recursive.
+ *         Should only be used for debuging.
+ */
+
+
+/*
+ * Check that we have all mandatory defines
+ */
+#ifndef ERTS_RBT_PREFIX
+#  error Missing definition of ERTS_RBT_PREFIX
+#endif
+#ifndef ERTS_RBT_T
+#  error Missing definition of ERTS_RBT_T
+#endif
+#ifndef ERTS_RBT_KEY_T
+#  error Missing definition of ERTS_RBT_KEY_T
+#endif
+#ifndef ERTS_RBT_FLAGS_T
+#  error Missing definition of ERTS_RBT_FLAGS_T
+#endif
+#ifndef ERTS_RBT_INIT_EMPTY_TNODE
+#  error Missing definition of ERTS_RBT_INIT_EMPTY_TNODE
+#endif
+#ifndef ERTS_RBT_IS_RED
+#  error Missing definition of ERTS_RBT_IS_RED
+#endif
+#ifndef ERTS_RBT_SET_RED
+#  error Missing definition of ERTS_RBT_SET_RED
+#endif
+#ifndef ERTS_RBT_IS_BLACK
+#  error Missing definition of ERTS_RBT_IS_BLACK
+#endif
+#ifndef ERTS_RBT_SET_BLACK
+#  error Missing definition of ERTS_RBT_SET_BLACK
+#endif
+#ifndef ERTS_RBT_GET_FLAGS
+#  error Missing definition of ERTS_RBT_GET_FLAGS
+#endif
+#ifndef ERTS_RBT_SET_FLAGS
+#  error Missing definition of ERTS_RBT_SET_FLAGS
+#endif
+#ifndef ERTS_RBT_GET_PARENT
+#  error Missing definition of ERTS_RBT_GET_PARENT
+#endif
+#ifndef ERTS_RBT_SET_PARENT
+#  error Missing definition of ERTS_RBT_SET_PARENT
+#endif
+#ifndef ERTS_RBT_GET_RIGHT
+#  error Missing definition of ERTS_RBT_GET_RIGHT
+#endif
+#ifndef ERTS_RBT_GET_LEFT
+#  error Missing definition of ERTS_RBT_GET_LEFT
+#endif
+#ifndef ERTS_RBT_IS_LT
+#  error Missing definition of ERTS_RBT_IS_LT
+#endif
+#ifndef ERTS_RBT_GET_KEY
+#  error Missing definition of ERTS_RBT_GET_KEY
+#endif
+#ifndef ERTS_RBT_IS_EQ
+#  error Missing definition of ERTS_RBT_IS_EQ
+#endif
+
+#if defined(ERTS_RBT_HARD_DEBUG) || defined(DEBUG)
+#  ifndef ERTS_RBT_DEBUG
+#    define ERTS_RBT_DEBUG 1
+#  endif
+#endif
+
+#if defined(ERTS_RBT_HARD_DEBUG) && defined(__GNUC__)
+#warning "* * * * * * * * * * * * * * * * * *"
+#warning "* ERTS_RBT_HARD_DEBUG IS ENABLED! *"
+#warning "* * * * * * * * * * * * * * * * * *"
+#endif
+
+#undef ERTS_RBT_ASSERT
+#if defined(ERTS_RBT_DEBUG)
+#define ERTS_RBT_ASSERT(E) ERTS_ASSERT(E)
+#else
+#define ERTS_RBT_ASSERT(E) ((void) 1)
+#endif
+
+#undef ERTS_RBT_API_INLINE__
+#if defined(ERTS_RBT_NO_API_INLINE) || defined(ERTS_RBT_DEBUG)
+#  define ERTS_RBT_API_INLINE__
+#else
+#  define ERTS_RBT_API_INLINE__ ERTS_INLINE
+#endif
+
+#ifndef ERTS_RBT_YIELD_STAT_INITER
+#  define ERTS_RBT_YIELD_STAT_INITER {NULL, 0}
+#endif
+
+#define ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) \
+    X ## Y
+#define ERTS_RBT_CONCAT_MACRO_VALUES__(X, Y) \
+    ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y)
+
+#undef ERTS_RBT_YIELD_STATE_T__
+#define ERTS_RBT_YIELD_STATE_T__ \
+    ERTS_RBT_CONCAT_MACRO_VALUES__(ERTS_RBT_PREFIX, _rbt_yield_state_t)
+
+typedef struct {
+    ERTS_RBT_T *x;
+    int up;
+} ERTS_RBT_YIELD_STATE_T__;
+
+#define ERTS_RBT_FUNC__(Name) \
+    ERTS_RBT_CONCAT_MACRO_VALUES__(ERTS_RBT_PREFIX, _rbt_ ## Name)
+
+#undef ERTS_RBT_NEED_REPLACE__
+#undef ERTS_RBT_NEED_INSERT__
+#undef ERTS_RBT_NEED_ROTATE__
+#undef ERTS_RBT_NEED_FOREACH_UNORDERED__
+#undef ERTS_RBT_NEED_FOREACH_ORDERED__
+#undef ERTS_RBT_NEED_HDBG_CHECK_TREE__
+#undef ERTS_RBT_HDBG_CHECK_TREE__
+
+#if defined(ERTS_RBT_WANT_REPLACE) || defined(ERTS_RBT_WANT_DELETE)
+#  define ERTS_RBT_NEED_REPLACE__
+#endif
+#if defined(ERTS_RBT_WANT_INSERT) || defined(ERTS_RBT_WANT_LOOKUP_INSERT)
+#  define ERTS_RBT_NEED_INSERT__
+#endif
+#if defined(ERTS_RBT_WANT_DELETE) || defined(ERTS_RBT_NEED_INSERT__)
+#  define ERTS_RBT_NEED_ROTATE__
+#endif
+#if defined(ERTS_RBT_WANT_FOREACH) \
+    || defined(ERTS_RBT_WANT_FOREACH_YIELDING) \
+    || defined(ERTS_RBT_WANT_FOREACH_DESTROY) \
+    || defined(ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING)
+#  define ERTS_RBT_NEED_FOREACH_UNORDERED__
+#endif
+#if defined(ERTS_RBT_WANT_FOREACH_SMALL) \
+    || defined(ERTS_RBT_WANT_FOREACH_LARGE) \
+    || defined(ERTS_RBT_WANT_FOREACH_SMALL_YIELDING) \
+    || defined(ERTS_RBT_WANT_FOREACH_LARGE_YIELDING) \
+    || defined(ERTS_RBT_WANT_FOREACH_SMALL_DESTROY) \
+    || defined(ERTS_RBT_WANT_FOREACH_LARGE_DESTROY) \
+    || defined(ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING) \
+    || defined(ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING)
+#  define ERTS_RBT_NEED_FOREACH_ORDERED__
+#endif
+#if defined(ERTS_RBT_HARD_DEBUG) \
+    && (defined(ERTS_RBT_WANT_DELETE) \
+	|| defined(ERTS_RBT_NEED_INSERT__))
+static void ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root);
+#  define ERTS_RBT_NEED_HDBG_CHECK_TREE__
+#  define ERTS_RBT_HDBG_CHECK_TREE__(R) \
+    ERTS_RBT_FUNC__(hdbg_check_tree)((R))
+#else
+#  define ERTS_RBT_HDBG_CHECK_TREE__(R) ((void) 1)
+#endif
+
+#ifdef ERTS_RBT_NEED_ROTATE__
+
+static ERTS_INLINE void
+ERTS_RBT_FUNC__(left_rotate__)(ERTS_RBT_T **root, ERTS_RBT_T *x)
+{
+    ERTS_RBT_T *y, *l, *p;
+
+    y = ERTS_RBT_GET_RIGHT(x);
+    l = ERTS_RBT_GET_LEFT(y);
+    ERTS_RBT_SET_RIGHT(x, l);
+
+    if (l)
+	ERTS_RBT_SET_PARENT(l, x);
+
+    p = ERTS_RBT_GET_PARENT(x);
+    ERTS_RBT_SET_PARENT(y, p);
+
+    if (!p) {
+	ERTS_RBT_ASSERT(*root == x);
+	*root = y;
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+	ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y);
+#endif
+    }
+    else if (x == ERTS_RBT_GET_LEFT(p))
+	ERTS_RBT_SET_LEFT(p, y);
+    else {
+	ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p));
+	ERTS_RBT_SET_RIGHT(p, y);
+    }
+    ERTS_RBT_SET_LEFT(y, x);
+    ERTS_RBT_SET_PARENT(x, y);
+
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE
+    ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(!0, x, y);
+#endif
+
+}
+
+static ERTS_INLINE void
+ERTS_RBT_FUNC__(right_rotate__)(ERTS_RBT_T **root, ERTS_RBT_T *x)
+{
+    ERTS_RBT_T *y, *r, *p;
+
+    y = ERTS_RBT_GET_LEFT(x);
+    r = ERTS_RBT_GET_RIGHT(y);
+    ERTS_RBT_SET_LEFT(x, r);
+
+    if (r)
+	ERTS_RBT_SET_PARENT(r, x);
+
+    p = ERTS_RBT_GET_PARENT(x);
+    ERTS_RBT_SET_PARENT(y, p);
+
+    if (!p) {
+	ERTS_RBT_ASSERT(*root == x);
+	*root = y;
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+	ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y);
+#endif
+    }
+    else if (x == ERTS_RBT_GET_RIGHT(p))
+	ERTS_RBT_SET_RIGHT(p, y);
+    else {
+	ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(p));
+	ERTS_RBT_SET_LEFT(p, y);
+    }
+
+    ERTS_RBT_SET_RIGHT(y, x);
+    ERTS_RBT_SET_PARENT(x, y);
+
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE
+    ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(0, x, y);
+#endif
+
+}
+
+#endif /* ERTS_RBT_NEED_ROTATE__ */
+
+#ifdef ERTS_RBT_NEED_REPLACE__
+
+/*
+ * Replace node x with node y
+ */
+static ERTS_INLINE void
+ERTS_RBT_FUNC__(replace__)(ERTS_RBT_T **root, ERTS_RBT_T *x, ERTS_RBT_T *y)
+{
+    ERTS_RBT_T *p, *r, *l;
+    ERTS_RBT_FLAGS_T f;
+
+    p = ERTS_RBT_GET_PARENT(x);
+    if (!p) {
+	ERTS_RBT_ASSERT(*root == x);
+	*root = y;
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+	ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y);
+#endif
+    }
+    else if (x == ERTS_RBT_GET_LEFT(p))
+	ERTS_RBT_SET_LEFT(p, y);
+    else {
+	ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p));
+	ERTS_RBT_SET_RIGHT(p, y);
+    }
+    l = ERTS_RBT_GET_LEFT(x);
+    if (l) {
+	ERTS_RBT_ASSERT(ERTS_RBT_GET_PARENT(l) == x);
+	ERTS_RBT_SET_PARENT(l, y);
+    }
+    r = ERTS_RBT_GET_RIGHT(x);
+    if (r) {
+	ERTS_RBT_ASSERT(ERTS_RBT_GET_PARENT(r) == x);
+	ERTS_RBT_SET_PARENT(r, y);
+    }
+
+    f = ERTS_RBT_GET_FLAGS(x);
+    ERTS_RBT_SET_FLAGS(y, f);
+    ERTS_RBT_SET_PARENT(y, p);
+    ERTS_RBT_SET_RIGHT(y, r);
+    ERTS_RBT_SET_LEFT(y, l);
+}
+
+#endif /* ERTS_RBT_NEED_REPLACE__ */
+
+#ifdef ERTS_RBT_WANT_REPLACE
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(replace)(ERTS_RBT_T **root, ERTS_RBT_T *x, ERTS_RBT_T *y)
+{
+    ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(x),
+				   ERTS_RBT_GET_KEY(y)));
+
+    ERTS_RBT_FUNC__(replace__)(root, x, y);
+}
+
+#endif /* ERTS_RBT_WANT_REPLACE */
+
+#ifdef ERTS_RBT_WANT_DELETE
+
+/*
+ * Delete a node.
+ */
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(delete)(ERTS_RBT_T **root, ERTS_RBT_T *n)
+{
+    int spliced_is_black;
+    ERTS_RBT_T *p, *x, *y, *z = n;
+    ERTS_RBT_T null_x; /* null_x is used to get the fixup started when we
+			  splice out a node without children. */
+
+    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+
+    ERTS_RBT_INIT_EMPTY_TNODE(&null_x);
+
+    /* Remove node from tree... */
+
+    /* Find node to splice out */
+    if (!ERTS_RBT_GET_LEFT(z) || !ERTS_RBT_GET_RIGHT(z))
+	y = z;
+    else {
+	/* Set y to z:s successor */
+	y = ERTS_RBT_GET_RIGHT(z);
+	while (1) {
+	    ERTS_RBT_T *t = ERTS_RBT_GET_LEFT(y);
+	    if (!t)
+		break;
+	    y = t;
+	}
+    }
+    /* splice out y */
+    x = ERTS_RBT_GET_LEFT(y);
+    if (!x)
+	x = ERTS_RBT_GET_RIGHT(y);
+    spliced_is_black = ERTS_RBT_IS_BLACK(y);
+    p = ERTS_RBT_GET_PARENT(y);
+    if (x)
+	ERTS_RBT_SET_PARENT(x, p);
+    else if (spliced_is_black) {
+	x = &null_x;
+	ERTS_RBT_SET_BLACK(x);
+	ERTS_RBT_SET_PARENT(x, p);
+	ERTS_RBT_SET_LEFT(y, x);
+    }
+
+    if (!p) {
+	ERTS_RBT_ASSERT(*root == y);
+	*root = x;
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+	ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(y, x);
+#endif
+    }
+    else {
+	if (y == ERTS_RBT_GET_LEFT(p))
+	    ERTS_RBT_SET_LEFT(p, x);
+	else {
+	    ERTS_RBT_ASSERT(y == ERTS_RBT_GET_RIGHT(p));
+	    ERTS_RBT_SET_RIGHT(p, x);
+	}
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD
+	if (p != z)
+	    ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(p, y == z ? NULL : z);
+#endif
+    }
+    if (y != z) {
+	/* We spliced out the successor of z; replace z by the successor */
+	ERTS_RBT_FUNC__(replace__)(root, z, y);
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD
+	ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(y, NULL);
+#endif
+    }
+
+    if (spliced_is_black) {
+	/* We removed a black node which makes the resulting tree
+	   violate the Red-Black Tree properties. Fixup tree... */
+
+	p = ERTS_RBT_GET_PARENT(x);
+	while (ERTS_RBT_IS_BLACK(x) && p) {
+	    ERTS_RBT_T *r, *l;
+
+	    /*
+	     * x has an "extra black" which we move up the tree
+	     * until we reach the root or until we can get rid of it.
+	     *
+	     * y is the sibbling of x, and p is their parent
+	     */
+
+	    if (x == ERTS_RBT_GET_LEFT(p)) {
+		y = ERTS_RBT_GET_RIGHT(p);
+
+		ERTS_RBT_ASSERT(y);
+
+		if (ERTS_RBT_IS_RED(y)) {
+		    ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y));
+		    ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y));
+
+		    ERTS_RBT_SET_BLACK(y);
+
+		    ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(p));
+
+		    ERTS_RBT_SET_RED(p);
+		    ERTS_RBT_FUNC__(left_rotate__)(root, p);
+		    p = ERTS_RBT_GET_PARENT(x);
+		    y = ERTS_RBT_GET_RIGHT(p);
+		}
+
+		ERTS_RBT_ASSERT(y);
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(y));
+
+		l = ERTS_RBT_GET_LEFT(y);
+		r = ERTS_RBT_GET_RIGHT(y);
+		if ((!l || ERTS_RBT_IS_BLACK(l))
+		    && (!r || ERTS_RBT_IS_BLACK(r))) {
+		    ERTS_RBT_SET_RED(y);
+		    x = p;
+		    p = ERTS_RBT_GET_PARENT(x);
+		}
+		else {
+		    if (!r || ERTS_RBT_IS_BLACK(r)) {
+			ERTS_RBT_SET_BLACK(l);
+			ERTS_RBT_SET_RED(y);
+			ERTS_RBT_FUNC__(right_rotate__)(root, y);
+			p = ERTS_RBT_GET_PARENT(x);
+			y = ERTS_RBT_GET_RIGHT(p);
+		    }
+
+		    ERTS_RBT_ASSERT(y);
+
+		    if (p && ERTS_RBT_IS_RED(p)) {
+
+			ERTS_RBT_SET_BLACK(p);
+			ERTS_RBT_SET_RED(y);
+		    }
+
+		    ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y));
+
+		    ERTS_RBT_SET_BLACK(ERTS_RBT_GET_RIGHT(y));
+		    ERTS_RBT_FUNC__(left_rotate__)(root, p);
+		    x = *root;
+		    break;
+		}
+	    }
+	    else {
+		ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p));
+
+		y = ERTS_RBT_GET_LEFT(p);
+
+		ERTS_RBT_ASSERT(y);
+
+		if (ERTS_RBT_IS_RED(y)) {
+		    ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y));
+		    ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y));
+
+		    ERTS_RBT_SET_BLACK(y);
+		    ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(p));
+		    ERTS_RBT_SET_RED(p);
+		    ERTS_RBT_FUNC__(right_rotate__)(root, p);
+
+		    p = ERTS_RBT_GET_PARENT(x);
+		    y = ERTS_RBT_GET_LEFT(p);
+		}
+
+		ERTS_RBT_ASSERT(y);
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(y));
+
+		l = ERTS_RBT_GET_LEFT(y);
+		r = ERTS_RBT_GET_RIGHT(y);
+
+		if ((!r || ERTS_RBT_IS_BLACK(r))
+		    && (!l || ERTS_RBT_IS_BLACK(l))) {
+		    ERTS_RBT_SET_RED(y);
+		    x = p;
+		    p = ERTS_RBT_GET_PARENT(x);
+		}
+		else {
+		    if (!l || ERTS_RBT_IS_BLACK(l)) {
+			ERTS_RBT_SET_BLACK(r);
+			ERTS_RBT_SET_RED(y);
+			ERTS_RBT_FUNC__(left_rotate__)(root, y);
+
+			p = ERTS_RBT_GET_PARENT(x);
+			y = ERTS_RBT_GET_LEFT(p);
+		    }
+
+		    ERTS_RBT_ASSERT(y);
+
+		    if (p && ERTS_RBT_IS_RED(p)) {
+			ERTS_RBT_SET_BLACK(p);
+			ERTS_RBT_SET_RED(y);
+		    }
+
+		    ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y));
+
+		    ERTS_RBT_SET_BLACK(ERTS_RBT_GET_LEFT(y));
+		    ERTS_RBT_FUNC__(right_rotate__)(root, p);
+		    x = *root;
+		    break;
+		}
+	    }
+	}
+
+	ERTS_RBT_SET_BLACK(x);
+
+	x = &null_x;
+	p = ERTS_RBT_GET_PARENT(x);
+
+	if (p) {
+	    if (ERTS_RBT_GET_LEFT(p) == x)
+		ERTS_RBT_SET_LEFT(p, NULL);
+	    else {
+		ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(p) == x);
+		ERTS_RBT_SET_RIGHT(p, NULL);
+	    }
+
+	    ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x));
+	    ERTS_RBT_ASSERT(!ERTS_RBT_GET_RIGHT(x));
+	}
+	else if (*root == x) {
+	    *root = NULL;
+
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+	    ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, NULL);
+#endif
+
+	    ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x));
+	    ERTS_RBT_ASSERT(!ERTS_RBT_GET_RIGHT(x));
+	}
+    }
+
+    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+
+}
+
+#endif /* ERTS_RBT_WANT_DELETE */
+
+#ifdef ERTS_RBT_NEED_INSERT__
+
+static void
+ERTS_RBT_FUNC__(insert_fixup__)(ERTS_RBT_T **root, ERTS_RBT_T *n)
+{
+    ERTS_RBT_T *x, *y;
+
+    x = n;
+
+    /*
+     * Rearrange the tree so that it satisfies the Red-Black Tree properties
+     */
+
+    ERTS_RBT_ASSERT(x != *root && ERTS_RBT_IS_RED(ERTS_RBT_GET_PARENT(x)));
+    do {
+	ERTS_RBT_T *p, *pp;
+
+	/*
+	 * x and its parent are both red. Move the red pair up the tree
+	 * until we get to the root or until we can separate them.
+	 */
+
+	p = ERTS_RBT_GET_PARENT(x);
+	pp = ERTS_RBT_GET_PARENT(p);
+
+	ERTS_RBT_ASSERT(p && pp);
+	ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x));
+	ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp));
+
+	if (p == ERTS_RBT_GET_LEFT(pp)) {
+	    y = ERTS_RBT_GET_RIGHT(pp);
+	    if (y && ERTS_RBT_IS_RED(y)) {
+		ERTS_RBT_SET_BLACK(y);
+		ERTS_RBT_SET_BLACK(p);
+		ERTS_RBT_SET_RED(pp);
+		x = pp;
+	    }
+	    else {
+
+		if (x == ERTS_RBT_GET_RIGHT(p)) {
+		    x = p;
+		    ERTS_RBT_FUNC__(left_rotate__)(root, x);
+		    p = ERTS_RBT_GET_PARENT(x);
+		    pp = ERTS_RBT_GET_PARENT(p);
+
+		    ERTS_RBT_ASSERT(p && pp);
+		}
+
+		ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(ERTS_RBT_GET_LEFT(pp)));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(p));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp));
+		ERTS_RBT_ASSERT(!y || ERTS_RBT_IS_BLACK(y));
+
+
+		ERTS_RBT_SET_BLACK(p);
+		ERTS_RBT_SET_RED(pp);
+		ERTS_RBT_FUNC__(right_rotate__)(root, pp);
+
+
+		ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(ERTS_RBT_GET_PARENT(x)) == x);
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(
+				    ERTS_RBT_GET_RIGHT(
+					ERTS_RBT_GET_PARENT(x))));
+		ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x)
+				|| ERTS_RBT_IS_BLACK(ERTS_RBT_GET_PARENT(x)));
+		break;
+	    }
+	}
+	else {
+	    ERTS_RBT_ASSERT(p == ERTS_RBT_GET_RIGHT(pp));
+
+	    y = ERTS_RBT_GET_LEFT(pp);
+	    if (y && ERTS_RBT_IS_RED(y)) {
+		ERTS_RBT_SET_BLACK(y);
+		ERTS_RBT_SET_BLACK(p);
+		ERTS_RBT_SET_RED(pp);
+		x = pp;
+	    }
+	    else {
+
+		if (x == ERTS_RBT_GET_LEFT(p)) {
+		    x = p;
+		    ERTS_RBT_FUNC__(right_rotate__)(root, x);
+		    p = ERTS_RBT_GET_PARENT(x);
+		    pp = ERTS_RBT_GET_PARENT(p);
+
+		    ERTS_RBT_ASSERT(p && pp);
+		}
+
+		ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(ERTS_RBT_GET_RIGHT(pp)));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(p));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp));
+		ERTS_RBT_ASSERT(!y || ERTS_RBT_IS_BLACK(y));
+
+
+		ERTS_RBT_SET_BLACK(p);
+		ERTS_RBT_SET_RED(pp);
+		ERTS_RBT_FUNC__(left_rotate__)(root, pp);
+
+
+		ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(ERTS_RBT_GET_PARENT(x)) == x);
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x));
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(
+				    ERTS_RBT_GET_LEFT(
+					ERTS_RBT_GET_PARENT(x))));
+		ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x)
+				|| ERTS_RBT_IS_BLACK(ERTS_RBT_GET_PARENT(x)));
+		break;
+	    }
+	}
+    } while (x != *root && ERTS_RBT_IS_RED(ERTS_RBT_GET_PARENT(x)));
+
+    ERTS_RBT_SET_BLACK(*root);
+
+}
+
+static ERTS_INLINE ERTS_RBT_T *
+ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup)
+{
+    ERTS_RBT_KEY_T kn = ERTS_RBT_GET_KEY(n);
+
+    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+
+    ERTS_RBT_INIT_EMPTY_TNODE(n);	
+
+    if (!*root) {
+	ERTS_RBT_SET_BLACK(n);
+	*root = n;
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+	ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(NULL, n);
+#endif
+    }
+    else {
+	ERTS_RBT_T *p, *x = *root;
+
+	while (1) {
+	    ERTS_RBT_KEY_T kx;
+	    ERTS_RBT_T *c;
+
+	    kx = ERTS_RBT_GET_KEY(x);
+
+	    if (lookup && ERTS_RBT_IS_EQ(kn, kx)) {
+
+		ERTS_RBT_HDBG_CHECK_TREE__(*root);
+
+		return x;
+	    }
+
+	    if (ERTS_RBT_IS_LT(kn, kx)) {
+		c = ERTS_RBT_GET_LEFT(x);
+		if (!c) {
+		    ERTS_RBT_SET_PARENT(n, x);
+		    ERTS_RBT_SET_LEFT(x, n);
+		    p = x;
+		    break;
+		}
+	    }
+	    else {
+		c = ERTS_RBT_GET_RIGHT(x);
+		if (!c) {
+		    ERTS_RBT_SET_PARENT(n, x);
+		    ERTS_RBT_SET_RIGHT(x, n);
+		    p = x;
+		    break;
+		}
+	    }
+
+	    x = c;
+	}
+
+	ERTS_RBT_ASSERT(p);
+
+	ERTS_RBT_SET_RED(n);
+	if (ERTS_RBT_IS_RED(p))
+	    ERTS_RBT_FUNC__(insert_fixup__)(root, n);
+    }
+
+    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+
+    return NULL;
+}
+
+#endif /* ERTS_RBT_NEED_INSERT__ */
+
+#ifdef ERTS_RBT_WANT_LOOKUP_INSERT
+
+static ERTS_RBT_API_INLINE__ ERTS_RBT_T *
+ERTS_RBT_FUNC__(lookup_insert)(ERTS_RBT_T **root, ERTS_RBT_T *n)
+{
+    return ERTS_RBT_FUNC__(insert_aux__)(root, n, !0);
+}
+
+#endif /* ERTS_RBT_WANT_LOOKUP_INSERT */
+
+#ifdef ERTS_RBT_WANT_INSERT
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(insert)(ERTS_RBT_T **root, ERTS_RBT_T *n)
+{
+    (void) ERTS_RBT_FUNC__(insert_aux__)(root, n, 0);
+}
+
+#endif /* ERTS_RBT_WANT_INSERT */
+
+#ifdef ERTS_RBT_WANT_LOOKUP
+
+static ERTS_RBT_API_INLINE__ ERTS_RBT_T *
+ERTS_RBT_FUNC__(lookup)(ERTS_RBT_T *root, ERTS_RBT_KEY_T key)
+{
+    ERTS_RBT_T *x = root;
+
+    if (!x)
+	return NULL;
+
+    while (1) {
+	ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x);
+	ERTS_RBT_T *c;
+
+	if (ERTS_RBT_IS_EQ(key, kx))
+	    return x;
+
+	if (ERTS_RBT_IS_LT(key, kx)) {
+	    c = ERTS_RBT_GET_LEFT(x);
+	    if (!c)
+		return NULL;
+	}
+	else {
+	    c = ERTS_RBT_GET_RIGHT(x);
+	    if (!c)
+		return NULL;
+	}
+
+	x = c;
+    }
+}
+
+#endif /* ERTS_RBT_WANT_LOOKUP */
+
+#ifdef ERTS_RBT_WANT_SMALLEST
+
+static ERTS_RBT_API_INLINE__ ERTS_RBT_T *
+ERTS_RBT_FUNC__(smallest)(ERTS_RBT_T *root)
+{
+    ERTS_RBT_T *x = root;
+
+    if (!x)
+	return NULL;
+
+    while (1) {
+	ERTS_RBT_T *c = ERTS_RBT_GET_LEFT(x);
+	if (!c)
+	    break;
+	x = c;
+    }
+
+    return x;
+}
+
+#endif /* ERTS_RBT_WANT_SMALLEST */
+
+#ifdef ERTS_RBT_WANT_LARGEST
+
+static ERTS_RBT_API_INLINE__ ERTS_RBT_T *
+ERTS_RBT_FUNC__(largest)(ERTS_RBT_T *root)
+{
+    ERTS_RBT_T *x = root;
+
+    if (!x)
+	return NULL;
+
+    while (1) {
+	ERTS_RBT_T *c = ERTS_RBT_GET_RIGHT(x);
+	if (!c)
+	    break;
+	x = c;
+    }
+
+    return x;
+}
+
+#endif /* ERTS_RBT_WANT_LARGEST */
+
+#ifdef ERTS_RBT_NEED_FOREACH_UNORDERED__
+
+static ERTS_INLINE int
+ERTS_RBT_FUNC__(foreach_unordered__)(ERTS_RBT_T **root,
+				     int destroying,
+				     void (*op)(ERTS_RBT_T *, void *),
+				     void *arg,
+				     int yielding,
+				     ERTS_RBT_YIELD_STATE_T__ *ystate,
+				     Sint ylimit)
+{
+    ERTS_RBT_T *c, *p, *x;
+
+    ERTS_RBT_ASSERT(!yielding || ystate);
+
+    if (yielding && ystate->x) {
+	x = ystate->x;
+	ERTS_RBT_ASSERT(ystate->up);
+	goto restart_up;
+    }
+    else {
+	x = *root;
+	if (!x)
+	    return 0;
+	if (destroying)
+	    *root = NULL;
+    }
+
+    while (1) {
+
+	while (1) {
+
+	    while (1) {
+		c = ERTS_RBT_GET_LEFT(x);
+		if (!c)
+		    break;
+		x = c;
+	    }
+
+	    c = ERTS_RBT_GET_RIGHT(x);
+	    if (!c)
+		break;
+	    x = c;
+	}
+
+	while (1) {
+#ifdef ERTS_RBT_DEBUG
+	    int cdir;
+#endif
+	    if (yielding && ylimit-- <= 0) {
+		ystate->x = x;
+		ystate->up = 1;
+		return 1;
+	    }
+
+	restart_up:
+
+	    p = ERTS_RBT_GET_PARENT(x);
+
+#ifdef ERTS_RBT_DEBUG
+	    ERTS_RBT_ASSERT(!destroying || !ERTS_RBT_GET_LEFT(x));
+	    ERTS_RBT_ASSERT(!destroying || !ERTS_RBT_GET_RIGHT(x));
+
+	    if (p) {
+		if (x == ERTS_RBT_GET_LEFT(p)) {
+		    cdir = -1;
+		    if (destroying)
+			ERTS_RBT_SET_LEFT(p, NULL);
+		}
+		else {
+		    ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p));
+		    cdir = 1;
+		    if (destroying)
+			ERTS_RBT_SET_RIGHT(p, NULL);
+		}
+	    }
+#endif
+
+	    (*op)(x, arg);
+
+	    if (!p) {
+		if (yielding) {
+		    ystate->x = NULL;
+		    ystate->up = 0;
+		}
+		return 0; /* Done */
+	    }
+
+	    c = ERTS_RBT_GET_RIGHT(p);
+	    if (c && c != x) {
+		ERTS_RBT_ASSERT(cdir < 0);
+
+		/* Go down tree of x's sibling... */
+		x = c;
+		break;
+	    }
+
+	    x = p;
+	}
+    }
+}
+
+#endif /* ERTS_RBT_NEED_FOREACH_UNORDERED__ */
+
+#ifdef ERTS_RBT_NEED_FOREACH_ORDERED__
+
+static ERTS_INLINE int
+ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root,
+				   int from_small,
+				   int destroying,
+				   void (*op)(ERTS_RBT_T *, void *),
+				   void (*destroy)(ERTS_RBT_T *, void *),
+				   void *arg,
+				   int yielding,
+				   ERTS_RBT_YIELD_STATE_T__ *ystate,
+				   Sint ylimit)
+{
+    ERTS_RBT_T *c, *p, *x;
+
+    ERTS_RBT_ASSERT(!yielding || ystate);
+    ERTS_RBT_ASSERT(!destroying || destroy);
+
+    if (yielding && ystate->x) {
+	x = ystate->x;
+	if (ystate->up)
+	    goto restart_up;
+	else
+	    goto restart_down;
+    }
+    else {
+	x = *root;
+	if (!x)
+	    return 0;
+	if (destroying)
+	    *root = NULL;
+    }
+
+    while (1) {
+
+	while (1) {
+
+	    while (1) {
+		c = from_small ? ERTS_RBT_GET_LEFT(x) : ERTS_RBT_GET_RIGHT(x);
+		if (!c)
+		    break;
+		x = c;
+	    }
+
+	    (*op)(x, arg);
+
+	    if (yielding && --ylimit <= 0) {
+		ystate->x = x;
+		ystate->up = 0;
+		return 1;
+	    }
+
+	restart_down:
+
+	    c = from_small ? ERTS_RBT_GET_RIGHT(x) : ERTS_RBT_GET_LEFT(x);
+	    if (!c)
+		break;
+	    x = c;
+	}
+
+	while (1) {
+	    p = ERTS_RBT_GET_PARENT(x);
+
+	    if (p) {
+
+		c = from_small ? ERTS_RBT_GET_RIGHT(p) : ERTS_RBT_GET_LEFT(p);
+		if (!c || c != x) {
+		    ERTS_RBT_ASSERT((from_small
+				     ? ERTS_RBT_GET_LEFT(p)
+				     : ERTS_RBT_GET_RIGHT(p)) == x);
+
+		    (*op)(p, arg);
+
+		    if (yielding && --ylimit <= 0) {
+			ystate->x = x;
+			ystate->up = 1;
+			return 1;
+		    restart_up:
+			p = ERTS_RBT_GET_PARENT(x);
+		    }
+		}
+
+		if (c && c != x) {
+		    ERTS_RBT_ASSERT((from_small
+				     ? ERTS_RBT_GET_LEFT(p)
+				     : ERTS_RBT_GET_RIGHT(p)) == x);
+
+		    /* Go down tree of x's sibling... */
+		    x = c;
+		    break;
+		}
+	    }
+
+	    if (destroying) {
+
+#ifdef ERTS_RBT_DEBUG
+		ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x)
+				&& !ERTS_RBT_GET_RIGHT(x));
+
+		if (p) {
+		    if (x == ERTS_RBT_GET_LEFT(p))
+			ERTS_RBT_SET_LEFT(p, NULL);
+		    else {
+			ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p));
+			ERTS_RBT_SET_RIGHT(p, NULL);
+		    }
+		}
+#endif
+
+		(*destroy)(x, arg);
+	    }
+
+	    if (!p) {
+		if (yielding) {
+		    ystate->x = NULL;
+		    ystate->up = 0;
+		}
+		return 1; /* Done */
+	    }
+	    x = p;
+	}
+    }
+}
+
+#endif /* ERTS_RBT_NEED_FOREACH_ORDERED__ */
+
+#ifdef ERTS_RBT_WANT_FOREACH
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(foreach)(ERTS_RBT_T *root,
+			 void (*op)(ERTS_RBT_T *, void *),
+			 void *arg)
+{
+    (void) ERTS_RBT_FUNC__(foreach_unordered__)(&root, 0, op, arg,
+						0, NULL, 0);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH */
+
+#ifdef ERTS_RBT_WANT_FOREACH_SMALL
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(foreach_small)(ERTS_RBT_T *root,
+			       void (*op)(ERTS_RBT_T *, void *),
+			       void *arg)
+{
+    (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0,
+					      op, NULL, arg,
+					      0, NULL, 0);
+}
+
+#endif /*  ERTS_RBT_WANT_FOREACH_SMALL */
+
+#ifdef ERTS_RBT_WANT_FOREACH_LARGE
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(foreach_large)(ERTS_RBT_T *root,
+			       void (*op)(ERTS_RBT_T *, void *),
+			       void *arg)
+{
+    (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0,
+					      op, NULL, arg,
+					      0, NULL, 0);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_LARGE */
+
+#ifdef ERTS_RBT_WANT_FOREACH_YIELDING
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(foreach_yielding)(ERTS_RBT_T *root,
+				  void (*op)(ERTS_RBT_T *, void *),
+				  void *arg,
+				  ERTS_RBT_YIELD_STATE_T__ *ystate,
+				  Sint ylimit)
+{
+    (void) ERTS_RBT_FUNC__(foreach_unordered__)(*root, 0, op, arg,
+						1, ystate, ylimit);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_YIELDING */
+
+#ifdef ERTS_RBT_WANT_FOREACH_SMALL_YIELDING
+
+static ERTS_RBT_API_INLINE__ int
+ERTS_RBT_FUNC__(foreach_small_yielding)(ERTS_RBT_T *root,
+					void (*op)(ERTS_RBT_T *, void *),
+					void *arg,
+					ERTS_RBT_YIELD_STATE_T__ *ystate,
+					Sint ylimit)
+{
+    return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0,
+					      op, NULL, arg,
+					      1, ystate, ylimit);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_SMALL_YIELDING */
+
+#ifdef ERTS_RBT_WANT_FOREACH_LARGE_YIELDING
+
+static ERTS_RBT_API_INLINE__ int
+ERTS_RBT_FUNC__(foreach_large_yielding)(ERTS_RBT_T *root,
+					void (*op)(ERTS_RBT_T *, void *),
+					void *arg,
+					ERTS_RBT_YIELD_STATE_T__ *ystate,
+					Sint ylimit)
+{
+    return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0,
+					      op, NULL, arg,
+					      1, ystate, ylimit);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_LARGE_YIELDING */
+
+#ifdef ERTS_RBT_WANT_FOREACH_DESTROY
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(foreach_destroy)(ERTS_RBT_T **root,
+				 void (*op)(ERTS_RBT_T *, void *),
+				 void *arg)
+{
+    (void) ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg,
+						0, NULL, 0);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_DESTROY */
+
+#ifdef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(foreach_small_destroy)(ERTS_RBT_T **root,
+				       void (*op)(ERTS_RBT_T *, void *),
+				       void (*destr)(ERTS_RBT_T *, void *),
+				       void *arg)
+{
+    (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1,
+					      op, destr, arg,
+					      0, NULL, 0);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_SMALL_DESTROY */
+
+#ifdef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY
+
+static ERTS_RBT_API_INLINE__ void
+ERTS_RBT_FUNC__(foreach_large_destroy)(ERTS_RBT_T **root,
+				       void (*op)(ERTS_RBT_T *, void *),
+				       void (*destr)(ERTS_RBT_T *, void *),
+				       void *arg)
+{
+    (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1,
+					      op, destr, arg,
+					      0, NULL, 0);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_LARGE_DESTROY */
+
+#ifdef ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+
+static ERTS_RBT_API_INLINE__ int
+ERTS_RBT_FUNC__(foreach_destroy_yielding)(ERTS_RBT_T **root,
+					  void (*op)(ERTS_RBT_T *, void *),
+					  void *arg,
+					  ERTS_RBT_YIELD_STATE_T__ *ystate,
+					  Sint ylimit)
+{
+    return ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg,
+						1, ystate, ylimit);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING */
+
+#ifdef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING
+
+static ERTS_RBT_API_INLINE__ int
+ERTS_RBT_FUNC__(foreach_small_destroy_yielding)(ERTS_RBT_T **root,
+						void (*op)(ERTS_RBT_T *, void *),
+						void (*destr)(ERTS_RBT_T *, void *),
+						void *arg,
+						ERTS_RBT_YIELD_STATE_T__ *ystate,
+						Sint ylimit)
+{
+    return ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1,
+					      op, destr, arg,
+					      1, ystate, ylimit);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING */
+
+#ifdef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING
+
+static ERTS_RBT_API_INLINE__ int
+ERTS_RBT_FUNC__(foreach_large_destroy_yielding)(ERTS_RBT_T **root,
+						void (*op)(ERTS_RBT_T *, void *),
+						void (*destr)(ERTS_RBT_T *, void *),
+						void *arg,
+						ERTS_RBT_YIELD_STATE_T__ *ystate,
+						Sint ylimit)
+{
+    return ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1,
+					      op, destr, arg,
+					      1, ystate, ylimit);
+}
+
+#endif /* ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING */
+
+#ifdef ERTS_RBT_WANT_DEBUG_PRINT
+
+static void
+ERTS_RBT_FUNC__(debug_print)(FILE *filep, ERTS_RBT_T *x, int indent,
+			     void (*print_node)(ERTS_RBT_T *))
+{
+    if (x) {
+	ERTS_RBT_FUNC__(debug_print)(filep, ERTS_RBT_GET_RIGHT(x),
+				     indent+2, print_node);
+	erts_fprintf(filep,
+		     "%*s[%s:%p:",
+		     indent, "",
+		     ERTS_RBT_IS_BLACK(x) ? "Black" : "Red",
+		     x);
+	(*print_node)(x);
+	erts_fprintf(filep, "]\n");
+	ERTS_RBT_FUNC__(debug_print)(filep, ERTS_RBT_GET_LEFT(x),
+				     indent+2, print_node);
+    }
+}
+
+#endif /* ERTS_RBT_WANT_DEBUG_PRINT */
+
+#ifdef ERTS_RBT_NEED_HDBG_CHECK_TREE__
+
+static void
+ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root)
+{
+    int black_depth = -1, no_black = 0;
+    ERTS_RBT_T *c, *p, *x = root;
+    ERTS_RBT_KEY_T kx;
+    ERTS_RBT_KEY_T kc;
+
+    if (!x)
+	return;
+
+    ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x));
+
+    while (1) {
+
+	while (1) {
+
+	    while (1) {
+
+		if (ERTS_RBT_IS_BLACK(x))
+		    no_black++;
+		else {
+		    c = ERTS_RBT_GET_RIGHT(x);
+		    ERTS_RBT_ASSERT(!c || ERTS_RBT_IS_BLACK(c));
+		    c = ERTS_RBT_GET_LEFT(x);
+		    ERTS_RBT_ASSERT(!c || ERTS_RBT_IS_BLACK(c));
+		}
+
+		c = ERTS_RBT_GET_LEFT(x);
+		if (!c)
+		    break;
+
+		ERTS_RBT_ASSERT(x == ERTS_RBT_GET_PARENT(c));
+
+		kx = ERTS_RBT_GET_KEY(x);
+		kc = ERTS_RBT_GET_KEY(c);
+
+		ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kc, kx)
+				|| ERTS_RBT_IS_EQ(kc, kx));
+
+		x = c;
+	    }
+
+	    c = ERTS_RBT_GET_RIGHT(x);
+	    if (!c) {
+		if (black_depth < 0)
+		    black_depth = no_black;
+		ERTS_RBT_ASSERT(black_depth == no_black);
+		break;
+	    }
+
+	    ERTS_RBT_ASSERT(x == ERTS_RBT_GET_PARENT(c));
+
+	    kx = ERTS_RBT_GET_KEY(x);
+	    kc = ERTS_RBT_GET_KEY(c);
+
+	    ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc)
+			    || ERTS_RBT_IS_EQ(kx, kc));
+	    x = c;
+	}
+
+	while (1) {
+	    p = ERTS_RBT_GET_PARENT(x);
+
+	    if (ERTS_RBT_IS_BLACK(x))
+		no_black--;
+
+	    if (p) {
+
+		ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(p)
+				|| x == ERTS_RBT_GET_RIGHT(p));
+
+		c = ERTS_RBT_GET_RIGHT(p);
+		if (c && c != x) {
+		    ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(p) == x);
+
+		    kx = ERTS_RBT_GET_KEY(x);
+		    kc = ERTS_RBT_GET_KEY(c);
+
+		    ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc)
+				    || ERTS_RBT_IS_EQ(kx, kc));
+		    /* Go down tree of x's sibling... */
+		    x = c;
+		    break;
+		}
+	    }
+
+	    if (!p) {
+		ERTS_RBT_ASSERT(root == x);
+		ERTS_RBT_ASSERT(no_black == 0);
+		return; /* Done */
+	    }
+
+	    x = p;
+	}
+    }
+}
+
+#undef ERTS_RBT_PRINT_TREE__
+
+#endif /* ERTS_RBT_NEED_HDBG_CHECK_TREE__ */
+
+#undef ERTS_RBT_ASSERT
+#undef ERTS_RBT_DEBUG
+#undef ERTS_RBT_API_INLINE__
+#undef ERTS_RBT_YIELD_STATE_T__
+#undef ERTS_RBT_NEED_REPLACE__
+#undef ERTS_RBT_NEED_INSERT__
+#undef ERTS_RBT_NEED_ROTATE__
+#undef ERTS_RBT_NEED_FOREACH_UNORDERED__
+#undef ERTS_RBT_NEED_FOREACH_ORDERED__
+#undef ERTS_RBT_NEED_HDBG_CHECK_TREE__
+#undef ERTS_RBT_HDBG_CHECK_TREE__
+
+#ifdef ERTS_RBT_UNDEF
+#  undef ERTS_RBT_PREFIX
+#  undef ERTS_RBT_T
+#  undef ERTS_RBT_KEY_T
+#  undef ERTS_RBT_FLAGS_T
+#  undef ERTS_RBT_INIT_EMPTY_TNODE
+#  undef ERTS_RBT_IS_RED
+#  undef ERTS_RBT_SET_RED
+#  undef ERTS_RBT_IS_BLACK
+#  undef ERTS_RBT_SET_BLACK
+#  undef ERTS_RBT_GET_FLAGS
+#  undef ERTS_RBT_SET_FLAGS
+#  undef ERTS_RBT_GET_PARENT
+#  undef ERTS_RBT_SET_PARENT
+#  undef ERTS_RBT_GET_RIGHT
+#  undef ERTS_RBT_SET_RIGHT
+#  undef ERTS_RBT_GET_LEFT
+#  undef ERTS_RBT_SET_LEFT
+#  undef ERTS_RBT_GET_KEY
+#  undef ERTS_RBT_IS_LT
+#  undef ERTS_RBT_IS_EQ
+#  undef ERTS_RBT_UNDEF
+#  undef ERTS_RBT_NO_API_INLINE
+#  undef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE
+#  undef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD
+#  undef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+#  undef ERTS_RBT_WANT_DELETE
+#  undef ERTS_RBT_WANT_INSERT
+#  undef ERTS_RBT_WANT_LOOKUP_INSERT
+#  undef ERTS_RBT_WANT_REPLACE
+#  undef ERTS_RBT_WANT_LOOKUP
+#  undef ERTS_RBT_WANT_SMALLEST
+#  undef ERTS_RBT_WANT_LARGEST
+#  undef ERTS_RBT_WANT_FOREACH
+#  undef ERTS_RBT_WANT_FOREACH_DESTROY
+#  undef ERTS_RBT_WANT_FOREACH_YIELDING
+#  undef ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+#  undef ERTS_RBT_WANT_FOREACH_SMALL
+#  undef ERTS_RBT_WANT_FOREACH_LARGE
+#  undef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY
+#  undef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY
+#  undef ERTS_RBT_WANT_FOREACH_SMALL_YIELDING
+#  undef ERTS_RBT_WANT_FOREACH_LARGE_YIELDING
+#  undef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING
+#  undef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING
+#  undef ERTS_RBT_WANT_DEBUG_PRINT
+#endif
-- 
cgit v1.2.3


From dc177967363d16c155042dbaa2c911f59f0efd55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 6 May 2015 19:29:34 +0200
Subject: erts: ETS ordered_set cannot use it's optimization with Maps

The optimization cannot be used due to that the pattern cannot be ordered.
---
 erts/emulator/beam/erl_db_tree.c |  2 +-
 erts/emulator/beam/erl_db_util.c | 31 +++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_db_util.h |  1 +
 3 files changed, 33 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index d90af46659..c7bccc78c3 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -2716,7 +2716,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 	*ret = this;
 	return 1;
     } else if (partly_bound != NULL && key != am_Underscore && 
-	       db_is_variable(key) < 0)
+	       db_is_variable(key) < 0 && !db_has_map(key))
 	*partly_bound = key;
 	
     return 0;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 0fb1c397c9..be988f0621 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3347,6 +3347,37 @@ int db_is_variable(Eterm obj)
     return N;
 }
 
+/* check if node is (or contains) a map
+ * return 1 if node contains a map
+ * return 0 otherwise
+ */
+
+int db_has_map(Eterm node) {
+    DECLARE_ESTACK(s);
+
+    ESTACK_PUSH(s,node);
+    while (!ESTACK_ISEMPTY(s)) {
+	node = ESTACK_POP(s);
+        if (is_list(node)) {
+	    while (is_list(node)) {
+		ESTACK_PUSH(s,CAR(list_val(node)));
+		node = CDR(list_val(node));
+	    }
+	    ESTACK_PUSH(s,node);    /* Non wellformed list or [] */
+        } else if (is_tuple(node)) {
+            Eterm *tuple = tuple_val(node);
+            int arity = arityval(*tuple);
+            while(arity--) {
+                ESTACK_PUSH(s,*(++tuple));
+            }
+        } else if is_map(node) {
+            DESTROY_ESTACK(s);
+            return 1;
+        }
+    }
+    DESTROY_ESTACK(s);
+    return 0;
+}
 
 /* check if obj is (or contains) a variable */
 /* return 1 if obj contains a variable or underscore */
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index ca206c7f58..b2d5a306cb 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -342,6 +342,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
 void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
 Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj,
 			       Uint pos, Eterm** hpp, Uint extra);
+int db_has_map(Eterm obj);
 int db_has_variable(Eterm obj);
 int db_is_variable(Eterm obj);
 void db_do_update_element(DbUpdateHandle* handle,
-- 
cgit v1.2.3


From 757d493cff18d0b52aeb1e32d01886551e16fd5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 7 May 2015 10:58:10 +0200
Subject: erts: Fix copy shallow for large Maps

There is no need to take special care of Maps at all since header_arity(hdr)
will take care of the normal case via its Map handling.
---
 erts/emulator/beam/copy.c | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 4d12dae787..850606dd86 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -608,11 +608,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    erts_refc_inc(&funp->fe->refc, 2);
 		}
 		goto off_heap_common;
-
-	    case MAP_SUBTAG:
-		*hp++ = *tp++;
-		sz--;
-		break;
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
@@ -648,7 +643,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	}
     }
     *hpp = hp;
-
     return res;
 }
 
-- 
cgit v1.2.3


From beb883ed443dfeb41f239363f36c511dd43c7eec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 7 May 2015 11:58:23 +0200
Subject: erts: Fix ETS db_has_variable check for large Maps

---
 erts/emulator/beam/erl_db_util.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index be988f0621..a75267ca97 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3411,6 +3411,11 @@ int db_has_variable(Eterm node) {
                 while (size--) {
                     ESTACK_PUSH(s, *(values++));
                 }
+            } else if (is_map(node)) { /* other map-nodes or map-heads */
+                Eterm *ptr = hashmap_val(node);
+                int i = hashmap_bitcount(MAP_HEADER_VAL(*ptr));
+                ptr += MAP_HEADER_ARITY(*ptr);
+                while(i--) { ESTACK_PUSH(s, *++ptr); }
             }
 	    break;
 	case TAG_PRIMARY_IMMED1:
-- 
cgit v1.2.3


From 39ce03094edb3bc53ad671636355be53564a222d Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Tue, 14 Apr 2015 15:16:25 +0200
Subject: Gracefully handle empty md5 field in module_info

---
 erts/emulator/beam/beam_load.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 282aa71109..29d4dce3e2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5695,7 +5695,11 @@ md5_of_module(Process* p, /* Process whose heap to use. */
 	return THE_NON_VALUE;
     }
     code = modp->curr.code;
-    res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE);
+    if (code[MI_MD5_PTR] != 0) {
+      res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE);
+    } else {
+      res = am_undefined;
+    }
     return res;
 }
 
-- 
cgit v1.2.3


From fc1029c74093fde14241ca664c9341fcc8be5fd3 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Thu, 16 Apr 2015 14:30:55 +0200
Subject: Add module_info entry for native code

---
 erts/emulator/beam/beam_bif_load.c | 29 ++-------------------
 erts/emulator/beam/beam_load.c     | 53 ++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/beam_load.h     |  2 +-
 3 files changed, 56 insertions(+), 28 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index df1983a83d..8c32fc7854 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -39,12 +39,9 @@ static void set_default_trace_pattern(Eterm module);
 static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp);
 static void delete_code(Module* modp);
 static void decrement_refc(BeamInstr* code);
-static int is_native(BeamInstr* code);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 
-
-
 BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
 {
     Module* modp;
@@ -59,8 +56,8 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
 	return am_undefined;
     }
     erts_rlock_old_code(code_ix);
-    res = ((modp->curr.code && is_native(modp->curr.code)) ||
-	    (modp->old.code != 0 && is_native(modp->old.code))) ?
+    res = (erts_is_module_native(modp->curr.code) ||
+           erts_is_module_native(modp->old.code)) ?
 		am_true : am_false;
     erts_runlock_old_code(code_ix);
     return res;
@@ -1106,25 +1103,3 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
     }
     return NIL;
 }
-
-static int
-is_native(BeamInstr* code)
-{
-    Uint i, num_functions = code[MI_NUM_FUNCTIONS];
-
-    /* Check NativeAdress of first real function in module
-     */
-    for (i=0; i<num_functions; i++) {
-	BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
-	Eterm name = (Eterm) func_info[3];
-
-	if (is_atom(name)) {
-	    return func_info[1] != 0;    
-	}
-	else ASSERT(is_nil(name)); /* ignore BIF stubs */
-    }
-    /* Not a single non-BIF function? */
-    return 0;
-}
-
-
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 29d4dce3e2..e2bd30bd6b 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -530,6 +530,7 @@ static Eterm functions_in_module(Process* p, Eterm mod);
 static Eterm attributes_for_module(Process* p, Eterm mod);
 static Eterm compilation_info_for_module(Process* p, Eterm mod);
 static Eterm md5_of_module(Process* p, Eterm mod);
+static Eterm has_native(Process* p, Eterm mod);
 static Eterm native_addresses(Process* p, Eterm mod);
 int patch_funentries(Eterm Patchlist);
 int patch(Eterm Addresses, Uint fe);
@@ -5408,6 +5409,9 @@ erts_module_info_0(Process* p, Eterm module)
     list = CONS(hp, tup, list)
 
     BUILD_INFO(am_md5);
+#ifdef HIPE
+    BUILD_INFO(am_native);
+#endif
     BUILD_INFO(am_compile);
     BUILD_INFO(am_attributes);
     BUILD_INFO(am_exports);
@@ -5433,6 +5437,8 @@ erts_module_info_1(Process* p, Eterm module, Eterm what)
 	return compilation_info_for_module(p, module);
     } else if (what == am_native_addresses) {
 	return native_addresses(p, module);
+    } else if (what == am_native) {
+	return has_native(p, module);
     }
     return THE_NON_VALUE;
 }
@@ -5492,6 +5498,53 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     return result;
 }
 
+/*
+ * Returns 'true' if mod has any native compiled functions, otherwise 'false'
+ */
+
+static Eterm
+has_native(Process* p, Eterm mod)
+{
+    Eterm result = am_false;
+#ifdef HIPE
+    Module* modp;
+
+    if (is_not_atom(mod)) {
+        return THE_NON_VALUE;
+    }
+
+    modp = erts_get_module(mod, erts_active_code_ix());
+    if (modp == NULL) {
+        return THE_NON_VALUE;
+    }
+
+    if (erts_is_module_native(modp->curr.code)) {
+      result = am_true;
+    }
+#endif
+    return result;
+}
+
+int
+erts_is_module_native(BeamInstr* code)
+{
+    Uint i, num_functions;
+
+    /* Check NativeAdress of first real function in module */
+    if (code != NULL) {
+        num_functions = code[MI_NUM_FUNCTIONS];
+        for (i=0; i<num_functions; i++) {
+            BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+            Eterm name = (Eterm) func_info[3];
+            if (is_atom(name)) {
+                return func_info[1] != 0;
+            }
+            else ASSERT(is_nil(name)); /* ignore BIF stubs */
+        }
+    }
+    return 0;
+}
+
 /*
  * Builds a list of all functions including native addresses.
  *     [{Name,Arity,NativeAddress},...]
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 0e3ca0bdb0..46b0c60ab0 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -23,10 +23,10 @@
 #include "beam_opcodes.h"
 #include "erl_process.h"
 
+int erts_is_module_native(BeamInstr* code);
 Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks,
 			    Eterm module);
 
-
 typedef struct gen_op_entry {
    char* name;
    int arity;
-- 
cgit v1.2.3


From e47490f83e561a45cee9e8f72f1e1f91f19c60b7 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Thu, 16 Apr 2015 11:28:22 +0200
Subject: Set module_info md5 for native modules properly

Use the md5 of the native code chunk instead of the Beam code md5.
---
 erts/emulator/beam/beam_load.c | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index e2bd30bd6b..0d40201934 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6225,6 +6225,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     LoaderState* stp;
     BeamInstr Funcs;
     BeamInstr Patchlist;
+    Eterm MD5Bin;
     Eterm* tp;
     BeamInstr* code = NULL;
     BeamInstr* ptrs;
@@ -6253,12 +6254,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	goto error;
     }
     tp = tuple_val(Info);
-    if (tp[0] != make_arityval(2)) {
+    if (tp[0] != make_arityval(3)) {
       goto error;
     }
     Funcs = tp[1];
-    Patchlist = tp[2];        
-   
+    Patchlist = tp[2];
+    MD5Bin = tp[3];
+    if (is_not_binary(MD5Bin) || (binary_size(MD5Bin) != MD5_SIZE)) {
+	goto error;
+    }
     if ((n = erts_list_length(Funcs)) < 0) {
 	goto error;
     }
@@ -6308,6 +6312,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr);
     code_size += stp->chunks[ATTR_CHUNK].size;
     code_size += stp->chunks[COMPILE_CHUNK].size;
+    code_size += MD5_SIZE;
     code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size);
     if (!code) {
 	goto error;
@@ -6414,6 +6419,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     if (info == NULL) {
 	goto error;
     }
+    {
+      byte *tmp = NULL;
+      byte *md5 = NULL;
+      if ((md5 = erts_get_aligned_binary_bytes(MD5Bin, &tmp)) != NULL) {
+        sys_memcpy(info, md5, MD5_SIZE);
+        code[MI_MD5_PTR] = (BeamInstr) info;
+      }
+      erts_free_aligned_binary_bytes(tmp);
+    }
 
     /*
      * Insert the module in the module table.
-- 
cgit v1.2.3


From b51bbf8b8a06e7bf18e0b837f50dad4a66a0fca7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 8 May 2015 16:13:09 +0200
Subject: erts: Make hashmap_get halfword safe

---
 erts/emulator/beam/erl_db_util.c | 2 +-
 erts/emulator/beam/erl_map.c     | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index a75267ca97..c6c3c55a7e 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2039,7 +2039,7 @@ restart:
             break;
         case matchKey:
             t = (Eterm) *pc++;
-            tp = erts_maps_get_rel(t, make_flatmap_rel(ep, base), base);
+            tp = erts_maps_get_rel(t, make_boxed_rel(ep, base), base);
             if (!tp) {
                 FAIL();
             }
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index bb2a2bcdf9..57f06c369c 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1884,7 +1884,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
     UseTmpHeapNoproc(2);
 
     ASSERT(is_boxed(node));
-    ptr = boxed_val(node);
+    ptr = boxed_val_rel(node, map_base);
     hdr = *ptr;
     ASSERT(is_header(hdr));
     ASSERT(is_hashmap_header_head(hdr));
@@ -1905,8 +1905,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
         node  = ptr[ix+1];
 
         if (is_list(node)) { /* LEAF NODE [K|V] */
-            ptr = list_val(node);
-
+            ptr = list_val_rel(node,map_base);
             res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL;
             break;
         }
@@ -1914,7 +1913,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
         hx = hashmap_shift_hash(th,hx,lvl,key);
 
         ASSERT(is_boxed(node));
-        ptr = boxed_val(node);
+        ptr = boxed_val_rel(node, map_base);
         hdr = *ptr;
         ASSERT(is_header(hdr));
         ASSERT(!is_hashmap_header_head(hdr));
-- 
cgit v1.2.3


From 9c78f149517dc02457d4c59e90bc9b03d411e28c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 5 May 2015 20:05:00 +0200
Subject: Optimized timer implementation

---
 erts/emulator/beam/atom.names         |    3 +
 erts/emulator/beam/beam_bif_load.c    |    4 +-
 erts/emulator/beam/beam_emu.c         |   60 +-
 erts/emulator/beam/bif.c              |   25 +-
 erts/emulator/beam/bif.h              |    1 +
 erts/emulator/beam/bif.tab            |   14 +-
 erts/emulator/beam/break.c            |   12 +-
 erts/emulator/beam/code_ix.c          |    4 +-
 erts/emulator/beam/erl_alloc.c        |   15 +-
 erts/emulator/beam/erl_alloc.types    |   19 +-
 erts/emulator/beam/erl_bif_info.c     |    8 +-
 erts/emulator/beam/erl_bif_timer.c    |  853 ----------
 erts/emulator/beam/erl_bif_timer.h    |   37 -
 erts/emulator/beam/erl_bif_trace.c    |    4 +-
 erts/emulator/beam/erl_db.c           |    8 +-
 erts/emulator/beam/erl_gc.c           |   56 +-
 erts/emulator/beam/erl_hl_timer.c     | 2901 +++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_hl_timer.h     |   80 +
 erts/emulator/beam/erl_init.c         |   26 +-
 erts/emulator/beam/erl_lock_check.c   |    1 +
 erts/emulator/beam/erl_message.c      |    2 +-
 erts/emulator/beam/erl_message.h      |   14 +-
 erts/emulator/beam/erl_nif.c          |    4 +-
 erts/emulator/beam/erl_node_tables.c  |    2 +-
 erts/emulator/beam/erl_port.h         |   28 +-
 erts/emulator/beam/erl_port_task.c    |   19 +-
 erts/emulator/beam/erl_process.c      |  517 +++---
 erts/emulator/beam/erl_process.h      |   91 +-
 erts/emulator/beam/erl_process_lock.c |  144 +-
 erts/emulator/beam/erl_process_lock.h |   92 +-
 erts/emulator/beam/erl_ptab.c         |   19 +-
 erts/emulator/beam/erl_ptab.h         |  102 +-
 erts/emulator/beam/erl_thr_progress.c |    5 +-
 erts/emulator/beam/erl_time.h         |  245 ++-
 erts/emulator/beam/erl_time_sup.c     |  283 ++--
 erts/emulator/beam/global.h           |    6 +-
 erts/emulator/beam/io.c               |  114 +-
 erts/emulator/beam/register.c         |    9 +-
 erts/emulator/beam/sys.h              |    5 +-
 erts/emulator/beam/time.c             |  699 ++++----
 erts/emulator/beam/utils.c            |  145 +-
 41 files changed, 4402 insertions(+), 2274 deletions(-)
 delete mode 100644 erts/emulator/beam/erl_bif_timer.c
 delete mode 100644 erts/emulator/beam/erl_bif_timer.h
 create mode 100644 erts/emulator/beam/erl_hl_timer.c
 create mode 100644 erts/emulator/beam/erl_hl_timer.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index ae3f30d82f..65cc5d0ceb 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -68,6 +68,7 @@ atom aborted
 atom abs_path
 atom absoluteURI
 atom ac
+atom accessor
 atom active
 atom all
 atom all_but_first
@@ -94,12 +95,14 @@ atom args
 atom arg0
 atom arity
 atom asn1
+atom async
 atom asynchronous
 atom atom
 atom atom_used
 atom attributes
 atom await_port_send_result
 atom await_proc_exit
+atom await_result
 atom await_sched_wall_time_modifications
 atom awaiting_load
 atom awaiting_unload
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index df1983a83d..fe0fa22751 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -371,7 +371,7 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 	ASSERT(commiter_state.stager == NULL);
 	commiter_state.stager = c_p;
 	erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &commiter_state.lop);
-	erts_smp_proc_inc_refc(c_p);
+	erts_proc_inc_refc(c_p);
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	/*
 	 * smp_code_ix_commiter() will do the rest "later"
@@ -398,7 +398,7 @@ static void smp_code_ix_commiter(void* null)
 	erts_resume(p, ERTS_PROC_LOCK_STATUS);
     }
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-    erts_smp_proc_dec_refc(p);
+    erts_proc_dec_refc(p);
 }
 #endif /* ERTS_SMP */
 
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8fcdc72895..4b2ab72a29 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2024,44 +2024,32 @@ void process_main(void)
      }
      GetArg1(1, timeout_value);
      if (timeout_value != make_small(0)) {
-#if !defined(ARCH_64) || HALFWORD_HEAP
-	 Uint time_val;
-#endif
 
-	 if (is_small(timeout_value) && signed_val(timeout_value) > 0 &&
-#if defined(ARCH_64) && !HALFWORD_HEAP
-	     ((unsigned_val(timeout_value) >> 32) == 0)
-#else
-	     1
-#endif
-	     ) {
-	     /*
-	      * The timer routiner will set c_p->i to the value in
-	      * c_p->def_arg_reg[0].  Note that it is safe to use this
-	      * location because there are no living x registers in
-	      * a receive statement.
-	      * Note that for the halfword emulator, the two first elements
-	      * of the array are used.
-	      */
-	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
-	     *pi = I+3;
-	     set_timer(c_p, unsigned_val(timeout_value));
-	 } else if (timeout_value == am_infinity) {
+	 if (timeout_value == am_infinity)
 	     c_p->flags |= F_TIMO;
-#if !defined(ARCH_64) || HALFWORD_HEAP
-	 } else if (term_to_Uint(timeout_value, &time_val)) {
-	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
-	     *pi = I+3;
-	     set_timer(c_p, time_val);
-#endif
-	 } else {		/* Wrong time */
-	     OpCase(i_wait_error_locked): {
-		 erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-		 /* Fall through */
+	 else {
+	     int tres = erts_set_proc_timer_term(c_p, timeout_value);
+	     if (tres == 0) {
+		 /*
+		  * The timer routiner will set c_p->i to the value in
+		  * c_p->def_arg_reg[0].  Note that it is safe to use this
+		  * location because there are no living x registers in
+		  * a receive statement.
+		  * Note that for the halfword emulator, the two first elements
+		  * of the array are used.
+		  */
+		 BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+		 *pi = I+3;
 	     }
+	     else { /* Wrong time */
+	     OpCase(i_wait_error_locked): {
+		     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+		     /* Fall through */
+		 }
 	     OpCase(i_wait_error): {
-		 c_p->freason = EXC_TIMEOUT_VALUE;
-		 goto find_func_info;
+		     c_p->freason = EXC_TIMEOUT_VALUE;
+		     goto find_func_info;
+		 }
 	     }
 	 }
 
@@ -2110,7 +2098,7 @@ void process_main(void)
      if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
 	 BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
 	 *p = I+3;
-	 set_timer(c_p, Arg(1));
+	 erts_set_proc_timer_uword(c_p, Arg(1));
      }
      goto wait2;
  }
@@ -5879,7 +5867,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
      * (e.g. spawn_link(erlang, abs, [1])).
      */
     if (fi.current == NULL) {
-	erts_set_current_function(&fi, c_p->initial);
+	erts_set_current_function(&fi, c_p->u.initial);
 	args = am_true; /* Just in case */
     } else {
 	args = get_args_from_exc(exc);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index cc20ec7440..0c9a1d079c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -44,6 +44,7 @@
 #include "erl_bits.h"
 #include "erl_bif_unique.h"
 
+Export *erts_await_result;
 static Export* flush_monitor_messages_trap = NULL;
 static Export* set_cpu_topology_trap = NULL;
 static Export* await_proc_exit_trap = NULL;
@@ -835,26 +836,6 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
     return ret;
 }
 
-BIF_RETTYPE erts_internal_monitor_process_2(BIF_ALIST_2)
-{
-    if (is_not_internal_pid(BIF_ARG_1)) {
-	if (is_external_pid(BIF_ARG_1)
-	    && (external_pid_dist_entry(BIF_ARG_1)
-		== erts_this_dist_entry)) {
-	    BIF_RET(am_false);
-	}
-	goto badarg;
-    }
-
-    if (is_not_internal_ref(BIF_ARG_2))
-	goto badarg;
-
-    BIF_RET(local_pid_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, 1));
-
-badarg:
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 /**********************************************************************/
 /* this is a combination of the spawn and link BIFs */
 
@@ -4906,6 +4887,10 @@ void erts_init_bif(void)
 #endif
 		     , &bif_return_trap);
 
+    erts_await_result = erts_export_put(am_erts_internal,
+					am_await_result,
+					1);
+
     erts_init_trap_export(&dsend_continue_trap_export,
 			  am_erts_internal, am_dsend_continue_trap, 1,
 			  dsend_continue_trap_1);
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index d461c3f479..b877711544 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -20,6 +20,7 @@
 #ifndef __BIF_H__
 #define __BIF_H__
 
+extern Export *erts_await_result;
 extern Export* erts_format_cpu_topology_trap;
 extern Export *erts_convert_time_unit_trap;
 
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 471f687101..eadba3eaff 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -171,11 +171,6 @@ bif erts_internal:map_hashmap_children/1
 
 bif erts_internal:time_unit/0
 
-bif erts_internal:get_bif_timer_servers/0
-bif erts_internal:create_bif_timer/0
-bif erts_internal:access_bif_timer/1
-
-bif erts_internal:monitor_process/2
 bif erts_internal:is_system_process/1
 
 # inet_db support
@@ -220,6 +215,15 @@ bif math:sqrt/1
 bif math:atan2/2
 bif math:pow/2
 
+bif erlang:start_timer/3
+bif erlang:start_timer/4
+bif erlang:send_after/3
+bif erlang:send_after/4
+bif erlang:cancel_timer/1
+bif erlang:cancel_timer/2
+bif erlang:read_timer/1
+bif erlang:read_timer/2
+
 bif erlang:make_tuple/2
 bif erlang:append_element/2
 bif erlang:make_tuple/3
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index e2fa572546..02e65cb9c6 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -36,7 +36,7 @@
 #include "atom.h"
 #include "beam_load.h"
 #include "erl_instrument.h"
-#include "erl_bif_timer.h"
+#include "erl_hl_timer.h"
 #include "erl_thr_progress.h"
 
 /* Forward declarations -- should really appear somewhere else */
@@ -108,7 +108,7 @@ process_killer(void)
 		case 'k': {
 		    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
 		    erts_aint32_t state;
-		    erts_smp_proc_inc_refc(rp);
+		    erts_proc_inc_refc(rp);
 		    erts_smp_proc_lock(rp, rp_locks);
 		    state = erts_smp_atomic32_read_acqb(&rp->state);
 		    if (state & (ERTS_PSFLG_FREE
@@ -131,7 +131,7 @@ process_killer(void)
 						     0);
 		    }
 		    erts_smp_proc_unlock(rp, rp_locks);
-		    erts_smp_proc_dec_refc(rp);
+		    erts_proc_dec_refc(rp);
 		}
 		case 'n': br = 1; break;
 		case 'r': return;
@@ -227,9 +227,9 @@ print_process_info(int to, void *to_arg, Process *p)
      * Display the initial function name
      */
     erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n",
-	       p->initial[INITIAL_MOD],
-	       p->initial[INITIAL_FUN],
-	       p->initial[INITIAL_ARI]);
+	       p->u.initial[INITIAL_MOD],
+	       p->u.initial[INITIAL_FUN],
+	       p->u.initial[INITIAL_ARI]);
     
     if (p->current != NULL) {
 	if (running) {
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 4344558348..d925709bd0 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -130,7 +130,7 @@ int erts_try_seize_code_write_permission(Process* c_p)
 	ASSERT(code_writing_process != c_p);
 	qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem));
 	qitem->p = c_p;
-	erts_smp_proc_inc_refc(c_p);
+	erts_proc_inc_refc(c_p);
 	qitem->next = code_write_queue;
 	code_write_queue = qitem;
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
@@ -151,7 +151,7 @@ void erts_release_code_write_permission(void)
 	}
 	erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS);
 	code_write_queue = qitem->next;
-	erts_smp_proc_dec_refc(qitem->p);
+	erts_proc_dec_refc(qitem->p);
 	erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem);
     }
     code_writing_process = NULL;
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index f2bceff4eb..71902c2f9f 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -39,7 +39,7 @@
 #include "erl_instrument.h"
 #include "erl_mseg.h"
 #include "erl_monitors.h"
-#include "erl_bif_timer.h"
+#include "erl_hl_timer.h"
 #include "erl_cpu_topology.h"
 #include "erl_thr_queue.h"
 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
@@ -575,6 +575,15 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
 	= sizeof(ErtsThrQElement_t);
 #endif
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)]
+	= erts_timer_type_size(ERTS_ALC_T_LL_PTIMER);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)]
+	= erts_timer_type_size(ERTS_ALC_T_HL_PTIMER);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)]
+	= erts_timer_type_size(ERTS_ALC_T_BIF_TIMER);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)]
+	= erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER);
+
 #ifdef HARD_DEBUG
     hdbg_init();
 #endif
@@ -3190,7 +3199,7 @@ reply_alloc_info(void *vair)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
  
     erts_smp_proc_unlock(rp, rp_locks);
-    erts_smp_proc_dec_refc(rp);
+    erts_proc_dec_refc(rp);
 
     if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0)
 	aireq_free(air);
@@ -3264,7 +3273,7 @@ erts_request_alloc_info(struct process *c_p,
     erts_smp_atomic32_init_nob(&air->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
-    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+    erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
 
 #ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index e2f8da38b9..57c506458c 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -163,9 +163,13 @@ type	MSG_ROOTS	TEMPORARY	PROCESSES	msg_roots
 type	ROOTSET		TEMPORARY	PROCESSES	root_set
 type	LOADER_TMP	TEMPORARY	CODE		loader_tmp
 type	PREPARED_CODE	SHORT_LIVED	CODE		prepared_code
-type	BIF_TIMER_TABLE	LONG_LIVED	SYSTEM		bif_timer_table
-type	SL_BIF_TIMER	SHORT_LIVED	PROCESSES	bif_timer_sl
-type	LL_BIF_TIMER	STANDARD	PROCESSES	bif_timer_ll
+type	TIMER_SERVICE	LONG_LIVED	SYSTEM		timer_service
+type    LL_PTIMER	FIXED_SIZE	PROCESSES	ll_ptimer
+type    HL_PTIMER	FIXED_SIZE	PROCESSES	hl_ptimer
+type    BIF_TIMER	FIXED_SIZE	PROCESSES	bif_timer
+type    ABIF_TIMER	FIXED_SIZE	PROCESSES	accessor_bif_timer
+type    TIMER_REQUEST	SHORT_LIVED	PROCESSES	timer_request
+type    BTM_YIELD_STATE	SHORT_LIVED	PROCESSES	btm_yield_state
 type	REG_TABLE	STANDARD	SYSTEM		reg_tab
 type	FUN_TABLE	STANDARD	CODE		fun_tab
 type	DIST_TABLE	STANDARD	SYSTEM		dist_tab
@@ -324,8 +328,6 @@ type	ACTIVE_PROCS	STANDARD	PROCESSES	active_procs
 +endif
 
 +if smp
-type	SL_PTIMER	SHORT_LIVED	SYSTEM		ptimer_sl
-type	LL_PTIMER	STANDARD	SYSTEM		ptimer_ll
 type	SYS_MSG_Q	SHORT_LIVED	PROCESSES	system_messages_queue
 type	FP_EXCEPTION	LONG_LIVED	SYSTEM		fp_exception
 type	LL_MPATHS	LONG_LIVED	SYSTEM		ll_migration_paths
@@ -365,7 +367,6 @@ type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	STANDARD_LOW	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD_LOW	SYSTEM		port_data_heap
-type	BIF_TIMER_DATA	LONG_LIVED_LOW	SYSTEM		bif_timer_data
 
 +else # "fullword"
 
@@ -386,7 +387,6 @@ type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
-type	BIF_TIMER_DATA	LONG_LIVED	SYSTEM		bif_timer_data
 
 +endif
 
@@ -419,7 +419,12 @@ type	ENVIRONMENT	TEMPORARY	SYSTEM		environment
 type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
 type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
 type	SYS_BLOCKING	STANDARD	SYSTEM		sys_blocking
+
++if smp
 type	SYS_WRITE_BUF	TEMPORARY	SYSTEM		sys_write_buf
++else
+type	SYS_WRITE_BUF	BINARY		SYSTEM		sys_write_buf
++endif
 
 +endif
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b2658a1fd6..e30687fca5 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1054,9 +1054,9 @@ process_info_aux(Process *BIF_P,
     case am_initial_call:
 	hp = HAlloc(BIF_P, 3+4);
 	res = TUPLE3(hp,
-		     rp->initial[INITIAL_MOD],
-		     rp->initial[INITIAL_FUN],
-		     make_small(rp->initial[INITIAL_ARI]));
+		     rp->u.initial[INITIAL_MOD],
+		     rp->u.initial[INITIAL_FUN],
+		     make_small(rp->u.initial[INITIAL_ARI]));
 	hp += 4;
 	break;
 
@@ -2129,6 +2129,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(erts_has_time_correction() ? am_true : am_false);
     } else if (ERTS_IS_ATOM_STR("start_time", BIF_ARG_1)) {
 	BIF_RET(erts_get_monotonic_start_time(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("end_time", BIF_ARG_1)) {
+	BIF_RET(erts_get_monotonic_end_time(BIF_P));
     } else if (ERTS_IS_ATOM_STR("time_warp_mode", BIF_ARG_1)) {
 	switch (erts_time_warp_mode()) {
 	case ERTS_NO_TIME_WARP_MODE: {
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
deleted file mode 100644
index 0bd8d20c34..0000000000
--- a/erts/emulator/beam/erl_bif_timer.c
+++ /dev/null
@@ -1,853 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2005-2012. All Rights Reserved.
- *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- *
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "erl_bif_timer.h"
-#include "global.h"
-#include "bif.h"
-#include "error.h"
-#include "big.h"
-#include "erl_thr_progress.h"
-#include "erl_bif_unique.h"
-
-/****************************************************************************
-** BIF Timer support
-****************************************************************************/
-
-#define BTM_FLG_SL_TIMER	(((Uint32) 1) << 0)
-#define BTM_FLG_CANCELED	(((Uint32) 1) << 1)
-#define BTM_FLG_HEAD		(((Uint32) 1) << 2)
-#define BTM_FLG_BYNAME		(((Uint32) 1) << 3)
-#define BTM_FLG_WRAP		(((Uint32) 1) << 4)
-
-struct ErtsBifTimer_ {
-    struct {
-	union {
-	    ErtsBifTimer **head;
-	    ErtsBifTimer *prev;
-	} u;
-	ErtsBifTimer *next;
-    } tab;
-    union {
-	Eterm name;
-	struct {
-	    ErtsBifTimer *prev;
-	    ErtsBifTimer *next;
-	    Process *ess;
-	} proc;
-    } receiver;
-    ErlTimer tm;
-    ErlHeapFragment* bp;
-    Uint32 flags;
-    Eterm message;
-    Uint32 ref_numbers[ERTS_REF_NUMBERS];
-};
-
-#ifdef SMALL_MEMORY
-#define TIMER_HASH_VEC_SZ	3331
-#define BTM_PREALC_SZ		10
-#else
-#define TIMER_HASH_VEC_SZ	10007
-#define BTM_PREALC_SZ		100
-#endif
-static ErtsBifTimer **bif_timer_tab;  
-static Uint no_bif_timers;
-
-
-static erts_smp_rwmtx_t bif_timer_lock;
-
-#define erts_smp_safe_btm_rwlock(P, L) \
-	safe_btm_lock((P), (L), 1)
-#define erts_smp_safe_btm_rlock(P, L) \
-	safe_btm_lock((P), (L), 0)
-#define erts_smp_btm_rwlock() \
-	erts_smp_rwmtx_rwlock(&bif_timer_lock)
-#define erts_smp_btm_tryrwlock() \
-	erts_smp_rwmtx_tryrwlock(&bif_timer_lock)
-#define erts_smp_btm_rwunlock() \
-	erts_smp_rwmtx_rwunlock(&bif_timer_lock)
-#define erts_smp_btm_rlock() \
-	erts_smp_rwmtx_rlock(&bif_timer_lock)
-#define erts_smp_btm_tryrlock() \
-	erts_smp_rwmtx_tryrlock(&bif_timer_lock)
-#define erts_smp_btm_runlock() \
-	erts_smp_rwmtx_runlock(&bif_timer_lock)
-#define erts_smp_btm_lock_init() \
-	erts_smp_rwmtx_init(&bif_timer_lock, "bif_timers")
-
-
-static ERTS_INLINE int
-safe_btm_lock(Process *c_p, ErtsProcLocks c_p_locks, int rw_lock)
-{
-    ASSERT(c_p && c_p_locks);
-#ifdef ERTS_SMP
-    if ((rw_lock ? erts_smp_btm_tryrwlock() : erts_smp_btm_tryrlock()) != EBUSY)
-	return 0;
-    erts_smp_proc_unlock(c_p, c_p_locks);
-    if (rw_lock)
-	erts_smp_btm_rwlock();
-    else
-	erts_smp_btm_rlock();
-    erts_smp_proc_lock(c_p, c_p_locks);
-    if (ERTS_PROC_IS_EXITING(c_p)) {
-	if (rw_lock)
-	    erts_smp_btm_rwunlock();
-	else
-	    erts_smp_btm_runlock();
-	return 1;
-    }
-#endif
-    return 0;
-}
-
-ERTS_SCHED_PREF_PALLOC_IMPL(btm_pre, ErtsBifTimer, BTM_PREALC_SZ)
-
-static ERTS_INLINE int
-get_index(Uint32 *ref_numbers, Uint32 len)
-{
-    Uint32 hash;
-    /* len can potentially be larger than ERTS_REF_NUMBERS
-       if it has visited another node... */
-    if (len > ERTS_REF_NUMBERS)
-	len = ERTS_REF_NUMBERS;
-
-#if ERTS_REF_NUMBERS != 3
-#error "ERTS_REF_NUMBERS changed. Update me..."
-#endif
-    switch (len) {
-    case 3: if (!ref_numbers[2]) len = 2;
-    case 2: if (!ref_numbers[1]) len = 1;
-    default:  break;
-    }
-
-    ASSERT(1 <= len && len <= ERTS_REF_NUMBERS);
-
-    hash = block_hash((byte *) ref_numbers, len * sizeof(Uint32), 0x08d12e65);
-    return (int) (hash % ((Uint32) TIMER_HASH_VEC_SZ));
-}
-
-static Eterm
-create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len)
-{
-    Uint32 *datap;
-    int i;
-
-
-    if (len > ERTS_MAX_REF_NUMBERS) {
-	/* Such large refs should no be able to appear in the emulator */
-	erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-    }
-
-#if defined(ARCH_64) && !HALFWORD_HEAP
-    hp[0] = make_ref_thing_header(len/2 + 1);
-    datap = (Uint32 *) &hp[1];
-    *(datap++) = len;
-#else
-    hp[0] = make_ref_thing_header(len);
-    datap = (Uint32 *) &hp[1];
-#endif
-
-    for (i = 0; i < len; i++)
-	datap[i] = ref_numbers[i];
-
-    return make_internal_ref(hp);
-}
-
-static int
-eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2)
-{
-#if defined(ARCH_64) && !HALFWORD_HEAP
-#define MAX_REF_HEAP_SZ (1+(ERTS_MAX_REF_NUMBERS/2+1))
-#else
-#define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS)
-#endif
-    DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2));
-    Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ;
-
-    return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2));
-#undef MAX_REF_HEAP_SZ
-}
-
-static ERTS_INLINE int
-eq_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2)
-{
-    int res;
-    if (len1 != ERTS_REF_NUMBERS || len2 != ERTS_REF_NUMBERS) {
-	/* Can potentially happen, but will never... */
-	return eq_non_standard_ref_numbers(rn1, len1, rn2, len2);
-    }
-
-#if ERTS_REF_NUMBERS != 3
-#error "ERTS_REF_NUMBERS changed. Update me..."
-#endif
-    res = rn1[0] == rn2[0] && rn1[1] == rn2[1] && rn1[2] == rn2[2];
-
-    ASSERT(res
-	   ? eq_non_standard_ref_numbers(rn1, len1, rn2, len2)
-	   : !eq_non_standard_ref_numbers(rn1, len1, rn2, len2));
-
-    return res;
-}
-
-static ERTS_INLINE ErtsBifTimer *
-tab_find(Eterm ref)
-{
-    Uint32 *ref_numbers = internal_ref_numbers(ref);
-    Uint32 ref_numbers_len = internal_ref_no_of_numbers(ref);
-    int ix = get_index(ref_numbers, ref_numbers_len);
-    ErtsBifTimer* btm;
-
-    for (btm = bif_timer_tab[ix]; btm; btm = btm->tab.next)
-	if (eq_ref_numbers(ref_numbers, ref_numbers_len,
-			   btm->ref_numbers, ERTS_REF_NUMBERS))
-	    return btm;
-    return NULL;
-}
-
-static ERTS_INLINE void
-tab_remove(ErtsBifTimer* btm)
-{
-    if (btm->flags & BTM_FLG_HEAD) {
-	*btm->tab.u.head = btm->tab.next;
-	if (btm->tab.next) {
-	    btm->tab.next->flags |= BTM_FLG_HEAD;
-	    btm->tab.next->tab.u.head = btm->tab.u.head;
-	}
-    }
-    else {
-	btm->tab.u.prev->tab.next = btm->tab.next;
-	if (btm->tab.next)
-	    btm->tab.next->tab.u.prev = btm->tab.u.prev;
-    }
-    btm->flags |= BTM_FLG_CANCELED;
-    ASSERT(no_bif_timers > 0);
-    no_bif_timers--;
-}
-
-static ERTS_INLINE void
-tab_insert(ErtsBifTimer* btm)
-{
-    int ix = get_index(btm->ref_numbers, ERTS_REF_NUMBERS);
-    ErtsBifTimer* btm_list = bif_timer_tab[ix];
-
-    if (btm_list) {
-	btm_list->flags &= ~BTM_FLG_HEAD;
-	btm_list->tab.u.prev = btm;
-    }
-
-    btm->flags |= BTM_FLG_HEAD;
-    btm->tab.u.head = &bif_timer_tab[ix];
-    btm->tab.next = btm_list;
-    bif_timer_tab[ix] = btm;
-    no_bif_timers++;
-}
-
-static ERTS_INLINE void
-link_proc(Process *p, ErtsBifTimer* btm)
-{
-    btm->receiver.proc.ess = p;
-    btm->receiver.proc.prev = NULL;
-    btm->receiver.proc.next = p->u.bif_timers;
-    if (p->u.bif_timers)	
-	p->u.bif_timers->receiver.proc.prev = btm;
-    p->u.bif_timers = btm;
-}
-
-static ERTS_INLINE void
-unlink_proc(ErtsBifTimer* btm)
-{
-    if (btm->receiver.proc.prev)
-	btm->receiver.proc.prev->receiver.proc.next = btm->receiver.proc.next;
-    else
-	btm->receiver.proc.ess->u.bif_timers = btm->receiver.proc.next;
-    if (btm->receiver.proc.next)
-	btm->receiver.proc.next->receiver.proc.prev = btm->receiver.proc.prev;
-}
-
-static void
-bif_timer_cleanup(ErtsBifTimer* btm)
-{
-    ASSERT(btm);
-
-    if (btm->bp)
-	free_message_buffer(btm->bp);
-
-    if (!btm_pre_free(btm)) {
-	if (btm->flags & BTM_FLG_SL_TIMER)
-	    erts_free(ERTS_ALC_T_SL_BIF_TIMER, (void *) btm);
-	else
-	    erts_free(ERTS_ALC_T_LL_BIF_TIMER, (void *) btm);
-    }
-}
-
-static void
-bif_timer_timeout(ErtsBifTimer* btm)
-{
-    ASSERT(btm);
-
-
-    erts_smp_btm_rwlock();
-
-    if (btm->flags & BTM_FLG_CANCELED) {
-    /*
-     * A concurrent cancel is ongoing. Do not send the timeout message,
-     * but cleanup here since the cancel call-back won't be called.
-     */
-#ifndef ERTS_SMP
-	ASSERT(0);
-#endif
-    }
-    else {
-	ErtsProcLocks rp_locks = 0;
-	Process* rp;
-
-	tab_remove(btm);
-
-	ASSERT(!erts_get_current_process());
-
-	if (btm->flags & BTM_FLG_BYNAME)
-	    rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0);
-	else {
-	    rp = btm->receiver.proc.ess;
-	    unlink_proc(btm);
-	}
-
-	if (rp) {
-	    Eterm message;
-	    ErlHeapFragment *bp;
-
-	    bp = btm->bp;
-	    btm->bp = NULL; /* Prevent cleanup of message buffer... */
-
-	    if (!(btm->flags & BTM_FLG_WRAP))
-		message = btm->message;
-	    else {
-#if ERTS_REF_NUMBERS != 3
-#error "ERTS_REF_NUMBERS changed. Update me..."
-#endif
-		Eterm ref;
-		Uint *hp;
-		Uint wrap_size = REF_THING_SIZE + 4;
-		message = btm->message;
-
-		if (!bp) {
-		    ErlOffHeap *ohp;
-		    ASSERT(is_immed(message));
-		    hp = erts_alloc_message_heap(wrap_size,
-						 &bp,
-						 &ohp,
-						 rp,
-						 &rp_locks);
-		} else {
-		    Eterm old_size = bp->used_size;
-		    bp = erts_resize_message_buffer(bp, old_size + wrap_size,
-						    &message, 1);
-		    hp = &bp->mem[0] + old_size;
-		}
-
-		write_ref_thing(hp,
-				btm->ref_numbers[0],
-				btm->ref_numbers[1],
-				btm->ref_numbers[2]);
-		ref = make_internal_ref(hp);
-		hp += REF_THING_SIZE;
-		message = TUPLE3(hp, am_timeout, ref, message);
-	    }
-
-	    erts_queue_message(rp, &rp_locks, bp, message, NIL
-#ifdef USE_VM_PROBES
-			       , NIL
-#endif
-			       );
-	    erts_smp_proc_unlock(rp, rp_locks);
-	}
-    }
-
-    erts_smp_btm_rwunlock();
-
-    bif_timer_cleanup(btm);
-}
-
-static Eterm
-setup_bif_timer(Uint32 xflags,
-		Process *c_p,
-		Eterm time,
-		Eterm receiver,
-		Eterm message)
-{
-    Process *rp;
-    ErtsBifTimer* btm;
-    Uint timeout;
-    Eterm ref;
-    Uint32 *ref_numbers;
-    
-    if (!term_to_Uint(time, &timeout))
-	return THE_NON_VALUE;
-#if defined(ARCH_64) && !HALFWORD_HEAP
-    if ((timeout >> 32) != 0)
-	return THE_NON_VALUE;
-#endif
-    if (is_not_internal_pid(receiver) && is_not_atom(receiver))
-	return THE_NON_VALUE;
-
-    ref = erts_make_ref(c_p);
-
-    if (is_atom(receiver))
-	rp = NULL;
-    else {
-	rp = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
-			   receiver, ERTS_PROC_LOCK_MSGQ);
-	if (!rp)
-	    return ref;
-    }
-
-    if (timeout < ERTS_ALC_MIN_LONG_LIVED_TIME) {
-	if (timeout < 1000) {
-	    btm = btm_pre_alloc();
-	    if (!btm)
-		goto sl_timer_alloc;
-	    btm->flags = 0;
-	}
-	else {
-	sl_timer_alloc:
-	    btm = (ErtsBifTimer *) erts_alloc(ERTS_ALC_T_SL_BIF_TIMER,
-					      sizeof(ErtsBifTimer));
-	    btm->flags = BTM_FLG_SL_TIMER;
-	}
-    }
-    else {
-	btm = (ErtsBifTimer *) erts_alloc(ERTS_ALC_T_LL_BIF_TIMER,
-					 sizeof(ErtsBifTimer));
-	btm->flags = 0;
-    }
-
-    if (rp) {
-	link_proc(rp, btm);
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
-    }
-    else {
-	ASSERT(is_atom(receiver));
-	btm->receiver.name = receiver;
-	btm->flags |= BTM_FLG_BYNAME;
-    }
-
-    btm->flags |= xflags;
-
-    ref_numbers = internal_ref_numbers(ref);
-    ASSERT(internal_ref_no_of_numbers(ref) == 3);
-#if ERTS_REF_NUMBERS != 3
-#error "ERTS_REF_NUMBERS changed. Update me..."
-#endif
-    btm->ref_numbers[0] = ref_numbers[0];
-    btm->ref_numbers[1] = ref_numbers[1];
-    btm->ref_numbers[2] = ref_numbers[2];
-
-    ASSERT(eq_ref_numbers(btm->ref_numbers, ERTS_REF_NUMBERS,
-			  ref_numbers, ERTS_REF_NUMBERS));
-
-    if (is_immed(message)) {
-	btm->bp = NULL;
-	btm->message = message;
-    }
-    else {
-	ErlHeapFragment* bp;
-	Eterm* hp;
-	Uint size;
-
-	size = size_object(message);
-	btm->bp = bp = new_message_buffer(size);
-	hp = bp->mem;
-	btm->message = copy_struct(message, size, &hp, &bp->off_heap);
-    }
-
-    tab_insert(btm);
-    ASSERT(btm == tab_find(ref));
-    erts_init_timer(&btm->tm);
-    erts_set_timer(&btm->tm,
-		  (ErlTimeoutProc) bif_timer_timeout,
-		  (ErlCancelProc) bif_timer_cleanup,
-		  (void *) btm,
-		  timeout);
-    return ref;
-}
-
-BIF_RETTYPE old_send_after_3(BIF_ALIST_3);
-/* send_after(Time, Pid, Message) -> Ref */
-BIF_RETTYPE old_send_after_3(BIF_ALIST_3)
-{
-    Eterm res;
-
-    if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN))
-	ERTS_BIF_EXITED(BIF_P);
-
-    res = setup_bif_timer(0, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
-
-    erts_smp_btm_rwunlock();
-
-    if (is_non_value(res)) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    else {
-	ASSERT(is_internal_ref(res));
-	BIF_RET(res);
-    }
-}
-
-BIF_RETTYPE old_start_timer_3(BIF_ALIST_3);
-/* start_timer(Time, Pid, Message) -> Ref */
-BIF_RETTYPE old_start_timer_3(BIF_ALIST_3)
-{
-    Eterm res;
-
-    if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN))
-	ERTS_BIF_EXITED(BIF_P);
-
-    res = setup_bif_timer(BTM_FLG_WRAP, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
-
-    erts_smp_btm_rwunlock();
-
-    if (is_non_value(res)) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    else {
-	ASSERT(is_internal_ref(res));
-	BIF_RET(res);
-    }
-}
-
-BIF_RETTYPE old_cancel_timer_1(BIF_ALIST_1);
-/* cancel_timer(Ref) -> false | RemainingTime */
-BIF_RETTYPE old_cancel_timer_1(BIF_ALIST_1)
-{
-    Eterm res;
-    ErtsBifTimer *btm;
-
-    if (is_not_internal_ref(BIF_ARG_1)) {
-	if (is_ref(BIF_ARG_1)) {
-	    BIF_RET(am_false);
-	}
-	BIF_ERROR(BIF_P, BADARG);
-    }
-
-    if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN))
-	ERTS_BIF_EXITED(BIF_P);
-
-    btm = tab_find(BIF_ARG_1);
-    if (!btm || btm->flags & BTM_FLG_CANCELED) {
-	erts_smp_btm_rwunlock();
-	res = am_false;
-    }
-    else {
-	Uint left = erts_time_left(&btm->tm);
-	if (!(btm->flags & BTM_FLG_BYNAME)) {
-	    erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ);
-	    unlink_proc(btm);
-	    erts_smp_proc_unlock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ);
-	}
-	tab_remove(btm);
-	ASSERT(!tab_find(BIF_ARG_1));
-	erts_cancel_timer(&btm->tm);
-	erts_smp_btm_rwunlock();
-	res = erts_make_integer(left, BIF_P);
-    }
-
-    BIF_RET(res);
-}
-
-BIF_RETTYPE old_read_timer_1(BIF_ALIST_1);
-/* read_timer(Ref) -> false | RemainingTime */
-BIF_RETTYPE old_read_timer_1(BIF_ALIST_1)
-{
-    Eterm res;
-    ErtsBifTimer *btm;
-
-    if (is_not_internal_ref(BIF_ARG_1)) {
-	if (is_ref(BIF_ARG_1)) {
-	    BIF_RET(am_false);
-	}
-	BIF_ERROR(BIF_P, BADARG);
-    }
-
-    if (erts_smp_safe_btm_rlock(BIF_P, ERTS_PROC_LOCK_MAIN))
-	ERTS_BIF_EXITED(BIF_P);
-
-    btm = tab_find(BIF_ARG_1);
-    if (!btm || btm->flags & BTM_FLG_CANCELED) {
-	res = am_false;
-    }
-    else {
-	Uint left = erts_time_left(&btm->tm);
-	res = erts_make_integer(left, BIF_P);
-    }
-
-    erts_smp_btm_runlock();
-
-    BIF_RET(res);
-}
-
-void
-erts_print_bif_timer_info(int to, void *to_arg)
-{
-    int i;
-    int lock = !ERTS_IS_CRASH_DUMPING;
-
-    if (lock)
-	erts_smp_btm_rlock();
-
-    for (i = 0; i < TIMER_HASH_VEC_SZ; i++) {
-	ErtsBifTimer *btm;
-	for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) {
-	    Eterm receiver = (btm->flags & BTM_FLG_BYNAME
-			      ? btm->receiver.name
-			      : btm->receiver.proc.ess->common.id);
-	    erts_print(to, to_arg, "=timer:%T\n", receiver);
-	    erts_print(to, to_arg, "Message: %T\n", btm->message);
-	    erts_print(to, to_arg, "Time left: %u\n",
-		       erts_time_left(&btm->tm));
-	}
-    }
-
-    if (lock)
-	erts_smp_btm_runlock();
-}
-
-
-void
-erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks)
-{
-    ErtsBifTimer *btm;
-
-    if (erts_smp_btm_tryrwlock() == EBUSY) {
-	erts_smp_proc_unlock(p, plocks);
-	erts_smp_btm_rwlock();
-	erts_smp_proc_lock(p, plocks);
-    }
-
-    btm = p->u.bif_timers;
-    while (btm) {
-	ErtsBifTimer *tmp_btm;
-	ASSERT(!(btm->flags & BTM_FLG_CANCELED));
-	tab_remove(btm);
-	tmp_btm = btm;
-	btm = btm->receiver.proc.next;
-	erts_cancel_timer(&tmp_btm->tm);
-    }
-
-    p->u.bif_timers = NULL;
-
-    erts_smp_btm_rwunlock();
-}
-
-static void erts_old_bif_timer_init(void)
-{
-    int i;
-    no_bif_timers = 0;
-    init_btm_pre_alloc();
-    erts_smp_btm_lock_init();
-    bif_timer_tab = erts_alloc(ERTS_ALC_T_BIF_TIMER_TABLE,
-			       sizeof(ErtsBifTimer *)*TIMER_HASH_VEC_SZ);
-    for (i = 0; i < TIMER_HASH_VEC_SZ; ++i)
-	bif_timer_tab[i] = NULL;
-}
-
-Uint
-erts_bif_timer_memory_size(void)
-{
-    Uint res;
-    int lock = !ERTS_IS_CRASH_DUMPING;
-
-    if (lock)
-	erts_smp_btm_rlock();
-
-    res = (sizeof(ErtsBifTimer *)*TIMER_HASH_VEC_SZ
-	   + no_bif_timers*sizeof(ErtsBifTimer));
-
-    if (lock)
-	erts_smp_btm_runlock();
-
-    return res;
-}
-
-
-void
-erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *),
-		       void *arg)
-{
-    int i;
-
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
-    for (i = 0; i < TIMER_HASH_VEC_SZ; i++) {
-	ErtsBifTimer *btm;
-	for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) {
-	    (*func)((btm->flags & BTM_FLG_BYNAME
-		     ? btm->receiver.name
-		     : btm->receiver.proc.ess->common.id),
-		    btm->message,
-		    btm->bp,
-		    arg);
-	}
-    }
-}
-
-typedef struct {
-    Uint ref_heap[REF_THING_SIZE];
-    Eterm pid[1];
-} ErtsBifTimerServers;
-
-static ErtsBifTimerServers *bif_timer_servers;
-
-void erts_bif_timer_init(void)
-{
-    erts_old_bif_timer_init();
-}
-
-void
-erts_bif_timer_start_servers(Eterm parent)
-{
-    Process *parent_proc;
-    Eterm *hp, btr_ref, arg_list_end;
-    ErlSpawnOpts so;
-    int i;
-
-    bif_timer_servers = erts_alloc(ERTS_ALC_T_BIF_TIMER_DATA,
-				   (sizeof(ErtsBifTimerServers)
-				    + (sizeof(Eterm)*(erts_no_schedulers-1))));
-
-    so.flags = SPO_USE_ARGS|SPO_SYSTEM_PROC|SPO_PREFER_SCHED|SPO_OFF_HEAP_MSGS;
-    so.min_heap_size  = H_MIN_SIZE;
-    so.min_vheap_size = BIN_VH_MIN_SIZE;
-    so.priority       = PRIORITY_MAX;
-    so.max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
-
-    /*
-     * Parent is "init" and schedulers have not yet been started, so it
-     * *should* be alive and well...
-     */
-    ASSERT(is_internal_pid(parent));
-    parent_proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
-							internal_pid_index(parent));
-    ASSERT(parent_proc);
-    ASSERT(parent_proc->common.id == parent);
-    ASSERT(!ERTS_PROC_IS_EXITING(parent_proc));
-
-    erts_smp_proc_lock(parent_proc, ERTS_PROC_LOCK_MAIN);
-
-    hp = HAlloc(parent_proc, 2*erts_no_schedulers + 2 + REF_THING_SIZE);
-
-    btr_ref = erts_make_ref_in_buffer(hp);
-    hp += REF_THING_SIZE;
-
-    arg_list_end = CONS(hp, btr_ref, NIL);
-    hp += 2;
-
-    for (i = 0; i < erts_no_schedulers; i++) {
-	int sched = i+1;
-	Eterm arg_list = CONS(hp, make_small(i+1), arg_list_end);
-	hp += 2;
-
-	so.scheduler = sched; /* Preferred scheduler */
-
-	bif_timer_servers->pid[i] = erl_create_process(parent_proc,
-						       am_erts_internal,
-						       am_bif_timer_server,
-						       arg_list,
-						       &so);
-    }
-
-    erts_smp_proc_unlock(parent_proc, ERTS_PROC_LOCK_MAIN);
-
-    hp = internal_ref_val(btr_ref);
-    for (i = 0; i < REF_THING_SIZE; i++)
-	bif_timer_servers->ref_heap[i] = hp[i];
-}
-
-BIF_RETTYPE
-erts_internal_get_bif_timer_servers_0(BIF_ALIST_0)
-{
-    int i;
-    Eterm *hp, res = NIL;
-
-    hp = HAlloc(BIF_P, erts_no_schedulers*2);
-    for (i = erts_no_schedulers-1; i >= 0; i--) {
-	res = CONS(hp, bif_timer_servers->pid[i], res);
-	hp += 2;
-    }
-    BIF_RET(res);
-}
-
-BIF_RETTYPE
-erts_internal_access_bif_timer_1(BIF_ALIST_1)
-{
-    int ix;
-    Uint32 *rdp;
-    Eterm ref, pid, *hp, res;
-
-    if (is_not_internal_ref(BIF_ARG_1)) {
-	if (is_not_ref(BIF_ARG_1))
-	    BIF_ERROR(BIF_P, BADARG);
-	BIF_RET(am_undefined);
-    }
-
-    rdp = internal_ref_numbers(BIF_ARG_1);
-    ix = (int) erts_get_ref_numbers_thr_id(rdp);
-    if (ix < 1 || erts_no_schedulers < ix)
-	BIF_RET(am_undefined);
-
-    pid = bif_timer_servers->pid[ix-1];
-    ASSERT(is_internal_pid(pid));
-
-    hp = HAlloc(BIF_P, 3 /* 2-tuple */ + REF_THING_SIZE);
-    for (ix = 0; ix < REF_THING_SIZE; ix++)
-	hp[ix] = bif_timer_servers->ref_heap[ix];
-    ref = make_internal_ref(&hp[0]);
-    hp += REF_THING_SIZE;
-
-    res = TUPLE2(hp, ref, pid);
-    BIF_RET(res);
-}
-
-BIF_RETTYPE
-erts_internal_create_bif_timer_0(BIF_ALIST_0)
-{
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(BIF_P);
-    Eterm *hp, btr_ref, t_ref, pid, res;
-    int ix;
-
-    hp = HAlloc(BIF_P, 4 /* 3-tuple */ + 2*REF_THING_SIZE);
-    for (ix = 0; ix < REF_THING_SIZE; ix++)
-	hp[ix] = bif_timer_servers->ref_heap[ix];
-    btr_ref = make_internal_ref(&hp[0]);
-    hp += REF_THING_SIZE;
-
-    t_ref = erts_sched_make_ref_in_buffer(esdp, hp);
-    hp += REF_THING_SIZE;
-
-    ASSERT(erts_get_ref_numbers_thr_id(internal_ref_numbers(t_ref))
-	   == (Uint32) esdp->no);
-
-    pid = bif_timer_servers->pid[((int) esdp->no) - 1];
-    
-    res = TUPLE3(hp, btr_ref, pid, t_ref);
-
-    BIF_RET(res);
-}
diff --git a/erts/emulator/beam/erl_bif_timer.h b/erts/emulator/beam/erl_bif_timer.h
deleted file mode 100644
index c2f5dfd3c3..0000000000
--- a/erts/emulator/beam/erl_bif_timer.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-
-#ifndef ERL_BIF_TIMER_H__
-#define ERL_BIF_TIMER_H__
-
-typedef struct ErtsBifTimer_ ErtsBifTimer;
-
-#include "sys.h"
-#include "erl_process.h"
-#include "erl_message.h"
-
-Uint erts_bif_timer_memory_size(void);
-void erts_print_bif_timer_info(int to, void *to_arg);
-void erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks);
-void erts_bif_timer_init(void);
-void erts_bif_timer_foreach(void (*func)(Eterm,Eterm,ErlHeapFragment *,void *),
-			    void *arg);
-void erts_bif_timer_start_servers(Eterm);
-#endif
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index ac57205c47..13e0160648 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -359,7 +359,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	ASSERT(finish_bp.stager == NULL);
 	finish_bp.stager = p;
 	erts_schedule_thr_prgr_later_op(smp_bp_finisher, NULL, &finish_bp.lop);
-	erts_smp_proc_inc_refc(p);
+	erts_proc_inc_refc(p);
 	erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL);
 	ERTS_BIF_YIELD_RETURN(p, make_small(matches));
     }
@@ -393,7 +393,7 @@ static void smp_bp_finisher(void* null)
 	    erts_resume(p, ERTS_PROC_LOCK_STATUS);
 	}
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	erts_smp_proc_dec_refc(p);
+	erts_proc_dec_refc(p);
     }
 }
 #endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index fff892ae54..2e2cb98354 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -620,7 +620,7 @@ BIF_RETTYPE ets_safe_fixtable_2(BIF_ALIST_2)
     erts_fprintf(stderr,
 		"ets:safe_fixtable(%T,%T); Process: %T, initial: %T:%T/%bpu\n",
 		BIF_ARG_1, BIF_ARG_2, BIF_P->common.id,
-		BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
+		BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]);
 #endif
     kind = (BIF_ARG_2 == am_true) ? LCK_READ : LCK_WRITE_REC; 
 
@@ -1247,7 +1247,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     erts_fprintf(stderr,
 		"ets:rename(%T,%T); Process: %T, initial: %T:%T/%bpu\n",
 		BIF_ARG_1, BIF_ARG_2, BIF_P->common.id,
-		BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
+		BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]);
 #endif
 
 
@@ -1563,7 +1563,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     erts_fprintf(stderr,
 		"ets:new(%T,%T)=%T; Process: %T, initial: %T:%T/%bpu\n",
 		 BIF_ARG_1, BIF_ARG_2, ret, BIF_P->common.id,
-		 BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
+		 BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]);
 	erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n",
 		     erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size));
 	erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n",
@@ -1696,7 +1696,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     erts_fprintf(stderr,
 		"ets:delete(%T); Process: %T, initial: %T:%T/%bpu\n",
 		BIF_ARG_1, BIF_P->common.id,
-		BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
+		BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]);
 #endif
 
     CHECK_TABLES();
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 307f9c93e0..6a986b31ed 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -97,10 +97,10 @@ typedef struct {
 
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
-static Uint combined_message_size(Process* p, int off_heap_msgs);
+static Uint combined_message_size(Process* p);
 static void remove_message_buffers(Process* p);
-static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs);
-static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs);
+static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
+static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
 static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
 static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size);
 static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size);
@@ -403,9 +403,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
     Uint reclaimed_now = 0;
     int done = 0;
-    int off_heap_msgs;
     ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
-    erts_aint32_t state;
     ErtsSchedulerData *esdp;
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
@@ -422,10 +420,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
         trace_gc(p, am_gc_start);
     }
 
-    state = erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
-    off_heap_msgs = state & ERTS_PSFLG_OFF_HEAP_MSGS;
+    (void) erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     if (erts_system_monitor_long_gc != 0)
-	start_time = erts_get_monotonic_time();
+	start_time = erts_get_monotonic_time(esdp);
 
     ERTS_CHK_OFFHEAP(p);
 
@@ -448,11 +445,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     while (!done) {
 	if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) {
 	    DTRACE2(gc_major_start, pidbuf, need);
-	    done = major_collection(p, need, objv, nobj, &reclaimed_now, off_heap_msgs);
+	    done = major_collection(p, need, objv, nobj, &reclaimed_now);
 	    DTRACE2(gc_major_end, pidbuf, reclaimed_now);
 	} else {
 	    DTRACE2(gc_minor_start, pidbuf, need);
-	    done = minor_collection(p, need, objv, nobj, &reclaimed_now, off_heap_msgs);
+	    done = minor_collection(p, need, objv, nobj, &reclaimed_now);
 	    DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
 	}
     }
@@ -477,7 +474,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 	Uint gc_time;
 	if (erts_test_long_gc_sleep)
 	    while (0 != erts_milli_sleep(erts_test_long_gc_sleep));
-	end_time = erts_get_monotonic_time();
+	end_time = erts_get_monotonic_time(esdp);
 	gc_time = (Uint) ERTS_MONOTONIC_TO_MSEC(end_time - start_time);
 	if (gc_time && gc_time > erts_system_monitor_long_gc) {
 	    monitor_long_gc(p, gc_time);
@@ -833,7 +830,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 }
 
 static int
-minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs)
+minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 {
     Uint mature = HIGH_WATER(p) - HEAP_START(p);
 
@@ -872,22 +869,20 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int of
 	Uint size_after;
 	Uint need_after;
 	Uint stack_size = STACK_SZ_ON_HEAP(p);
-	Uint fragments = MBUF_SIZE(p) + combined_message_size(p, off_heap_msgs);
+	Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
 	Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p));
 	Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0);
 
         do_minor(p, new_sz, objv, nobj);
 
-	if (!off_heap_msgs) {
-	    /*
-	     * Copy newly received message onto the end of the new heap.
-	     */
-	    ErtsGcQuickSanityCheck(p);
-	    for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-		if (msgp->data.attached) {
-		    erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
-		    ErtsGcQuickSanityCheck(p);
-		}
+	/*
+	 * Copy newly received message onto the end of the new heap.
+	 */
+	ErtsGcQuickSanityCheck(p);
+	for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+	    if (msgp->data.attached) {
+		erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
+		ErtsGcQuickSanityCheck(p);
 	    }
 	}
 	ErtsGcQuickSanityCheck(p);
@@ -1213,7 +1208,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
  */
 
 static int
-major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs)
+major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 {
     Rootset rootset;
     Roots* roots;
@@ -1226,7 +1221,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int of
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
     Uint n;
     Uint new_sz;
-    Uint fragments = MBUF_SIZE(p) + combined_message_size(p, off_heap_msgs);
+    Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
 
     size_before = fragments + (HEAP_TOP(p) - HEAP_START(p));
 
@@ -1436,7 +1431,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int of
 
     ErtsGcQuickSanityCheck(p);
 
-    if (!off_heap_msgs) {
+    {
 	ErlMessage *msgp;
 	/*
 	 * Copy newly received message onto the end of the new heap.
@@ -1506,14 +1501,11 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int
  * mbuf list.
  */
 static Uint
-combined_message_size(Process* p, int off_heap_msgs)
+combined_message_size(Process* p)
 {
     Uint sz;
     ErlMessage *msgp;
 
-    if (off_heap_msgs)
-	return 0;
-
     for (sz = 0, msgp = p->msg.first; msgp; msgp = msgp->next) {
 	if (msgp->data.attached)
 	    sz += erts_msg_attached_data_size(msgp);
@@ -2666,7 +2658,7 @@ reply_gc_info(void *vgcirp)
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
 
-    erts_smp_proc_dec_refc(rp);
+    erts_proc_dec_refc(rp);
 
     if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0)
 	gcireq_free(vgcirp);
@@ -2690,7 +2682,7 @@ erts_gc_info_request(Process *c_p)
     erts_smp_atomic32_init_nob(&gcirp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
-    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+    erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
 
 #ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
new file mode 100644
index 0000000000..2245799819
--- /dev/null
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -0,0 +1,2901 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2015. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: High level timers implementing BIF timers
+ *              as well as process and port timers.
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "global.h"
+#include "bif.h"
+#include "erl_bif_unique.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
+#include "erl_hl_timer.h"
+
+#define ERTS_TMR_CHECK_CANCEL_ON_CREATE 0
+
+#if 0
+#  define ERTS_HLT_HARD_DEBUG
+#endif
+#if 0
+#  define ERTS_HLT_DEBUG
+#endif
+
+#if defined(ERTS_HLT_HARD_DEBUG) || defined(DEBUG)
+#  if defined(ERTS_HLT_HARD_DEBUG)
+#    undef ERTS_RBT_HARD_DEBUG
+#    define ERTS_RBT_HARD_DEBUG 1
+#  endif
+#  ifndef ERTS_HLT_DEBUG
+#    define ERTS_HLT_DEBUG 1
+#  endif
+#endif
+
+#undef ERTS_HLT_ASSERT
+#if defined(ERTS_HLT_DEBUG)
+#  define ERTS_HLT_ASSERT(E) ERTS_ASSERT(E)
+#  undef ERTS_RBT_DEBUG
+#  define ERTS_RBT_DEBUG
+#else
+#  define ERTS_HLT_ASSERT(E) ((void) 1)
+#endif
+
+#if defined(ERTS_HLT_HARD_DEBUG) && defined(__GNUC__)
+#warning "* * * * * * * * * * * * * * * * * *"
+#warning "* ERTS_HLT_HARD_DEBUG IS ENABLED! *"
+#warning "* * * * * * * * * * * * * * * * * *"
+#endif
+
+#ifdef ERTS_HLT_HARD_DEBUG
+#  define ERTS_HLT_HDBG_CHK_SRV(SRV) hdbg_chk_srv((SRV))
+static void hdbg_chk_srv(ErtsHLTimerService *srv);
+#else
+#  define ERTS_HLT_HDBG_CHK_SRV(SRV) ((void) 1)
+#endif
+
+#if ERTS_REF_NUMBERS != 3
+#error "ERTS_REF_NUMBERS changed. Update me..."
+#endif
+
+#define ERTS_BIF_TIMER_SHORT_TIME 5000
+
+#ifdef ERTS_SMP
+#  define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore \
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore)
+#else
+#  define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore
+#endif
+
+/* Bit 0 to 9 contains scheduler id (see mask below) */
+#define ERTS_TMR_ROFLG_HLT		(((Uint32) 1) << 10)
+#define ERTS_TMR_ROFLG_BIF_TMR		(((Uint32) 1) << 11)
+#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 12)
+#define ERTS_TMR_ROFLG_PRE_ALC		(((Uint32) 1) << 13)
+#define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 14)
+#define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 15)
+#define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 16)
+
+#define ERTS_TMR_ROFLG_SID_MASK	\
+    (ERTS_TMR_ROFLG_HLT - (Uint32) 1)
+
+#define ERTS_TMR_STATE_ACTIVE		((erts_aint32_t) 0)
+#define ERTS_TMR_STATE_CANCELED		((erts_aint32_t) 1)
+#define ERTS_TMR_STATE_TIMED_OUT	((erts_aint32_t) 2)
+
+typedef struct ErtsHLTimer_ ErtsHLTimer;
+
+#define ERTS_HLT_PFLG_RED		(((UWord) 1) << 0)
+#define ERTS_HLT_PFLG_SAME_TIME		(((UWord) 1) << 1)
+
+#define ERTS_HLT_PFLGS_MASK \
+    (ERTS_HLT_PFLG_RED|ERTS_HLT_PFLG_SAME_TIME)
+
+#define ERTS_HLT_PFIELD_NOT_IN_TABLE	(~((UWord) 0))
+
+typedef struct {
+    UWord parent; /* parent pointer and flags... */
+    union {
+	struct {
+	    ErtsHLTimer *right;
+	    ErtsHLTimer *left;
+	} t;
+	struct {
+	    ErtsHLTimer *prev;
+	    ErtsHLTimer *next;
+	} l;
+    } u;
+    ErtsHLTimer *same_time;
+} ErtsHLTimerTimeTree;
+
+typedef struct {
+    UWord parent; /* parent pointer and flags... */
+    ErtsHLTimer *right;
+    ErtsHLTimer *left;
+} ErtsHLTimerTree;
+
+typedef struct {
+    Uint32 roflgs;
+    erts_smp_atomic32_t refc;
+    union {
+	erts_atomic_t next;
+    } u;
+} ErtsTmrHead;
+
+struct ErtsHLTimer_ {
+    ErtsTmrHead head; /* NEED to be first! */
+    union {
+	ErtsThrPrgrLaterOp cleanup;
+	ErtsHLTimerTimeTree tree;
+    } time;
+    ErtsMonotonicTime timeout;
+    union {
+	Process *proc;
+	Port *port;
+	Eterm name;
+    } receiver;
+
+#ifdef ERTS_HLT_HARD_DEBUG
+    int pending_timeout;
+#endif
+
+    erts_smp_atomic32_t state;
+
+    /* BIF timer only fields follow... */
+    struct {
+	Uint32 refn[ERTS_REF_NUMBERS];
+	ErtsHLTimerTree proc_tree;
+	ErtsHLTimerTree tree;
+	Eterm message;
+	ErlHeapFragment *bp;
+    } btm;
+    struct {
+	Eterm accessor;
+	ErtsHLTimerTree tree;
+    } abtm;
+};
+
+#define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm)
+#define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm)
+#define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer)
+
+typedef struct {
+    ErtsTmrHead head; /* NEED to be first! */
+    void *p;
+    ErtsTWheelTimer tw_tmr;
+} ErtsTWTimer;
+
+typedef union {
+    ErtsTmrHead head;
+    ErtsHLTimer hlt;
+    ErtsTWTimer twt;
+} ErtsTimer;
+
+#ifdef SMALL_MEMORY
+#define BIF_TIMER_PREALC_SZ	10
+#define PTIMER_PREALC_SZ	10
+#else
+#define BIF_TIMER_PREALC_SZ	100
+#define PTIMER_PREALC_SZ	100
+#endif
+
+ERTS_SCHED_PREF_PALLOC_IMPL(bif_timer_pre,
+			    ErtsHLTimer,
+			    BIF_TIMER_PREALC_SZ)
+
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(tw_timer,
+				 ErtsTWTimer,
+				 PTIMER_PREALC_SZ,
+				 ERTS_ALC_T_LL_PTIMER)
+
+#ifdef ERTS_HLT_DEBUG
+#define ERTS_TMR_TIMEOUT_YIELD_LIMIT 5
+#else
+#define ERTS_TMR_TIMEOUT_YIELD_LIMIT 100
+#endif
+#define ERTS_TMR_CANCELED_TIMER_LIMIT 100
+#define ERTS_TMR_CANCELED_TIMER_SMALL_LIMIT 5
+
+#define ERTS_TMR_TIMEOUT_YIELD_STATE_T same_time_list_yield_state_t
+#define ERTS_TMR_YIELDING_TIMEOUT_STATE_INITER {NULL, {0}}
+typedef struct {
+    int dummy;
+} ERTS_TMR_TIMEOUT_YIELD_STATE_T;
+
+typedef struct {
+    ErtsTmrHead marker;
+    erts_atomic_t last;
+} ErtsHLTCncldTmrQTail;
+
+#ifdef ERTS_SMP
+
+typedef struct {
+    /*
+     * This structure needs to be cache line aligned for best
+     * performance.
+     */
+    union {
+	/*
+	 * Modified by threads returning canceled
+	 * timers to this timer service.
+	 */
+	ErtsHLTCncldTmrQTail data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
+		sizeof(ErtsHLTCncldTmrQTail))];
+    } tail;
+    /*
+     * Everything below this point is *only* accessed by the
+     * thread managing this timer service.
+     */
+    struct {
+	ErtsTimer *first;
+	ErtsTimer *unref_end;
+	struct {
+	    ErtsThrPrgrVal thr_progress;
+	    int thr_progress_reached;
+	    ErtsTimer *unref_end;
+	} next;
+	int used_marker;
+    } head;
+} ErtsHLTCncldTmrQ;
+
+#endif /* ERTS_SMP */
+
+typedef struct {
+    ErtsHLTimer *root;
+    ERTS_TMR_TIMEOUT_YIELD_STATE_T state;
+} ErtsYieldingTimeoutState;
+
+struct ErtsHLTimerService_ {
+#ifdef ERTS_SMP
+    ErtsHLTCncldTmrQ canceled_queue;
+#endif
+    ErtsHLTimer *time_tree;
+    ErtsHLTimer *btm_tree;
+    ErtsHLTimer *next_timeout;
+    ErtsYieldingTimeoutState yield;
+    ErtsTWheelTimer service_timer;
+};
+
+static ERTS_INLINE int
+refn_is_lt(Uint32 *x, Uint32 *y)
+{
+    /* !0 if x < y */
+    if (x[2] < y[2])
+	return 1;
+    if (x[2] != y[2])
+	return 0;
+    if (x[1] < y[1])
+	return 1;
+    if (x[1] != y[1])
+	return 0;
+    return x[0] < y[0];
+}
+
+#define ERTS_RBT_PREFIX time
+#define ERTS_RBT_T ErtsHLTimer
+#define ERTS_RBT_KEY_T ErtsMonotonicTime
+#define ERTS_RBT_FLAGS_T UWord
+#define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
+    do {								\
+	(T)->time.tree.parent = (UWord) NULL;				\
+	(T)->time.tree.u.t.right = NULL;				\
+	(T)->time.tree.u.t.left = NULL;					\
+    } while (0)
+#define ERTS_RBT_IS_RED(T)						\
+    ((int) ((T)->time.tree.parent & ERTS_HLT_PFLG_RED))
+#define ERTS_RBT_SET_RED(T)						\
+    ((T)->time.tree.parent |= ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_IS_BLACK(T)						\
+    (!ERTS_RBT_IS_RED((T)))
+#define ERTS_RBT_SET_BLACK(T)						\
+    ((T)->time.tree.parent &= ~ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_GET_FLAGS(T)						\
+    ((T)->time.tree.parent & ERTS_HLT_PFLGS_MASK)
+#define ERTS_RBT_SET_FLAGS(T, F)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->time.tree.parent &= ~ERTS_HLT_PFLGS_MASK;			\
+	(T)->time.tree.parent |= (F);					\
+    } while (0)
+#define ERTS_RBT_GET_PARENT(T)						\
+    ((ErtsHLTimer *) ((T)->time.tree.parent & ~ERTS_HLT_PFLGS_MASK))
+#define ERTS_RBT_SET_PARENT(T, P)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->time.tree.parent &= ERTS_HLT_PFLGS_MASK;			\
+	(T)->time.tree.parent |= (UWord) (P);				\
+    } while (0)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->time.tree.u.t.right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->time.tree.u.t.right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->time.tree.u.t.left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->time.tree.u.t.left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->timeout)
+#define ERTS_RBT_IS_LT(KX, KY) ((KX) < (KY))
+#define ERTS_RBT_IS_EQ(KX, KY) ((KX) == (KY))
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_WANT_SMALLEST
+#define ERTS_RBT_WANT_LOOKUP_INSERT
+#define ERTS_RBT_WANT_REPLACE
+#ifdef ERTS_HLT_HARD_DEBUG
+#  define ERTS_RBT_WANT_FOREACH
+#  define ERTS_RBT_WANT_LOOKUP
+#endif
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+/* Use circular list for timers at same time */
+
+static ERTS_INLINE void
+same_time_list_insert(ErtsHLTimer **root, ErtsHLTimer *tmr)
+{
+    ErtsHLTimer *first = *root;
+    if (!first) {
+	ERTS_HLT_ASSERT((((UWord) root) & ERTS_HLT_PFLG_SAME_TIME) == 0);
+	tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME;
+	tmr->time.tree.u.l.next = tmr;
+	tmr->time.tree.u.l.prev = tmr;
+	*root = tmr;
+    }
+    else {
+	tmr->time.tree.parent = ERTS_HLT_PFLG_SAME_TIME;
+	tmr->time.tree.u.l.next = first;
+	tmr->time.tree.u.l.prev = first->time.tree.u.l.prev;
+	first->time.tree.u.l.prev = tmr;
+	tmr->time.tree.u.l.prev->time.tree.u.l.next = tmr;
+    }
+}
+
+static ERTS_INLINE void
+same_time_list_delete(ErtsHLTimer *tmr)
+{
+    ErtsHLTimer **root, *next;
+    
+    root = (ErtsHLTimer **) (tmr->time.tree.parent & ~ERTS_HLT_PFLG_SAME_TIME);
+    next = tmr->time.tree.u.l.next;
+
+    ERTS_HLT_ASSERT((tmr->time.tree.parent
+		     == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME))
+		    || (tmr->time.tree.parent
+			== ERTS_HLT_PFLG_SAME_TIME));
+
+    if (next == tmr) {
+	ERTS_HLT_ASSERT(root && *root == tmr);
+	ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev == tmr);
+	*root = NULL;
+    }
+    else {
+	if (root) {
+	    ERTS_HLT_ASSERT(*root == tmr);
+	    *root = next;
+	    next->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME;
+	}
+	tmr->time.tree.u.l.next->time.tree.u.l.prev = tmr->time.tree.u.l.prev;
+	tmr->time.tree.u.l.prev->time.tree.u.l.next = next;
+    }
+}
+
+static ERTS_INLINE void
+same_time_list_new_root(ErtsHLTimer **root)
+{
+    ErtsHLTimer *tmr = *root;
+    if (tmr) {
+	ERTS_HLT_ASSERT(root);
+	tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME;
+    }
+}
+
+static ERTS_INLINE int
+same_time_list_foreach_destroy_yielding(ErtsHLTimer **root,
+					void (*op)(ErtsHLTimer *, void *),
+					void *arg,
+					ERTS_TMR_TIMEOUT_YIELD_STATE_T *ys,
+					Sint ylimit)
+{
+    Sint ycnt = ylimit;
+    ErtsHLTimer *end, *tmr = *root;
+    if (!tmr)
+	return 0;
+
+    ERTS_HLT_ASSERT(tmr->time.tree.parent
+		    == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME));
+
+    end = tmr->time.tree.u.l.prev;
+    end->time.tree.u.l.next = NULL;
+
+    while (1) {
+	ErtsHLTimer *op_tmr = tmr;
+
+	ERTS_HLT_ASSERT((tmr->time.tree.parent
+			 == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME))
+			|| (tmr->time.tree.parent
+			    == ERTS_HLT_PFLG_SAME_TIME));
+
+	tmr = tmr->time.tree.u.l.next;
+	(*op)(op_tmr, arg);
+	if (!tmr) {
+	    *root = NULL;
+	    return 0;
+	}
+	if (--ycnt <= 0) {
+	    /* Make new circle of timers left to process... */
+	    *root = tmr;
+	    end->time.tree.u.l.next = tmr;
+	    tmr->time.tree.u.l.prev = end;
+	    tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME;
+	    return 1;
+	}
+    }    
+}
+
+#ifdef ERTS_HLT_HARD_DEBUG
+
+static ERTS_INLINE void
+same_time_list_foreach(ErtsHLTimer *root,
+		       void (*op)(ErtsHLTimer *, void *),
+		       void *arg)
+{
+    if (root) {
+	ErtsHLTimer *tmr = root;
+	do {
+	    (*op)(tmr, arg);
+	    tmr = tmr->time.tree.u.l.next;
+	} while (root != tmr);
+    }
+}
+
+static ERTS_INLINE ErtsHLTimer *
+same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
+{
+    if (root) {
+	ErtsHLTimer *tmr = root;
+	do {
+	    if (tmr == x)
+		return tmr;
+	    tmr = tmr->time.tree.u.l.next;
+	} while (root != tmr);
+    }
+    return NULL;
+}
+
+#endif /* ERTS_HLT_HARD_DEBUG */
+
+#define ERTS_RBT_PREFIX btm
+#define ERTS_RBT_T ErtsHLTimer
+#define ERTS_RBT_KEY_T Uint32 *
+#define ERTS_RBT_FLAGS_T UWord
+#define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
+    do {								\
+	(T)->btm.tree.parent = (UWord) NULL;				\
+	(T)->btm.tree.right = NULL;					\
+	(T)->btm.tree.left = NULL;					\
+    } while (0)
+#define ERTS_RBT_IS_RED(T)						\
+    ((int) ((T)->btm.tree.parent & ERTS_HLT_PFLG_RED))
+#define ERTS_RBT_SET_RED(T)						\
+    ((T)->btm.tree.parent |= ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_IS_BLACK(T)						\
+    (!ERTS_RBT_IS_RED((T)))
+#define ERTS_RBT_SET_BLACK(T)						\
+    ((T)->btm.tree.parent &= ~ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_GET_FLAGS(T)						\
+    ((T)->btm.tree.parent & ERTS_HLT_PFLGS_MASK)
+#define ERTS_RBT_SET_FLAGS(T, F)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->btm.tree.parent &= ~ERTS_HLT_PFLGS_MASK;			\
+	(T)->btm.tree.parent |= (F);					\
+    } while (0)
+#define ERTS_RBT_GET_PARENT(T)						\
+    ((ErtsHLTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK))
+#define ERTS_RBT_SET_PARENT(T, P)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->btm.tree.parent &= ERTS_HLT_PFLGS_MASK;			\
+	(T)->btm.tree.parent |= (UWord) (P);				\
+    } while (0)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->btm.tree.right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.tree.right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->btm.tree.left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.tree.left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn)
+#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY))
+#define ERTS_RBT_IS_EQ(KX, KY) \
+    (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2]))
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_WANT_FOREACH
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+#define ERTS_RBT_PREFIX proc_btm
+#define ERTS_RBT_T ErtsHLTimer
+#define ERTS_RBT_KEY_T Uint32 *
+#define ERTS_RBT_FLAGS_T UWord
+#define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
+    do {								\
+	(T)->btm.proc_tree.parent = (UWord) NULL;			\
+	(T)->btm.proc_tree.right = NULL;				\
+	(T)->btm.proc_tree.left = NULL;					\
+    } while (0)
+#define ERTS_RBT_IS_RED(T)						\
+    ((int) ((T)->btm.proc_tree.parent & ERTS_HLT_PFLG_RED))
+#define ERTS_RBT_SET_RED(T)						\
+    ((T)->btm.proc_tree.parent |= ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_IS_BLACK(T)						\
+    (!ERTS_RBT_IS_RED((T)))
+#define ERTS_RBT_SET_BLACK(T)						\
+    ((T)->btm.proc_tree.parent &= ~ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_GET_FLAGS(T)						\
+    ((T)->btm.proc_tree.parent & ERTS_HLT_PFLGS_MASK)
+#define ERTS_RBT_SET_FLAGS(T, F)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->btm.proc_tree.parent &= ~ERTS_HLT_PFLGS_MASK;		\
+	(T)->btm.proc_tree.parent |= (F);				\
+    } while (0)
+#define ERTS_RBT_GET_PARENT(T)						\
+    ((ErtsHLTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK))
+#define ERTS_RBT_SET_PARENT(T, P)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->btm.proc_tree.parent &= ERTS_HLT_PFLGS_MASK;		\
+	(T)->btm.proc_tree.parent |= (UWord) (P);			\
+    } while (0)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->btm.proc_tree.right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.proc_tree.right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->btm.proc_tree.left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.proc_tree.left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn)
+#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY))
+#define ERTS_RBT_IS_EQ(KX, KY) \
+    (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2]))
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+#define ERTS_RBT_PREFIX abtm
+#define ERTS_RBT_T ErtsHLTimer
+#define ERTS_RBT_KEY_T Uint32 *
+#define ERTS_RBT_FLAGS_T UWord
+#define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
+    do {								\
+	(T)->abtm.tree.parent = (UWord) NULL;				\
+	(T)->abtm.tree.right = NULL;					\
+	(T)->abtm.tree.left = NULL;					\
+    } while (0)
+#define ERTS_RBT_IS_RED(T)						\
+    ((int) ((T)->abtm.tree.parent & ERTS_HLT_PFLG_RED))
+#define ERTS_RBT_SET_RED(T)						\
+    ((T)->abtm.tree.parent |= ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_IS_BLACK(T)						\
+    (!ERTS_RBT_IS_RED((T)))
+#define ERTS_RBT_SET_BLACK(T)						\
+    ((T)->abtm.tree.parent &= ~ERTS_HLT_PFLG_RED)
+#define ERTS_RBT_GET_FLAGS(T)						\
+    ((T)->abtm.tree.parent & ERTS_HLT_PFLGS_MASK)
+#define ERTS_RBT_SET_FLAGS(T, F)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->abtm.tree.parent &= ~ERTS_HLT_PFLGS_MASK;			\
+	(T)->abtm.tree.parent |= (F);					\
+    } while (0)
+#define ERTS_RBT_GET_PARENT(T)						\
+    ((ErtsHLTimer *) ((T)->abtm.tree.parent & ~ERTS_HLT_PFLGS_MASK))
+#define ERTS_RBT_SET_PARENT(T, P)					\
+    do {								\
+	ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0);	\
+	(T)->abtm.tree.parent &= ERTS_HLT_PFLGS_MASK;			\
+	(T)->abtm.tree.parent |= (UWord) (P);				\
+    } while (0)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->abtm.tree.right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->abtm.tree.right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->abtm.tree.left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->abtm.tree.left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn)
+#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY))
+#define ERTS_RBT_IS_EQ(KX, KY) \
+    (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2]))
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+#ifdef ERTS_SMP
+static void init_canceled_queue(ErtsHLTCncldTmrQ *cq);
+#endif
+
+void
+erts_hl_timer_init(void)
+{
+    init_tw_timer_alloc();
+    init_bif_timer_pre_alloc();
+}
+
+ErtsHLTimerService *
+erts_create_timer_service(void)
+{
+    ErtsYieldingTimeoutState init_yield = ERTS_TMR_YIELDING_TIMEOUT_STATE_INITER;
+    ErtsHLTimerService *srv;
+
+    srv = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_SERVICE,
+					     sizeof(ErtsHLTimerService));
+    srv->time_tree = NULL;
+    srv->btm_tree = NULL;
+    srv->next_timeout = NULL;
+    srv->yield = init_yield;
+    erts_twheel_init_timer(&srv->service_timer);
+
+#ifdef ERTS_SMP
+    init_canceled_queue(&srv->canceled_queue);
+#endif
+
+    return srv;
+}
+
+size_t
+erts_timer_type_size(ErtsAlcType_t type)
+{
+    switch (type) {
+    case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer);
+    case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE;
+    case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE;
+    case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE;
+    default: ERTS_INTERNAL_ERROR("Unknown type");
+    }
+    return 0;
+}
+
+static ERTS_INLINE ErtsMonotonicTime
+get_timeout_pos(ErtsMonotonicTime now, ErtsMonotonicTime msec)
+{
+    ErtsMonotonicTime timeout_pos;
+    if (msec <= 0)
+	return ERTS_MONOTONIC_TO_CLKTCKS(now);
+    timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(now-1);
+    timeout_pos += ERTS_MSEC_TO_CLKTCKS(msec) + 1;
+    return timeout_pos;
+}
+
+static ERTS_INLINE Sint64
+get_time_left(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos)
+{
+    ErtsMonotonicTime now = erts_get_monotonic_time(esdp);
+
+    now = ERTS_MONOTONIC_TO_CLKTCKS(now-1)+1;
+    if (timeout_pos <= now)
+	return (Sint64) 0;
+    return (Sint64) ERTS_CLKTCKS_TO_MSEC(timeout_pos - now);
+}
+
+static ERTS_INLINE int
+proc_timeout_common(Process *proc, void *tmr)
+{
+    if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&proc->common.timer,
+						   ERTS_PTMR_TIMEDOUT,
+						   (erts_aint_t) tmr)) {
+	erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state);
+	if (!(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_EXITING)))
+	    erts_schedule_process(proc, state, 0);
+	return 1;
+    }
+    return 0;
+}
+
+static ERTS_INLINE int
+port_timeout_common(Port *port, void *tmr)
+{
+    if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&port->common.timer,
+						   ERTS_PTMR_TIMEDOUT,
+						   (erts_aint_t) tmr)) {
+	erts_port_task_schedule(port->common.id,
+				&port->timeout_task,
+				ERTS_PORT_TASK_TIMEOUT);
+	return 1;
+    }
+    return 0;
+}
+
+/*
+ * Basic timer wheel timer stuff
+ */
+
+static void
+scheduled_tw_timer_destroy(void *vtmr)
+{
+    tw_timer_free((ErtsTWTimer *) vtmr);
+}
+
+static void
+schedule_tw_timer_destroy(ErtsTWTimer *tmr)
+{
+    /*
+     * Reference to process/port can be
+     * dropped at once...
+     */
+    if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC)
+	erts_proc_dec_refc((Process *) tmr->p);
+    else
+	erts_port_dec_refc((Port *) tmr->p);
+
+    erts_schedule_thr_prgr_later_cleanup_op(
+	scheduled_tw_timer_destroy,
+	(void *) tmr,
+	&tmr->tw_tmr.u.cleanup,
+	sizeof(ErtsTWTimer));
+}
+
+static ERTS_INLINE void
+tw_timer_dec_refc(ErtsTWTimer *tmr)
+{
+    if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
+	ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore;
+	schedule_tw_timer_destroy(tmr);
+    }
+}
+
+static void
+tw_proc_timeout(void *vtwtp)
+{
+    ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
+    Process *proc = (Process *) twtp->p;
+    if (proc_timeout_common(proc, vtwtp))
+	tw_timer_dec_refc(twtp);
+    tw_timer_dec_refc(twtp);
+}
+
+static void
+tw_port_timeout(void *vtwtp)
+{
+    ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
+    Port *port = (Port *) twtp->p;
+    if (port_timeout_common(port, vtwtp))
+	tw_timer_dec_refc(twtp);
+    tw_timer_dec_refc(twtp);
+}
+
+static void
+tw_ptimer_cancel(void *vtwtp)
+{
+    tw_timer_dec_refc((ErtsTWTimer *) vtwtp);    
+}
+
+static void
+cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr)
+{
+    ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK)
+		    == (Uint32) esdp->no);
+    erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr);
+}
+
+static ErtsTWTimer *
+create_tw_timer(ErtsSchedulerData *esdp, 
+		void *p, int is_proc,
+		ErtsMonotonicTime timeout_pos)
+{
+    ErtsTWTimer *tmr;
+    void (*timeout_func)(void *);
+
+    tmr = tw_timer_alloc();
+    erts_twheel_init_timer(&tmr->tw_tmr);
+
+    tmr->head.roflgs = (Uint32) esdp->no;
+    ERTS_HLT_ASSERT((tmr->head.roflgs
+		     & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
+    tmr->p = p;
+    if (is_proc) {
+	tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
+	timeout_func = tw_proc_timeout;
+	erts_proc_inc_refc((Process *) p);
+    }
+    else {
+	tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT;
+	timeout_func = tw_port_timeout;
+	erts_port_inc_refc((Port *) p);
+    }
+
+    erts_smp_atomic32_init_nob(&tmr->head.refc, 2);
+
+    erts_twheel_set_timer(esdp->timer_wheel,
+			  &tmr->tw_tmr,
+			  timeout_func,
+			  tw_ptimer_cancel,
+			  tmr,
+			  timeout_pos);
+
+    return tmr;
+}
+
+/*
+ * Basic high level timer stuff
+ */
+
+static ERTS_INLINE void
+hl_timer_destroy(ErtsHLTimer *tmr)
+{
+    Uint32 roflgs = tmr->head.roflgs;
+    if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR))
+	erts_free(ERTS_ALC_T_HL_PTIMER, tmr);
+    else {
+	if (tmr->btm.bp)
+	    free_message_buffer(tmr->btm.bp);
+	if (roflgs & ERTS_TMR_ROFLG_PRE_ALC)
+	    bif_timer_pre_free(tmr);
+	else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
+	    erts_free(ERTS_ALC_T_ABIF_TIMER, tmr);
+	else
+	    erts_free(ERTS_ALC_T_BIF_TIMER, tmr);
+    }
+}
+
+static void
+scheduled_hl_timer_destroy(void *vtmr)
+{
+    hl_timer_destroy((ErtsHLTimer *) vtmr);
+}
+
+static void
+schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
+{
+    UWord size;
+
+    /*
+     * Reference to process/port can be dropped
+     * at once...
+     */
+
+    ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0);
+
+    if (roflgs & ERTS_TMR_ROFLG_REG_NAME) {
+	ERTS_HLT_ASSERT(is_atom(tmr->receiver.name));
+    }
+    else if (roflgs & ERTS_TMR_ROFLG_PROC) {
+	ERTS_HLT_ASSERT(tmr->receiver.proc);
+	erts_proc_dec_refc(tmr->receiver.proc);
+    }
+    else if (roflgs & ERTS_TMR_ROFLG_PORT) {
+	ERTS_HLT_ASSERT(tmr->receiver.port);
+	erts_port_dec_refc(tmr->receiver.port);
+    }
+
+    if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR))
+	size = ERTS_HL_PTIMER_SIZE;
+    else {
+	/*
+	 * Message buffer can be dropped at
+	 * once...
+	 */
+	if (tmr->btm.bp) {
+	    free_message_buffer(tmr->btm.bp);
+	    tmr->btm.bp = NULL;
+	}
+	size = sizeof(ErtsHLTimer);
+    }
+
+    erts_schedule_thr_prgr_later_cleanup_op(
+	scheduled_hl_timer_destroy, tmr,
+	&tmr->time.cleanup, size);
+}
+
+static ERTS_INLINE void
+hl_timer_pre_dec_refc(ErtsHLTimer *tmr)
+{
+#ifdef ERTS_HLT_DEBUG
+    erts_aint_t refc;
+    refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc);
+    ERTS_HLT_ASSERT(refc > 0);
+#else
+    erts_smp_atomic32_dec_nob(&tmr->head.refc);
+#endif
+}
+
+static ERTS_INLINE void
+hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs)
+{
+    if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
+	ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore;
+	schedule_hl_timer_destroy(tmr, roflgs);
+    }
+}
+
+static void hlt_service_timeout(void *vesdp);
+#ifdef ERTS_SMP
+static void handle_canceled_queue(ErtsSchedulerData *esdp,
+				  ErtsHLTCncldTmrQ *cq,
+				  int use_limit,
+				  int ops_limit,
+				  int *need_thr_progress,
+				  ErtsThrPrgrVal *thr_prgr_p,
+				  int *need_more_work);
+#endif
+
+static ERTS_INLINE void
+check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
+{
+#if defined(ERTS_SMP) && ERTS_TMR_CHECK_CANCEL_ON_CREATE
+    ErtsHLTCncldTmrQ *cq = &srv->canceled_queue;
+    if (cq->head.first != cq->head.unref_end)
+	handle_canceled_queue(esdp, cq, 1,
+			      ERTS_TMR_CANCELED_TIMER_SMALL_LIMIT,
+			      NULL, NULL, NULL);
+#endif
+}
+
+static void
+hlt_delete_abtm(ErtsHLTimer *tmr)
+{
+    Process *proc;
+
+    ERTS_HLT_ASSERT(tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR);
+
+    proc = erts_proc_lookup(tmr->abtm.accessor);
+
+    if (proc) {
+	int deref = 0;
+	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
+	if (tmr->abtm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	    abtm_rbt_delete(&proc->accessor_bif_timers, tmr);
+	    deref = 1;
+	    tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+	}
+	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+	if (deref)
+	    hl_timer_pre_dec_refc(tmr);
+    }
+}
+
+static ErtsHLTimer *
+create_hl_timer(ErtsSchedulerData *esdp,
+		ErtsMonotonicTime timeout_pos,
+		int short_time, int is_bif_tmr,
+		void *rcvrp, Eterm rcvr, Eterm acsr,
+		Eterm msg, Uint32 *refn)
+{
+    ErtsHLTimerService *srv = esdp->timer_service;
+    ErtsHLTimer *tmr, *st_tmr;
+    erts_aint32_t refc;
+    Uint32 roflgs;
+    int is_abif_tmr = is_bif_tmr && is_value(acsr) && acsr != rcvr;
+
+    check_canceled_queue(esdp, srv);
+
+    ERTS_HLT_ASSERT((esdp->no & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
+
+    roflgs = ((Uint32) esdp->no) | ERTS_TMR_ROFLG_HLT;
+
+    if (!is_bif_tmr)
+	tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER,
+			 ERTS_HL_PTIMER_SIZE);
+    else if (short_time) {
+	tmr = bif_timer_pre_alloc();
+	if (!tmr)
+	    goto alloc_bif_timer;
+	roflgs |= ERTS_TMR_ROFLG_PRE_ALC;
+    }
+    else {
+    alloc_bif_timer:
+	if (is_abif_tmr)
+	    tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER,
+			     ERTS_ABIF_TIMER_SIZE);
+	else
+	    tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
+			     ERTS_BIF_TIMER_SIZE);
+    }
+
+    tmr->timeout = timeout_pos;
+
+    if (!is_bif_tmr) {
+	if (is_internal_pid(rcvr)) {
+	    erts_proc_inc_refc((Process *) rcvrp);
+	    tmr->receiver.proc = (Process *) rcvrp;
+	    roflgs |= ERTS_TMR_ROFLG_PROC;
+	}
+	else {
+	    erts_port_inc_refc((Port *) rcvrp);
+	    ERTS_HLT_ASSERT(is_internal_port(rcvr));
+	    tmr->receiver.port = (Port *) rcvrp;
+	    roflgs |= ERTS_TMR_ROFLG_PORT;
+	}
+	refc = 2;
+    }
+    else {
+	Uint hsz;
+
+	roflgs |= ERTS_TMR_ROFLG_BIF_TMR;
+	if (is_internal_pid(rcvr)) {
+	    roflgs |= ERTS_TMR_ROFLG_PROC;
+	    tmr->receiver.proc = (Process *) rcvrp;
+	    refc = 2;
+	}
+	else {
+	    ERTS_HLT_ASSERT(is_atom(rcvr));
+	    roflgs |= ERTS_TMR_ROFLG_REG_NAME;
+	    tmr->receiver.name = rcvr;
+	    refc = 1;
+	}
+
+	hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg);
+	if (!hsz) {
+	    tmr->btm.message = msg;
+	    tmr->btm.bp = NULL;
+	}
+	else {
+	    ErlHeapFragment *bp = new_message_buffer(hsz);
+	    Eterm *hp = bp->mem;
+	    tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap);
+	    tmr->btm.bp = bp;
+	}
+	tmr->btm.refn[0] = refn[0];
+	tmr->btm.refn[1] = refn[1];
+	tmr->btm.refn[2] = refn[2];
+
+	tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+	if (is_abif_tmr) {
+	    Process *aproc;
+	    roflgs |= ERTS_TMR_ROFLG_ABIF_TMR;
+	    tmr->abtm.accessor = acsr;
+	    aproc = erts_proc_lookup(acsr);
+	    if (!aproc)
+		tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+	    else {
+		refc++;
+		erts_smp_proc_lock(aproc, ERTS_PROC_LOCK_BTM);
+		abtm_rbt_insert(&aproc->accessor_bif_timers, tmr);
+		erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM);
+	    }
+	}
+    }
+
+    tmr->head.roflgs = roflgs;
+    erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
+    erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE);
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
+    if (!srv->next_timeout
+	|| tmr->timeout < srv->next_timeout->timeout) {
+	if (srv->next_timeout)
+	    erts_twheel_cancel_timer(esdp->timer_wheel,
+				     &srv->service_timer);
+	erts_twheel_set_timer(esdp->timer_wheel,
+			      &srv->service_timer,
+			      hlt_service_timeout,
+			      NULL,
+			      (void *) esdp,
+			      tmr->timeout);
+	srv->next_timeout = tmr;
+    }
+
+    st_tmr = time_rbt_lookup_insert(&srv->time_tree, tmr);
+    tmr->time.tree.same_time = st_tmr;
+    if (st_tmr)
+	same_time_list_insert(&st_tmr->time.tree.same_time, tmr);
+
+    if (is_bif_tmr)
+	btm_rbt_insert(&srv->btm_tree, tmr);
+
+#ifdef ERTS_HLT_HARD_DEBUG
+    tmr->pending_timeout = 0;
+#endif
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
+    return tmr;
+}
+
+static ERTS_INLINE void
+hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
+{
+    ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND;
+    Process *proc;
+    int dec_refc = 0;
+    Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
+    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
+
+    if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
+	hlt_delete_abtm(tmr);
+
+    if (is_reg_name) {
+	Eterm pid;
+	ERTS_HLT_ASSERT(is_atom(tmr->receiver.name));
+	pid = erts_whereis_name_to_id(NULL, tmr->receiver.name);
+	proc = erts_proc_lookup(pid);
+    }
+    else {
+	ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC);
+	ERTS_HLT_ASSERT(tmr->receiver.proc);
+
+	proc = tmr->receiver.proc;
+	proc_locks |= ERTS_PROC_LOCK_BTM;
+    }
+    if (proc) {
+	erts_smp_proc_lock(proc, proc_locks);
+	/*
+	 * If process is exiting, let it clean up
+	 * the btm tree by itself (it may be in
+	 * the middle of tree destruction).
+	 */
+	if (!ERTS_PROC_IS_EXITING(proc)) {
+	    erts_queue_message(proc,
+			       &proc_locks,
+			       tmr->btm.bp,
+			       tmr->btm.message,
+			       NIL
+#ifdef USE_VM_PROBES
+			       , NIL
+#endif
+		);
+	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND);
+	    proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND;
+	    tmr->btm.bp = NULL;
+	    if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+		proc_btm_rbt_delete(&proc->bif_timers, tmr);
+		tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+		dec_refc = 1;
+	    }
+	}
+	if (proc_locks)
+	    erts_smp_proc_unlock(proc, proc_locks);
+	if (dec_refc)
+	    hl_timer_pre_dec_refc(tmr);
+    }
+}
+
+static ERTS_INLINE void
+hlt_proc_timeout(ErtsHLTimer *tmr)
+{
+    if (proc_timeout_common(tmr->receiver.proc, (void *) tmr))
+	hl_timer_dec_refc(tmr, tmr->head.roflgs);
+}
+
+static ERTS_INLINE void
+hlt_port_timeout(ErtsHLTimer *tmr)
+{
+    if (port_timeout_common(tmr->receiver.port, (void *) tmr))
+	hl_timer_dec_refc(tmr, tmr->head.roflgs);
+}
+
+static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv)
+{
+    ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv;
+    Uint32 roflgs;
+    erts_aint32_t state;
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
+    roflgs = tmr->head.roflgs;
+    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT);
+
+    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state,
+					   ERTS_TMR_STATE_TIMED_OUT,
+					   ERTS_TMR_STATE_ACTIVE);
+
+    ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED
+		    || state == ERTS_TMR_STATE_ACTIVE);
+
+    if (state == ERTS_TMR_STATE_ACTIVE) {
+
+	if (roflgs & ERTS_TMR_ROFLG_BIF_TMR)
+	    hlt_bif_timer_timeout(tmr, roflgs);
+	else if (roflgs & ERTS_TMR_ROFLG_PROC)
+	    hlt_proc_timeout(tmr);
+	else {
+	    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PORT);
+	    hlt_port_timeout(tmr);
+	}
+    }
+
+    tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+    if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR)
+	&& tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	btm_rbt_delete(&srv->btm_tree, tmr);
+	tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+    }
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
+    hl_timer_dec_refc(tmr, roflgs);
+}
+
+#ifdef ERTS_HLT_HARD_DEBUG
+static void
+set_pending_timeout(ErtsHLTimer *tmr, void *unused)
+{
+    tmr->pending_timeout = -1;
+}
+#endif
+
+static void
+hlt_service_timeout(void *vesdp)
+{
+    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
+    ErtsHLTimerService *srv = esdp->timer_service;
+    ErtsHLTimer *tmr = srv->next_timeout;
+    int yield;
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
+    ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data());
+
+    ERTS_HLT_ASSERT(!srv->yield.root || srv->yield.root == tmr);
+    ERTS_HLT_ASSERT(tmr);
+    ERTS_HLT_ASSERT(tmr->timeout <= erts_get_monotonic_time(esdp));
+
+    if (!srv->yield.root) {
+	ERTS_HLT_ASSERT(tmr->time.tree.parent
+			!= ERTS_HLT_PFIELD_NOT_IN_TABLE);
+	time_rbt_delete(&srv->time_tree, tmr);
+	tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+#ifdef ERTS_HLT_HARD_DEBUG
+	tmr->pending_timeout = 1;
+	if (tmr->time.tree.same_time)
+	    same_time_list_foreach(tmr->time.tree.same_time, set_pending_timeout, NULL);
+#endif
+    }
+
+    if (!tmr->time.tree.same_time && !srv->yield.root)
+	yield = 0;
+    else {
+	yield = same_time_list_foreach_destroy_yielding(
+	    &tmr->time.tree.same_time, hlt_timeout, (void *) srv,
+	    &srv->yield.state, ERTS_TMR_TIMEOUT_YIELD_LIMIT);
+    }
+
+    if (yield)
+	srv->yield.root = tmr;
+    else {
+	srv->yield.root = NULL;
+	hlt_timeout(tmr, (void *) srv);
+
+	tmr = time_rbt_smallest(srv->time_tree);
+	srv->next_timeout = tmr;
+    }
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
+    if (tmr)
+	erts_twheel_set_timer(esdp->timer_wheel,
+			      &srv->service_timer,
+			      hlt_service_timeout,
+			      NULL,
+			      vesdp,
+			      tmr->timeout);
+}
+
+static void
+hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
+{
+    ErtsHLTimerService *srv = esdp->timer_service;
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
+    if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) {
+
+	if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	    btm_rbt_delete(&srv->btm_tree, tmr);
+	    tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+	}
+
+	if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
+	    hlt_delete_abtm(tmr);
+    }
+
+    if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	/* Already removed... */
+	ERTS_HLT_HDBG_CHK_SRV(srv);
+	return;
+    }
+
+    if (tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) {
+	same_time_list_delete(tmr);
+    }
+    else if (tmr->time.tree.same_time) {
+	ErtsHLTimer *st_container;
+
+	ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0);
+	st_container = tmr->time.tree.same_time->time.tree.u.l.prev;
+
+	ERTS_HLT_ASSERT(st_container);
+	ERTS_HLT_ASSERT(st_container->time.tree.parent
+			 & ERTS_HLT_PFLG_SAME_TIME);
+	ERTS_HLT_ASSERT(tmr->timeout == st_container->timeout);
+
+	same_time_list_delete(st_container);
+	st_container->time.tree.same_time = tmr->time.tree.same_time;
+	same_time_list_new_root(&st_container->time.tree.same_time);
+
+	time_rbt_replace(&srv->time_tree, tmr, st_container);
+	ERTS_HLT_ASSERT((st_container->time.tree.parent
+			 & ERTS_HLT_PFLG_SAME_TIME) == 0);
+
+	if (srv->next_timeout == tmr)
+	    srv->next_timeout = st_container;
+    }
+    else {
+	ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0);
+	time_rbt_delete(&srv->time_tree, tmr);
+	if (tmr == srv->next_timeout) {
+	    ErtsHLTimer *smlst;
+	    erts_twheel_cancel_timer(esdp->timer_wheel,
+				     &srv->service_timer);
+	    smlst = time_rbt_smallest(srv->time_tree);
+	    srv->next_timeout = smlst;
+	    if (smlst) {
+		ERTS_HLT_ASSERT(smlst->timeout > tmr->timeout);
+		erts_twheel_set_timer(esdp->timer_wheel,
+				      &srv->service_timer,
+				      hlt_service_timeout,
+				      NULL,
+				      (void *) esdp,
+				      smlst->timeout);
+	    }
+	}
+    }
+    tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+
+    hl_timer_dec_refc(tmr, tmr->head.roflgs);
+
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+}
+
+/*
+ * Pass canceled timers back to originating scheduler
+ */
+
+static ERTS_INLINE void
+cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp,
+				   ErtsTimer *tmr)
+{
+    Uint32 roflgs = tmr->head.roflgs;
+    ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data());
+    ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK)
+		    == (Uint32) esdp->no);
+    if (roflgs & ERTS_TMR_ROFLG_HLT) {
+	hlt_delete_timer(esdp, &tmr->hlt);
+	hl_timer_dec_refc(&tmr->hlt, roflgs);
+    }
+    else {
+	cancel_tw_timer(esdp, &tmr->twt);
+	tw_timer_dec_refc(&tmr->twt);
+    }
+}
+
+#ifdef ERTS_SMP
+
+static void
+init_canceled_queue(ErtsHLTCncldTmrQ *cq)
+{
+    erts_atomic_init_nob(&cq->tail.data.marker.u.next, ERTS_AINT_NULL);
+    erts_atomic_init_nob(&cq->tail.data.last,
+			 (erts_aint_t) &cq->tail.data.marker);
+    cq->head.first = (ErtsTimer *) &cq->tail.data.marker;
+    cq->head.unref_end = (ErtsTimer *) &cq->tail.data.marker;
+    cq->head.next.thr_progress = erts_thr_progress_current();
+    cq->head.next.thr_progress_reached = 1;
+    cq->head.next.unref_end = (ErtsTimer *) &cq->tail.data.marker;
+    cq->head.used_marker = 1;
+}
+
+static ERTS_INLINE int
+cq_enqueue(ErtsHLTCncldTmrQ *cq, ErtsTimer *tmr, int cinit)
+{
+    erts_aint_t itmp;
+    ErtsTimer *enq, *this = tmr;
+
+    erts_atomic_init_nob(&this->head.u.next, ERTS_AINT_NULL);
+    /* Enqueue at end of list... */
+
+    enq = (ErtsTimer *) erts_atomic_read_nob(&cq->tail.data.last);
+    itmp = erts_atomic_cmpxchg_relb(&enq->head.u.next,
+				    (erts_aint_t) this,
+				    ERTS_AINT_NULL);
+    if (itmp == ERTS_AINT_NULL) {
+	/* We are required to move last pointer */
+#ifdef DEBUG
+	ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->head.u.next));
+	ASSERT(((erts_aint_t) enq)
+	       == erts_atomic_xchg_relb(&cq->tail.data.last,
+					(erts_aint_t) this));
+#else
+	erts_atomic_set_relb(&cq->tail.data.last, (erts_aint_t) this);
+#endif
+	return 1;
+    }
+    else {
+	/*
+	 * We *need* to insert element somewhere in between the
+	 * last element we read earlier and the actual last element.
+	 */
+	int i = cinit;
+
+	while (1) {
+	    erts_aint_t itmp2;
+	    erts_atomic_set_nob(&this->head.u.next, itmp);
+	    itmp2 = erts_atomic_cmpxchg_relb(&enq->head.u.next,
+					     (erts_aint_t) this,
+					     itmp);
+	    if (itmp == itmp2)
+		return 0; /* inserted this */
+	    if ((i & 1) == 0)
+		itmp = itmp2;
+	    else {
+		enq = (ErtsTimer *) itmp2;
+		itmp = erts_atomic_read_acqb(&enq->head.u.next);
+		ASSERT(itmp != ERTS_AINT_NULL);
+	    }
+	    i++;
+	}
+    }
+}
+
+static ERTS_INLINE erts_aint_t
+check_insert_marker(ErtsHLTCncldTmrQ *cq, erts_aint_t ilast)
+{
+    if (!cq->head.used_marker
+	&& cq->head.unref_end == (ErtsTimer *) ilast) {
+	erts_aint_t itmp;
+	ErtsTimer *last = (ErtsTimer *) ilast;
+
+	erts_atomic_init_nob(&cq->tail.data.marker.u.next, ERTS_AINT_NULL);
+	itmp = erts_atomic_cmpxchg_relb(&last->head.u.next,
+					(erts_aint_t) &cq->tail.data.marker,
+					ERTS_AINT_NULL);
+	if (itmp == ERTS_AINT_NULL) {
+	    ilast = (erts_aint_t) &cq->tail.data.marker;
+	    cq->head.used_marker = !0;
+	    erts_atomic_set_relb(&cq->tail.data.last, ilast);
+	}
+    }
+    return ilast;
+}
+
+static ERTS_INLINE ErtsTimer *
+cq_dequeue(ErtsHLTCncldTmrQ *cq)
+{
+    ErtsTimer *tmr;
+
+    if (cq->head.first == cq->head.unref_end)
+	return NULL;
+
+    tmr = cq->head.first;
+    if (tmr == (ErtsTimer *) &cq->tail.data.marker) {
+	ASSERT(cq->head.used_marker);
+	cq->head.used_marker = 0;
+	tmr = (ErtsTimer *) erts_atomic_read_nob(&tmr->head.u.next);
+	if (tmr == cq->head.unref_end) {
+	    cq->head.first = tmr;
+	    return NULL;
+	}
+    }
+
+    cq->head.first = (ErtsTimer *) erts_atomic_read_nob(&tmr->head.u.next);
+
+    ASSERT(cq->head.first);
+
+    return tmr;
+}
+
+static int
+cq_check_incoming(ErtsSchedulerData *esdp, ErtsHLTCncldTmrQ *cq)
+{
+    erts_aint_t ilast = erts_atomic_read_nob(&cq->tail.data.last);
+    if (((ErtsTimer *) ilast) == (ErtsTimer *) &cq->tail.data.marker
+	&& cq->head.first == (ErtsTimer *) &cq->tail.data.marker) {
+	/* Nothing more to do... */
+	return 0;
+    }
+
+    if (cq->head.next.thr_progress_reached
+	|| erts_thr_progress_has_reached(cq->head.next.thr_progress)) {
+	cq->head.next.thr_progress_reached = 1;
+	/* Move unreferenced end pointer forward... */
+
+	ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore;
+
+	cq->head.unref_end = cq->head.next.unref_end;
+
+	ilast = check_insert_marker(cq, ilast);
+
+	if (cq->head.unref_end != (ErtsTimer *) ilast) {
+	    cq->head.next.unref_end = (ErtsTimer *) ilast;
+	    cq->head.next.thr_progress = erts_thr_progress_later(esdp);
+	    cq->head.next.thr_progress_reached = 0;
+	}
+    }
+    return 1;
+}
+
+static ERTS_INLINE void
+store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsHLTCncldTmrQ *cq)
+{
+    if (!cq->head.next.thr_progress_reached
+	&& (*prev_val == ERTS_THR_PRGR_INVALID
+	    || erts_thr_progress_cmp(cq->head.next.thr_progress,
+				     *prev_val) < 0)) {
+	*prev_val = cq->head.next.thr_progress;
+    }
+}
+
+static void
+handle_canceled_queue(ErtsSchedulerData *esdp,
+		      ErtsHLTCncldTmrQ *cq,
+		      int use_limit,
+		      int ops_limit,
+		      int *need_thr_progress,
+		      ErtsThrPrgrVal *thr_prgr_p,
+		      int *need_more_work)
+{
+    int need_thr_prgr = 0;
+    int need_mr_wrk = 0;
+    int have_checked_incoming = 0;
+    int ops = 0;
+
+    ERTS_HLT_ASSERT(cq == &esdp->timer_service->canceled_queue);
+
+    while (1) {
+	ErtsTimer *tmr = cq_dequeue(cq);
+
+	if (tmr)
+	    cleanup_sched_local_canceled_timer(esdp, tmr);
+	else {
+	    if (have_checked_incoming)
+		break;
+	    need_thr_prgr = cq_check_incoming(esdp, cq);
+	    if (need_thr_progress) {
+		*need_thr_progress |= need_thr_prgr;
+		if (need_thr_prgr)
+		    store_earliest_thr_prgr(thr_prgr_p, cq);
+	    }
+	    have_checked_incoming = 1;
+	    continue;
+	}
+
+	if (use_limit && ++ops >= ops_limit) {
+	    if (cq->head.first != cq->head.unref_end) {
+		need_mr_wrk = 1;
+		if (need_more_work)
+		    *need_more_work |= 1;
+	    }
+	    break;
+	}
+    }
+
+    if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) {
+	need_thr_prgr = cq_check_incoming(esdp, cq);
+	*need_thr_progress |= need_thr_prgr;
+	if (need_thr_prgr)
+	    store_earliest_thr_prgr(thr_prgr_p, cq);
+    }
+}
+
+void
+erts_handle_canceled_timers(void *vesdp,
+			    int *need_thr_progress,
+			    ErtsThrPrgrVal *thr_prgr_p,
+			    int *need_more_work)
+{
+    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
+    ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data());
+
+    handle_canceled_queue(esdp, &esdp->timer_service->canceled_queue,
+			  1, ERTS_TMR_CANCELED_TIMER_LIMIT,
+			  need_thr_progress, thr_prgr_p,
+			  need_more_work);
+}
+
+#endif /* ERTS_SMP */
+
+static void
+queue_canceled_timer(ErtsSchedulerData *esdp, int rsched_id, ErtsTimer *tmr)
+{
+#ifdef ERTS_SMP
+    ErtsHLTCncldTmrQ *cq;
+    cq = &ERTS_SCHEDULER_IX(rsched_id-1)->timer_service->canceled_queue;
+    if (cq_enqueue(cq, tmr, rsched_id - (int) esdp->no))
+	erts_notify_canceled_timer(esdp, rsched_id);
+#else
+    ERTS_INTERNAL_ERROR("Unexpected enqueue of canceled timer");
+#endif
+}
+
+static void
+continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr)
+{
+#ifdef ERTS_SMP
+    Uint32 sid = (tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK);
+
+    if (esdp->no != sid)
+	queue_canceled_timer(esdp, sid, tmr);
+    else
+#endif
+	cleanup_sched_local_canceled_timer(esdp, tmr);
+}
+
+/*
+ * BIF timer specific
+ */
+
+Uint erts_bif_timer_memory_size(void)
+{
+    return (Uint) 0;
+}
+
+static BIF_RETTYPE
+setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
+		int short_time, Eterm rcvr, Eterm acsr,
+		Eterm msg, int wrap)
+{
+    BIF_RETTYPE ret;
+    Eterm ref, tmo_msg, *hp;
+    ErtsHLTimer *tmr;
+    ErtsSchedulerData *esdp;
+    DeclareTmpHeap(tmp_hp, 4, c_p);
+
+    if (is_not_internal_pid(rcvr) && is_not_atom(rcvr))
+	goto badarg;
+
+    esdp = ERTS_PROC_GET_SCHDATA(c_p);
+
+    hp = HAlloc(c_p, REF_THING_SIZE);
+    ref = erts_sched_make_ref_in_buffer(esdp, hp);
+
+    ASSERT(erts_get_ref_numbers_thr_id(
+	       internal_ref_numbers(ref)) == (Uint32) esdp->no);
+
+    UseTmpHeap(4, c_p);
+
+    tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg;
+
+    tmr = create_hl_timer(esdp, timeout_pos, short_time, 1, NULL, 
+			  rcvr, acsr, tmo_msg, internal_ref_numbers(ref));
+
+    UnUseTmpHeap(4, c_p);
+
+    if (is_internal_pid(rcvr)) {
+	Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+					  rcvr, ERTS_PROC_LOCK_BTM,
+					  ERTS_P2P_FLG_INC_REFC);
+	if (!proc) {
+	    hlt_delete_timer(esdp, tmr);
+	    hl_timer_destroy(tmr);
+	}
+	else {
+	    proc_btm_rbt_insert(&proc->bif_timers, tmr);
+	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+	    tmr->receiver.proc = proc;
+	}
+    }
+
+    ERTS_BIF_PREP_RET(ret, ref);
+    return ret;
+
+badarg:
+
+    ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+    return ret;
+}
+
+static int
+cancel_bif_timer(ErtsHLTimer *tmr)
+{
+    erts_aint_t state;
+    Uint32 roflgs;
+    int res;
+
+    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state,
+					   ERTS_TMR_STATE_CANCELED,
+					   ERTS_TMR_STATE_ACTIVE);
+    if (state != ERTS_TMR_STATE_ACTIVE)
+	return 0;
+
+    res = -1;
+
+    roflgs = tmr->head.roflgs;
+    if (roflgs & ERTS_TMR_ROFLG_PROC) {
+	Process *proc = tmr->receiver.proc;
+	ERTS_HLT_ASSERT(!(tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME));
+
+	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
+	/*
+	 * If process is exiting, let it clean up
+	 * the btm tree by itself (it may be in
+	 * the middle of tree destruction).
+	 */
+	if (!ERTS_PROC_IS_EXITING(proc)
+	    && tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	    proc_btm_rbt_delete(&proc->bif_timers, tmr);
+	    tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+	    res = 1;
+	}
+	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+    }
+
+    return res;
+}
+
+static ERTS_INLINE Eterm
+access_sched_local_btm(Process *c_p, Eterm pid,
+		       Eterm tref, Uint32 *trefn,
+		       Uint32 *rrefn,
+		       int async, int cancel,
+		       int return_res,
+		       int info)
+{
+    ErtsSchedulerData *esdp;
+    ErtsHLTimerService *srv;
+    ErtsHLTimer *tmr;
+    Sint64 time_left;
+    Process *proc;
+    ErtsProcLocks proc_locks;
+
+    time_left = -1;
+
+    if (!c_p)
+	esdp = erts_get_scheduler_data();
+    else {
+	esdp = ERTS_PROC_GET_SCHDATA(c_p);
+	ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data());
+    }
+
+    ERTS_HLT_ASSERT(erts_get_ref_numbers_thr_id(trefn)
+		    == (Uint32) esdp->no);
+
+    srv = esdp->timer_service;
+
+    tmr = btm_rbt_lookup(srv->btm_tree, trefn);
+    if (tmr) {
+	if (!cancel) {
+	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state);
+	    if (state == ERTS_TMR_STATE_ACTIVE)
+		time_left = get_time_left(esdp, tmr->timeout);
+	}
+	else {
+	    int cncl_res = cancel_bif_timer(tmr);
+	    if (cncl_res) {
+
+		time_left = get_time_left(esdp, tmr->timeout);
+
+		if (cncl_res > 0)
+		    hl_timer_dec_refc(tmr, tmr->head.roflgs);
+
+		hlt_delete_timer(esdp, tmr);
+	    }
+	}
+    }
+
+    if (!info)
+	return am_ok;
+
+    if (return_res) {
+	ERTS_HLT_ASSERT(c_p);
+	if (time_left < 0)
+	    return am_false;
+	else if (time_left <= (Sint64) MAX_SMALL)
+	    return make_small((Sint) time_left);
+	else {
+	    Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left);
+	    Eterm *hp = HAlloc(c_p, hsz);
+	    return erts_sint64_to_big(time_left, &hp);
+	}
+    }
+
+    if (c_p) {
+	proc = c_p;
+	proc_locks = ERTS_PROC_LOCK_MAIN;
+    }
+    else {
+	proc = erts_proc_lookup(pid);
+	proc_locks = 0;
+    }
+
+    if (proc) {
+	Uint hsz;
+	ErlOffHeap *ohp;
+	ErlHeapFragment* bp;
+	Eterm *hp, msg, ref, result;
+#ifdef ERTS_HLT_DEBUG
+	Eterm *hp_end;
+#endif
+
+	hsz = 3; /* 2-tuple */
+	if (!async)
+	    hsz += REF_THING_SIZE;
+	else {
+	    if (is_non_value(tref))
+		hsz += REF_THING_SIZE;
+	    hsz += 1; /* upgrade to 3-tuple */
+	}
+	if (time_left > (Sint64) MAX_SMALL)
+	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
+
+	hp = erts_alloc_message_heap(hsz,
+				     &bp,
+				     &ohp,
+				     proc,
+				     &proc_locks);
+
+#ifdef ERTS_HLT_DEBUG
+	hp_end = hp + hsz;
+#endif
+
+	if (time_left < 0)
+	    result = am_false;
+	else if (time_left <= (Sint64) MAX_SMALL)
+	    result = make_small((Sint) time_left);
+	else
+	    result = erts_sint64_to_big(time_left, &hp);
+
+	if (!async) {
+	    write_ref_thing(hp,
+			    rrefn[0],
+			    rrefn[1],
+			    rrefn[2]);
+	    ref = make_internal_ref(hp);
+	    hp += REF_THING_SIZE;
+	    msg = TUPLE2(hp, ref, result);
+
+	    ERTS_HLT_ASSERT(hp + 3 == hp_end);
+	}
+	else {
+	    Eterm tag = cancel ? am_cancel_timer : am_read_timer;
+	    if (is_value(tref))
+		ref = tref;
+	    else {
+		write_ref_thing(hp,
+				trefn[0],
+				trefn[1],
+				trefn[2]);
+		ref = make_internal_ref(hp);
+		hp += REF_THING_SIZE;
+	    }
+	    msg = TUPLE3(hp, tag, ref, result);
+
+	    ERTS_HLT_ASSERT(hp + 4 == hp_end);
+
+	}
+	erts_queue_message(proc, &proc_locks, bp,
+			   msg, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+	    );
+
+	if (c_p)
+	    proc_locks &= ~ERTS_PROC_LOCK_MAIN;
+	if (proc_locks)
+	    erts_smp_proc_unlock(proc, proc_locks);
+    }
+
+    return am_ok;
+}
+
+#define ERTS_BTM_REQ_FLG_ASYNC		(((Uint32) 1) << 0)
+#define ERTS_BTM_REQ_FLG_CANCEL		(((Uint32) 1) << 1)
+#define ERTS_BTM_REQ_FLG_INFO		(((Uint32) 1) << 2)
+
+typedef struct {
+    Eterm pid;
+    Uint32 trefn[ERTS_REF_NUMBERS];
+    Uint32 rrefn[ERTS_REF_NUMBERS];
+    Uint32 flags;
+} ErtsBifTimerRequest;
+
+static void
+bif_timer_access_request(void *vreq)
+{
+    ErtsBifTimerRequest *req = (ErtsBifTimerRequest *) vreq;
+    int async = (int) (req->flags & ERTS_BTM_REQ_FLG_ASYNC);
+    int cancel = (int) (req->flags & ERTS_BTM_REQ_FLG_CANCEL);
+    int info = (int) (req->flags & ERTS_BTM_REQ_FLG_INFO);
+    (void) access_sched_local_btm(NULL, req->pid, THE_NON_VALUE,
+				  req->trefn, req->rrefn, async,
+				  cancel, 0, info);
+    erts_free(ERTS_ALC_T_TIMER_REQUEST, vreq);
+}
+
+static int
+try_access_sched_remote_btm(ErtsSchedulerData *esdp,
+			    Process *c_p, Uint32 sid,
+			    Eterm tref, Uint32 *trefn,
+			    int async, int cancel,
+			    int info, Eterm *resp)
+{
+    ErtsHLTimer *tmr;
+    Sint64 time_left;
+
+    ERTS_HLT_ASSERT(c_p);
+
+    /*
+     * Check if the timer is aimed at current
+     * process of if this process is an accessor
+     * of the timer...
+     */
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM);
+    tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn);
+    if (!tmr)
+	tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn);
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM);
+    if (!tmr)
+	return 0;
+
+    if (!cancel) {
+	erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state);
+	if (state == ERTS_TMR_STATE_ACTIVE)
+	    time_left = get_time_left(esdp, tmr->timeout);
+	else
+	    time_left = -1;
+    }
+    else {
+	int cncl_res = cancel_bif_timer(tmr);
+	if (!cncl_res)
+	    time_left = -1;
+	else {
+	    time_left = get_time_left(esdp, tmr->timeout);
+	    if (cncl_res > 0)
+		queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
+	}
+    }
+
+    if (!info) {
+	*resp = am_ok;
+	return 1;
+    }
+
+    if (!async) {
+	if (time_left < 0)
+	    *resp = am_false;
+	else if (time_left <= (Sint64) MAX_SMALL)
+	    *resp = make_small((Sint) time_left);
+	else {
+	    Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left);
+	    Eterm *hp = HAlloc(c_p, hsz);
+	    *resp = erts_sint64_to_big(time_left, &hp);
+	}
+    }
+    else {
+	Eterm tag, res, msg;
+	ErlOffHeap *ohp;
+	ErlHeapFragment* bp;
+	Uint hsz;
+	Eterm *hp;
+	ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
+
+	hsz = 4;
+	if (time_left > (Sint64) MAX_SMALL)
+	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
+
+	hp = erts_alloc_message_heap(hsz,
+				     &bp,
+				     &ohp,
+				     c_p,
+				     &proc_locks);
+	if (cancel)
+	    tag = am_cancel_timer;
+	else
+	    tag = am_read_timer;
+
+	if (time_left < 0)
+	    res = am_false;
+	else if (time_left <= (Sint64) MAX_SMALL)
+	    res = make_small((Sint) time_left);
+	else
+	    res = erts_sint64_to_big(time_left, &hp);
+
+	msg = TUPLE3(hp, tag, tref, res);
+
+	erts_queue_message(c_p, &proc_locks, bp,
+			   msg, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+	    );
+
+	proc_locks &= ~ERTS_PROC_LOCK_MAIN;
+	if (proc_locks)
+	    erts_smp_proc_unlock(c_p, proc_locks);
+
+	*resp = am_ok;
+    }
+    return 1;
+}
+
+static BIF_RETTYPE
+access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
+{
+    BIF_RETTYPE ret;
+    ErtsSchedulerData *esdp;
+    Uint32 sid;
+    Uint32 *trefn;
+    Eterm res;
+
+    if (is_not_internal_ref(tref)) {
+	if (is_not_ref(tref))
+	    goto badarg;
+	else
+	    goto no_timer;
+    }
+
+    esdp = ERTS_PROC_GET_SCHDATA(c_p);
+
+    trefn = internal_ref_numbers(tref);
+    sid = erts_get_ref_numbers_thr_id(trefn);
+    if (sid < 1 || erts_no_schedulers < sid)
+	goto no_timer;
+
+    if (sid == (Uint32) esdp->no) {
+	res = access_sched_local_btm(c_p, c_p->common.id,
+				     tref, trefn, NULL,
+				     async, cancel, !async,
+				     info);
+	ERTS_BIF_PREP_RET(ret, res);
+    }
+    else if (try_access_sched_remote_btm(esdp, c_p, sid,
+					 tref, trefn,
+					 async, cancel,
+					 info, &res)) {
+	ERTS_BIF_PREP_RET(ret, res);
+    }
+    else {
+	/*
+	 * Schedule access for execution on
+	 * remote scheduler...
+	 */
+	ErtsBifTimerRequest *req = erts_alloc(ERTS_ALC_T_TIMER_REQUEST,
+					      sizeof(ErtsBifTimerRequest));
+
+	req->flags = 0;
+	if (cancel)
+	    req->flags |= ERTS_BTM_REQ_FLG_CANCEL;
+	if (async)
+	    req->flags |= ERTS_BTM_REQ_FLG_ASYNC;
+	if (info)
+	    req->flags |= ERTS_BTM_REQ_FLG_INFO;
+
+	req->pid = c_p->common.id;
+
+	req->trefn[0] = trefn[0];
+	req->trefn[1] = trefn[1];
+	req->trefn[2] = trefn[2];
+
+	if (async)
+	    ERTS_BIF_PREP_RET(ret, am_ok);
+	else {
+	    Eterm *hp, rref;
+	    Uint32 *rrefn;
+
+	    hp = HAlloc(c_p, REF_THING_SIZE);
+	    rref = erts_sched_make_ref_in_buffer(esdp, hp);
+	    rrefn = internal_ref_numbers(rref);
+
+	    req->rrefn[0] = rrefn[0];
+	    req->rrefn[1] = rrefn[1];
+	    req->rrefn[2] = rrefn[2];
+
+	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+
+	    if (ERTS_PROC_PENDING_EXIT(c_p))
+		ERTS_VBUMP_ALL_REDS(c_p);
+	    else {
+		/*
+		 * Caller needs to wait for a message containing
+		 * the ref that we just created. No such message
+		 * can exist in callers message queue at this time.
+		 * We therefore move the save pointer of the
+		 * callers message queue to the end of the queue.
+		 *
+		 * NOTE: It is of vital importance that the caller
+		 *       immediately do a receive unconditionaly
+		 *       waiting for the message with the reference;
+		 *       otherwise, next receive will *not* work
+		 *       as expected!
+		 */
+		ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+		c_p->msg.save = c_p->msg.last;
+	    }
+	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+
+	    ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, rref);
+	}
+
+	erts_schedule_misc_aux_work(sid,
+				    bif_timer_access_request,
+				    (void *) req);
+    }
+
+    return ret;
+
+badarg:
+    ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+    return ret;
+
+no_timer:
+    ERTS_BIF_PREP_RET(ret, am_false);
+    return ret;
+    
+}
+
+static ERTS_INLINE int
+bool_arg(Eterm val, int *argp)
+{
+    switch (val) {
+    case am_true: *argp = 1; return 1;
+    case am_false: *argp = 0; return 1;
+    default: return 0;
+    }
+}
+
+static ERTS_INLINE int
+parse_bif_timer_options(Eterm option_list, int *async, int *info,
+			int *abs, Eterm *accessor)
+{
+    Eterm list = option_list;
+
+    if (async)
+	*async = 0;
+    if (info)
+	*info = 0;
+    if (abs)
+	*abs = 0;
+    if (accessor)
+	*accessor = THE_NON_VALUE;
+
+    while (is_list(list)) {
+	Eterm *consp, *tp, opt;
+
+	consp = list_val(list);
+	opt = CAR(consp);
+	if (is_not_tuple(opt))
+	    return 0;
+
+	tp = tuple_val(opt);
+	if (arityval(tp[0]) != 2)
+	    return 0;
+
+	switch (tp[1]) {
+	case am_async:
+	    if (!async || !bool_arg(tp[2], async))
+		return 0;
+	    break;
+	case am_info:
+	    if (!info || !bool_arg(tp[2], info))
+		return 0;
+	    break;
+	case am_abs:
+	    if (!abs || !bool_arg(tp[2], abs))
+		return 0;
+	    break;
+	case am_accessor:
+	    if (!accessor || is_not_internal_pid(tp[2]))
+		return 0;
+	    *accessor = tp[2];
+	    break;
+	default:
+	    return 0;
+	}
+
+	list = CDR(consp);	
+    }
+
+    if (is_not_nil(list))
+	return 0;
+    return 1;
+}
+
+static void
+exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp)
+{
+    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
+    Uint32 sid, roflgs;
+    erts_aint_t state;
+
+    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state,
+					   ERTS_TMR_STATE_CANCELED,
+					   ERTS_TMR_STATE_ACTIVE);
+
+    roflgs = tmr->head.roflgs;
+    sid = roflgs & ERTS_TMR_ROFLG_SID_MASK;
+
+    ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(tmr->btm.refn));
+    ERTS_HLT_ASSERT(tmr->btm.proc_tree.parent
+		    != ERTS_HLT_PFIELD_NOT_IN_TABLE);
+
+    tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+
+    if (sid == (Uint32) esdp->no) {
+	if (state == ERTS_TMR_STATE_ACTIVE)
+	    hlt_delete_timer(esdp, tmr);
+	hl_timer_dec_refc(tmr, roflgs);
+    }
+    else {
+	if (state == ERTS_TMR_STATE_ACTIVE)
+	    queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
+	else
+	    hl_timer_dec_refc(tmr, roflgs);
+    }
+}
+
+#ifdef ERTS_HLT_DEBUG
+#  define ERTS_BTM_MAX_DESTROY_LIMIT 2
+#else
+#  define ERTS_BTM_MAX_DESTROY_LIMIT 50
+#endif
+
+typedef struct {
+    ErtsBifTimers *bif_timers;
+    union {
+	proc_btm_rbt_yield_state_t proc_btm_yield_state;
+	abtm_rbt_yield_state_t abtm_yield_state;
+    } u;
+} ErtsBifTimerYieldState;
+
+int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p);
+    ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}};
+    ErtsBifTimerYieldState *ysp;
+    int res;
+
+    ysp = (ErtsBifTimerYieldState *) *vyspp;
+    if (!ysp)
+	ysp = &ys;
+
+    res = proc_btm_rbt_foreach_destroy_yielding(&ysp->bif_timers,
+						exit_cancel_bif_timer,
+						(void *) esdp,
+						&ysp->u.proc_btm_yield_state,
+						ERTS_BTM_MAX_DESTROY_LIMIT);
+
+    if (res == 0) {
+	if (ysp != &ys)
+	    erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp);
+	*vyspp = NULL;
+    }
+    else {
+
+	if (ysp == &ys) {
+	    ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE,
+			     sizeof(ErtsBifTimerYieldState));
+	    sys_memcpy((void *) ysp, (void *) &ys,
+		       sizeof(ErtsBifTimerYieldState));
+	}
+
+	*vyspp = (void *) ysp;
+    }
+
+    return res;
+}
+
+static void
+detach_bif_timer(ErtsHLTimer *tmr, void *vesdp)
+{
+    tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+    hl_timer_dec_refc(tmr, tmr->head.roflgs);
+}
+
+int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p);
+    ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}};
+    ErtsBifTimerYieldState *ysp;
+    int res;
+
+    ysp = (ErtsBifTimerYieldState *) *vyspp;
+    if (!ysp)
+	ysp = &ys;
+
+    res = abtm_rbt_foreach_destroy_yielding(&ysp->bif_timers,
+					    detach_bif_timer,
+					    (void *) esdp,
+					    &ysp->u.abtm_yield_state,
+					    ERTS_BTM_MAX_DESTROY_LIMIT);
+
+    if (res == 0) {
+	if (ysp != &ys)
+	    erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp);
+	*vyspp = NULL;
+    }
+    else {
+
+	if (ysp == &ys) {
+	    ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE,
+			     sizeof(ErtsBifTimerYieldState));
+	    sys_memcpy((void *) ysp, (void *) &ys,
+		       sizeof(ErtsBifTimerYieldState));
+	}
+
+	*vyspp = (void *) ysp;
+    }
+
+    return res;
+}
+
+static ERTS_INLINE int
+parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
+		  ErtsMonotonicTime *conv_arg, int abs,
+		  ErtsMonotonicTime *tposp, int *stimep)
+{
+    ErtsMonotonicTime t;
+
+    if (!term_to_Sint64(arg, &t)) {
+	ERTS_HLT_ASSERT(!is_small(arg));
+	if (!is_big(arg))
+	    return -1;
+
+	if (abs || !big_sign(arg))
+	    return 1;
+
+	return -1;
+    }
+
+    if (conv_arg)
+	*conv_arg = t;
+
+    if (abs) {
+	t += -1*ERTS_MONOTONIC_OFFSET_MSEC; /* external to internal */
+	if (t < ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN))
+	    return 1;
+	if (t > ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_END))
+	    return 1;
+	*stimep = (t - ERTS_MONOTONIC_TO_MSEC(esdp->last_monotonic_time)
+		   < ERTS_BIF_TIMER_SHORT_TIME);
+	*tposp = ERTS_MSEC_TO_CLKTCKS(t);
+    }
+    else {
+	ErtsMonotonicTime now, ticks;
+	
+	if (t < 0)
+	    return -1;
+
+	ticks = ERTS_MSEC_TO_CLKTCKS(t);
+
+	if (ERTS_CLKTCK_RESOLUTION > 1000 && ticks < 0)
+	    return 1;
+
+	ERTS_HLT_ASSERT(ticks >= 0);
+
+	now = erts_get_monotonic_time(esdp);
+	ticks += ERTS_MONOTONIC_TO_CLKTCKS(now-1);
+	ticks += 1;
+
+	if (ticks < ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_BEGIN))
+	    return 1;
+	if (ticks > ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_END))
+	    return 1;
+
+	*stimep = (t < ERTS_BIF_TIMER_SHORT_TIME);
+	*tposp = ticks;
+    }
+
+    return 0;
+}
+
+/*
+ *
+ * The BIF timer BIFs...
+ */
+
+BIF_RETTYPE send_after_3(BIF_ALIST_3)
+{
+    ErtsMonotonicTime timeout_pos;
+    int short_time, tres;
+
+    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+			     0, &timeout_pos, &short_time);
+    if (tres != 0)
+	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+
+    return setup_bif_timer(BIF_P, timeout_pos, short_time,
+			   BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0);
+}
+
+BIF_RETTYPE send_after_4(BIF_ALIST_4)
+{
+    ErtsMonotonicTime timeout_pos;
+    Eterm accessor;
+    int short_time, abs, tres;
+
+    if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor))
+	BIF_ERROR(BIF_P, BADARG);
+    
+    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+			     abs, &timeout_pos, &short_time);
+    if (tres != 0)
+	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+
+    return setup_bif_timer(BIF_P, timeout_pos, short_time,
+			   BIF_ARG_2, accessor, BIF_ARG_3, 0);
+}
+
+BIF_RETTYPE start_timer_3(BIF_ALIST_3)
+{
+    ErtsMonotonicTime timeout_pos;
+    int short_time, tres;
+
+    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+			     0, &timeout_pos, &short_time);
+    if (tres != 0)
+	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+
+    return setup_bif_timer(BIF_P, timeout_pos, short_time,
+			   BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0);
+}
+
+BIF_RETTYPE start_timer_4(BIF_ALIST_4)
+{
+    ErtsMonotonicTime timeout_pos;
+    Eterm accessor;
+    int short_time, abs, tres;
+
+    if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor))
+	BIF_ERROR(BIF_P, BADARG);
+
+    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+			     abs, &timeout_pos, &short_time);
+    if (tres != 0)
+	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+
+    return setup_bif_timer(BIF_P, timeout_pos, short_time,
+			   BIF_ARG_2, accessor, BIF_ARG_3, !0);
+}
+
+BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
+{
+    return access_bif_timer(BIF_P, BIF_ARG_1, 1, 0, 1);
+}
+
+BIF_RETTYPE cancel_timer_2(BIF_ALIST_2)
+{
+    BIF_RETTYPE ret;
+    int async, info;
+
+    if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL, NULL))
+	return access_bif_timer(BIF_P, BIF_ARG_1, 1, async, info);
+
+    ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+    return ret;
+}
+
+BIF_RETTYPE read_timer_1(BIF_ALIST_1)
+{
+    return access_bif_timer(BIF_P, BIF_ARG_1, 0, 0, 1);
+}
+
+BIF_RETTYPE read_timer_2(BIF_ALIST_2)
+{
+    BIF_RETTYPE ret;
+    int async;
+
+    if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL, NULL))
+	return access_bif_timer(BIF_P, BIF_ARG_1, 0, async, 1);
+
+    ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+    return ret;
+}
+
+/*
+ * Process and Port timer functionality.
+ *
+ * NOTE! These are only allowed to be called by a
+ *       scheduler thread that currently is
+ *       executing the process or port.
+ */
+
+static ERTS_INLINE void
+set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo,
+		      ErtsMonotonicTime timeout_pos, int short_time)
+{
+    void *tmr;
+    check_canceled_queue(esdp, esdp->timer_service);
+
+    if (tmo == 0)
+	c_p->flags |= F_TIMO;
+    else {
+
+	c_p->flags |= F_INSLPQUEUE;
+	c_p->flags &= ~F_TIMO;
+
+	if (tmo < ERTS_TIMER_WHEEL_MSEC)
+	    tmr = (void *) create_tw_timer(esdp, (void *) c_p, 1, timeout_pos);
+	else
+	    tmr = (void *) create_hl_timer(esdp, timeout_pos,
+					   short_time, 0, (void *) c_p,
+					   c_p->common.id, NIL, NIL, NULL);
+	erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr);
+    }
+}
+
+int
+erts_set_proc_timer_term(Process *c_p, Eterm etmo)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsMonotonicTime tmo, timeout_pos;
+    int short_time, tres;
+
+    ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer)
+		    == ERTS_PTMR_NONE);
+
+    tres = parse_timeout_pos(esdp, etmo, &tmo, 0,
+			     &timeout_pos, &short_time);
+    if (tres != 0)
+	return tres;
+
+    if ((tmo >> 32) != 0)
+	return 1;
+
+    set_proc_timer_common(c_p, esdp, tmo, timeout_pos, short_time);
+    return 0;
+}
+
+void
+erts_set_proc_timer_uword(Process *c_p, UWord tmo)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+
+    ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer)
+		    == ERTS_PTMR_NONE);
+
+#ifndef ARCH_32
+    ERTS_HLT_ASSERT((tmo >> 32) == (UWord) 0);
+#endif
+
+    if (tmo == 0)
+	c_p->flags |= F_TIMO;
+    else {
+	ErtsMonotonicTime timeout_pos;
+	timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp),
+				      (ErtsMonotonicTime) tmo);
+	set_proc_timer_common(c_p, esdp, (ErtsMonotonicTime) tmo,
+			      timeout_pos,
+			      tmo < ERTS_BIF_TIMER_SHORT_TIME);
+    }
+}
+
+void
+erts_cancel_proc_timer(Process *c_p)
+{
+    erts_aint_t tval;
+    tval = erts_smp_atomic_xchg_acqb(&c_p->common.timer,
+				     ERTS_PTMR_NONE);
+    c_p->flags &= ~(F_INSLPQUEUE|F_TIMO);
+    if (tval == ERTS_PTMR_NONE)
+	return;
+    if (tval == ERTS_PTMR_TIMEDOUT) {
+	erts_smp_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE);
+	return;
+    }
+    continue_cancel_ptimer(ERTS_PROC_GET_SCHDATA(c_p),
+			   (ErtsTimer *) tval);
+}
+
+void
+erts_set_port_timer(Port *c_prt, Sint64 tmo)
+{
+    void *tmr;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsMonotonicTime timeout_pos;
+
+    if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE)
+	erts_cancel_port_timer(c_prt);
+
+    check_canceled_queue(esdp, esdp->timer_service);
+
+    timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo);
+
+    if (tmo < ERTS_TIMER_WHEEL_MSEC)
+	tmr = (void *) create_tw_timer(esdp, (void *) c_prt, 0,
+				       timeout_pos);
+    else
+	tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, 0,
+				       (void *) c_prt,
+				       c_prt->common.id, NIL, NIL,
+				       NULL);
+    erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
+}
+
+void
+erts_cancel_port_timer(Port *c_prt)
+{
+    erts_aint_t tval;
+    tval = erts_smp_atomic_xchg_acqb(&c_prt->common.timer,
+				     ERTS_PTMR_NONE);
+    if (tval == ERTS_PTMR_NONE)
+	return;
+    if (tval == ERTS_PTMR_TIMEDOUT) {
+	while (!erts_port_task_is_scheduled(&c_prt->timeout_task))
+	    erts_thr_yield();
+	erts_port_task_abort(&c_prt->timeout_task);
+	erts_smp_atomic_set_nob(&c_prt->common.timer, ERTS_PTMR_NONE);
+	return;
+    }
+    continue_cancel_ptimer(erts_get_scheduler_data(),
+			   (ErtsTimer *) tval);
+}
+
+Sint64
+erts_read_port_timer(Port *c_prt)
+{
+    ErtsTimer *tmr;
+    erts_aint_t itmr;
+    ErtsMonotonicTime timeout_pos;
+
+    itmr = erts_smp_atomic_read_acqb(&c_prt->common.timer);
+    if (itmr == ERTS_PTMR_NONE)
+	return (Sint64) -1;
+    if (itmr == ERTS_PTMR_TIMEDOUT)
+	return (Sint64) 0;
+    tmr = (ErtsTimer *) itmr;
+    if (tmr->head.roflgs & ERTS_TMR_ROFLG_HLT)
+	timeout_pos = tmr->hlt.timeout;
+    else
+	timeout_pos = tmr->twt.tw_tmr.timeout_pos;
+    return get_time_left(NULL, timeout_pos);
+}
+
+/*
+ * Debug stuff...
+ */
+
+typedef struct {
+    int to;
+    void *to_arg;
+    ErtsMonotonicTime now;
+} ErtsBTMPrint;
+
+static void
+btm_print(ErtsHLTimer *tmr, void *vbtmp)
+{
+    ErtsBTMPrint *btmp = (ErtsBTMPrint *) vbtmp;
+    ErtsMonotonicTime left;
+    Eterm receiver;
+
+    if (tmr->timeout <= btmp->now)
+	left = 0;
+    left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now);
+
+    receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+		? tmr->receiver.name
+		: tmr->receiver.proc->common.id);
+
+    erts_print(btmp->to, btmp->to_arg,
+	       "=timer:%T\n"
+	       "Message: %T\n"
+	       "Time left: %b64d\n",
+	       receiver,
+	       tmr->btm.message,
+	       (Sint64) left);
+}
+
+void
+erts_print_bif_timer_info(int to, void *to_arg)
+{
+    ErtsBTMPrint btmp;
+    int six;
+
+    if (!ERTS_IS_CRASH_DUMPING)
+	ERTS_INTERNAL_ERROR("Not crash dumping");
+
+    btmp.to = to;
+    btmp.to_arg = to_arg;
+    btmp.now = erts_get_monotonic_time(NULL);
+    btmp.now = ERTS_MONOTONIC_TO_CLKTCKS(btmp.now);
+
+    for (six = 0; six < erts_no_schedulers; six++) {
+	ErtsHLTimerService *srv =
+	    erts_aligned_scheduler_data[six].esd.timer_service;
+	btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp);
+    }
+}
+
+typedef struct {
+    void (*func)(Eterm,
+		 Eterm,
+		 ErlHeapFragment *,
+		 void *);
+    void *arg;
+} ErtsBTMForeachDebug;
+
+static void
+debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd)
+{
+    if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) {
+	ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd;
+	(*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+			? tmr->receiver.name
+			: tmr->receiver.proc->common.id),
+		       tmr->btm.message,
+		       tmr->btm.bp,
+		       btmfd->arg);
+    }
+}
+
+void
+erts_debug_bif_timer_foreach(void (*func)(Eterm,
+					  Eterm,
+					  ErlHeapFragment *,
+					  void *),
+			     void *arg)
+{
+    ErtsBTMForeachDebug btmfd;
+    int six;
+
+    btmfd.func = func;
+    btmfd.arg = arg;
+
+    if (!erts_smp_thr_progress_is_blocking())
+	ERTS_INTERNAL_ERROR("Not blocking thread progress");
+
+    for (six = 0; six < erts_no_schedulers; six++) {
+	ErtsHLTimerService *srv =
+	    erts_aligned_scheduler_data[six].esd.timer_service;
+	btm_rbt_foreach(srv->btm_tree,
+			debug_btm_foreach,
+			(void *) &btmfd);
+    }
+}
+
+#ifdef ERTS_HLT_HARD_DEBUG
+
+typedef struct {
+    ErtsHLTimerService *srv;
+    int found_root;
+    ErtsHLTimer **rootpp;
+} ErtsHdbgHLT;
+
+static void
+st_hdbg_func(ErtsHLTimer *tmr, void *vhdbg)
+{
+    ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg;
+    ErtsHLTimer **rootpp;
+    ERTS_HLT_ASSERT(tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME);
+    if (tmr->time.tree.parent == ERTS_HLT_PFLG_SAME_TIME) {
+	ERTS_HLT_ASSERT(tmr != *hdbg->rootpp);
+    }
+    else {
+	rootpp = (ErtsHLTimer **) (tmr->time.tree.parent
+				   & ~ERTS_HLT_PFLG_SAME_TIME);
+	ERTS_HLT_ASSERT(rootpp == hdbg->rootpp);
+	ERTS_HLT_ASSERT(tmr == *rootpp);
+	ERTS_HLT_ASSERT(!hdbg->found_root);
+	hdbg->found_root = 1;
+    }
+    ERTS_HLT_ASSERT(tmr->time.tree.u.l.next->time.tree.u.l.prev == tmr);
+    ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev->time.tree.u.l.next == tmr);
+    ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr);
+}
+
+static void
+tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg)
+{
+    ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg;
+    ErtsHLTimer *prnt;
+    ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0);
+    prnt = (ErtsHLTimer *) (tmr->time.tree.parent & ~ERTS_HLT_PFLGS_MASK);
+    if (prnt) {
+	ERTS_HLT_ASSERT(prnt->time.tree.u.t.left == tmr
+			|| prnt->time.tree.u.t.right == tmr);
+    }
+    else {
+	ERTS_HLT_ASSERT(!hdbg->found_root);
+	hdbg->found_root = 1;
+	ERTS_HLT_ASSERT(tmr == *hdbg->rootpp);
+    }
+    if (tmr->time.tree.u.t.left) {
+	prnt = (ErtsHLTimer *) (tmr->time.tree.u.t.left->time.tree.parent
+				& ~ERTS_HLT_PFLGS_MASK);
+	ERTS_HLT_ASSERT(tmr == prnt);
+    }
+    if (tmr->time.tree.u.t.right) {
+	prnt = (ErtsHLTimer *) (tmr->time.tree.u.t.right->time.tree.parent
+				& ~ERTS_HLT_PFLGS_MASK);
+	ERTS_HLT_ASSERT(tmr == prnt);
+    }
+    ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr);
+    if (tmr->time.tree.same_time) {
+	ErtsHdbgHLT st_hdbg;
+	st_hdbg.srv = hdbg->srv;
+	st_hdbg.found_root = 0;
+	st_hdbg.rootpp = &tmr->time.tree.same_time;
+	same_time_list_foreach(tmr->time.tree.same_time, st_hdbg_func, (void *) &st_hdbg);
+	ERTS_HLT_ASSERT(st_hdbg.found_root);
+    }
+}
+
+static void
+bt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg)
+{
+    ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg;
+    ErtsHLTimer *prnt;
+    ERTS_HLT_ASSERT((tmr->btm.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0);
+    prnt = (ErtsHLTimer *) (tmr->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK);
+    if (prnt) {
+	ERTS_HLT_ASSERT(prnt->btm.tree.left == tmr
+			|| prnt->btm.tree.right == tmr);
+    }
+    else {
+	ERTS_HLT_ASSERT(!hdbg->found_root);
+	hdbg->found_root = 1;
+	ERTS_HLT_ASSERT(tmr == *hdbg->rootpp);
+    }
+    if (tmr->btm.tree.left) {
+	prnt = (ErtsHLTimer *) (tmr->btm.tree.left->btm.tree.parent
+				& ~ERTS_HLT_PFLGS_MASK);
+	ERTS_HLT_ASSERT(tmr == prnt);
+    }
+    if (tmr->btm.tree.right) {
+	prnt = (ErtsHLTimer *) (tmr->btm.tree.right->btm.tree.parent
+				& ~ERTS_HLT_PFLGS_MASK);
+	ERTS_HLT_ASSERT(tmr == prnt);
+    }
+    if (tmr->pending_timeout) {
+	if (tmr->pending_timeout > 0) /* container > 0 */
+	    ERTS_HLT_ASSERT(tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE);
+	else {
+	    ERTS_HLT_ASSERT(tmr->time.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE);
+	    ERTS_HLT_ASSERT(tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME);
+	}
+    }
+    else {
+	ErtsHLTimer *ttmr = time_rbt_lookup(hdbg->srv->time_tree, tmr->timeout);
+	ERTS_HLT_ASSERT(ttmr);
+	if (ttmr != tmr) {
+	    ERTS_HLT_ASSERT(ttmr->time.tree.same_time);
+	    ERTS_HLT_ASSERT(tmr == same_time_list_lookup(ttmr->time.tree.same_time, tmr));
+	}
+    }
+}
+
+static void
+hdbg_chk_srv(ErtsHLTimerService *srv)
+{
+    if (srv->time_tree) {
+	ErtsHdbgHLT hdbg;
+	hdbg.srv = srv;
+	hdbg.found_root = 0;
+	hdbg.rootpp = &srv->time_tree;
+	time_rbt_foreach(srv->time_tree, tt_hdbg_func, (void *) &hdbg);
+	ERTS_HLT_ASSERT(hdbg.found_root);
+    }
+    if (srv->btm_tree) {
+	ErtsHdbgHLT hdbg;
+	hdbg.srv = srv;
+	hdbg.found_root = 0;
+	hdbg.rootpp = &srv->btm_tree;
+	btm_rbt_foreach(srv->btm_tree, bt_hdbg_func, (void *) &hdbg);
+	ERTS_HLT_ASSERT(hdbg.found_root);
+    }
+}
+
+#endif /* ERTS_HLT_HARD_DEBUG */
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
new file mode 100644
index 0000000000..30889a71da
--- /dev/null
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -0,0 +1,80 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2015. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+#ifndef ERL_HL_TIMER_H__
+#define ERL_HL_TIMER_H__
+
+typedef struct ErtsHLTimer_ ErtsBifTimers;
+typedef struct ErtsHLTimerService_ ErtsHLTimerService;
+
+#include "sys.h"
+#include "erl_process.h"
+#define ERL_PORT_GET_PORT_TYPE_ONLY__
+#include "erl_port.h"
+#undef ERL_PORT_GET_PORT_TYPE_ONLY__
+#include "erl_message.h"
+#include "erl_alloc_types.h"
+
+#define ERTS_PTMR_NONE ((erts_aint_t) NULL)
+#define ERTS_PTMR_TIMEDOUT (ERTS_PTMR_NONE + ((erts_aint_t) 1))
+
+#define ERTS_PTMR_INIT(P) \
+    erts_smp_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE)
+#define ERTS_PTMR_IS_SET(P) \
+    (ERTS_PTMR_NONE != erts_smp_atomic_read_nob(&(P)->common.timer))
+#define ERTS_PTMR_IS_TIMED_OUT(P) \
+    (ERTS_PTMR_TIMEDOUT == erts_smp_atomic_read_nob(&(P)->common.timer))
+
+#define ERTS_PTMR_CLEAR(P)					\
+    do {							\
+	ASSERT(ERTS_PTMR_IS_TIMED_OUT((P)));			\
+	erts_smp_atomic_set_nob(&(P)->common.timer,		\
+				ERTS_PTMR_NONE);		\
+    } while (0)
+
+size_t erts_timer_type_size(ErtsAlcType_t type);
+int erts_set_proc_timer_term(Process *, Eterm);
+void erts_set_proc_timer_uword(Process *, UWord);
+void erts_cancel_proc_timer(Process *);
+void erts_set_port_timer(Port *, Sint64);
+void erts_cancel_port_timer(Port *);
+Sint64 erts_read_port_timer(Port *);
+int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **);
+int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **);
+ErtsHLTimerService *erts_create_timer_service(void);
+void erts_hl_timer_init(void);
+
+#ifdef ERTS_SMP
+void
+erts_handle_canceled_timers(void *vesdp,
+			    int *need_thr_progress,
+			    ErtsThrPrgrVal *thr_prgr_p,
+			    int *need_more_work);
+#endif
+
+Uint erts_bif_timer_memory_size(void);
+void erts_print_bif_timer_info(int to, void *to_arg);
+
+void erts_debug_bif_timer_foreach(void (*func)(Eterm,
+					       Eterm,
+					       ErlHeapFragment *,
+					       void *),
+				  void *arg);
+
+#endif /* ERL_HL_TIMER_H__ */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 86d3416423..9769f36e42 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -35,7 +35,7 @@
 #include "dist.h"
 #include "erl_mseg.h"
 #include "erl_threads.h"
-#include "erl_bif_timer.h"
+#include "erl_hl_timer.h"
 #include "erl_instrument.h"
 #include "erl_printf_term.h"
 #include "erl_misc_utils.h"
@@ -46,6 +46,8 @@
 #include "erl_async.h"
 #include "erl_ptab.h"
 #include "erl_bif_unique.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
 
 #ifdef HIPE
 #include "hipe_mode_switch.h"	/* for hipe_mode_switch_init() */
@@ -363,7 +365,6 @@ erl_init(int ncpu,
     erts_init_binary(); /* Must be after init_emulator() */
     erts_bp_init();
     init_db(); /* Must be after init_emulator */
-    erts_bif_timer_init();
     erts_init_node_tables();
     init_dist();
     erl_drv_thr_init();
@@ -2098,11 +2099,8 @@ erl_start(int argc, char **argv)
 
     erts_initialized = 1;
 
-    {
-	Eterm init = erl_first_process_otp("otp_ring0", NULL, 0,
-					   boot_argc, boot_argv);
-	erts_bif_timer_start_servers(init);
-    }
+    (void) erl_first_process_otp("otp_ring0", NULL, 0,
+				 boot_argc, boot_argv);
 
 #ifdef ERTS_SMP
     erts_start_schedulers();
@@ -2110,13 +2108,17 @@ erl_start(int argc, char **argv)
 
     erts_sys_main_thread(); /* May or may not return! */
 #else
-    erts_thr_set_main_status(1, 1);
+    {
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+	erts_thr_set_main_status(1, 1);
 #if ERTS_USE_ASYNC_READY_Q
-    erts_get_scheduler_data()->aux_work_data.async_ready.queue
-	= erts_get_async_ready_queue(1);
+	esdp->aux_work_data.async_ready.queue
+	    = erts_get_async_ready_queue(1);
 #endif
-    set_main_stack_size();
-    process_main();
+	set_main_stack_size();
+	erts_sched_init_time_sup(esdp);
+	process_main();
+    }
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 261460d054..617ce84895 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -91,6 +91,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {   "driver_list",                          NULL                    },
     {	"proc_link",				"pid"			},
     {	"proc_msgq",				"pid"			},
+    {	"proc_btm",				"pid"			},
     {	"dist_entry",				"address"		},
     {	"dist_entry_links",			"address"		},
     {   "code_write_permission",                NULL                    },
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 22cbae10d1..43a03c793e 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -994,7 +994,7 @@ erts_send_message(Process* sender,
 #endif
 	    );
         BM_SWAP_TIMER(send,system);
-    } else if (sender == receiver && !(sender->flags & F_OFF_HEAP_MSGS)) {
+    } else if (sender == receiver) {
 	/* Drop message if receiver has a pending exit ... */
 #ifdef ERTS_SMP
 	ErtsProcLocks need_locks = (~(*receiver_locks)
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 8713941769..6b8c3cebc7 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -213,25 +213,15 @@ do {									\
     if ((M)->data.attached) {						\
 	Uint need__ = erts_msg_attached_data_size((M));			\
  	if ((ST) - (HT) >= need__) {					\
-	    Uint *htop__;						\
-	move__attached__msg__data____:					\
-	    htop__ = (HT);						\
+	    Uint *htop__ = (HT);					\
 	    erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\
 	    ASSERT(htop__ - (HT) <= need__);				\
 	    (HT) = htop__;						\
 	}								\
 	else {								\
-	    int off_heap_msgs__ = (int) (P)->flags & F_OFF_HEAP_MSGS;	\
-	    if (!off_heap_msgs__)					\
-		need__ = 0;						\
 	    { SWPO ; }							\
-	    (FC) -= erts_garbage_collect((P), need__, NULL, 0);		\
+	    (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
 	    { SWPI ; }							\
-	    if (off_heap_msgs__) {					\
-		ASSERT((M)->data.attached);				\
-		ASSERT((ST) - (HT) >= need__);				\
-		goto move__attached__msg__data____;			\
-	    }								\
 	}								\
 	ASSERT(!(M)->data.attached);					\
     }									\
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 660f446a52..62f018bd41 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -335,7 +335,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     rp = (scheduler
 	  ? erts_proc_lookup(receiver)
 	  : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
-			      receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC));
+			      receiver, rp_locks, ERTS_P2P_FLG_INC_REFC));
     if (rp == NULL) {
 	ASSERT(env == NULL || receiver != c_p->common.id);
 	return 0;
@@ -367,7 +367,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
-	erts_smp_proc_dec_refc(rp);
+	erts_proc_dec_refc(rp);
     if (flush_me) {
 	cache_env(env);
     }
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index c6d136f951..bcf6311079 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1469,7 +1469,7 @@ setup_reference_table(void)
     erts_db_foreach_table(insert_ets_table, NULL);
 
     /* Insert all bif timers */
-    erts_bif_timer_foreach(insert_bif_timer, NULL);
+    erts_debug_bif_timer_foreach(insert_bif_timer, NULL);
 
     /* Insert node table (references to dist) */
     hash_foreach(&erts_node_table, insert_erl_node, NULL);
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index ad3f104a68..3920fae2d9 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -350,6 +350,7 @@ int erts_lc_is_port_locked(Port *);
 ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt);
 ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt);
 ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc);
+ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt);
 
 ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
 ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
@@ -359,37 +360,26 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);
 
 ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt)
 {
-#ifdef ERTS_SMP
-    erts_ptab_inc_refc(&prt->common);
-#else
-    erts_atomic32_inc_nob(&prt->refc);
-#endif
+    erts_ptab_atmc_inc_refc(&prt->common);
 }
 
 ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt)
 {
-#ifdef ERTS_SMP
-    int referred = erts_ptab_dec_test_refc(&prt->common);
+    int referred = erts_ptab_atmc_dec_test_refc(&prt->common);
     if (!referred)
 	erts_port_free(prt);
-#else
-    int refc = erts_atomic32_dec_read_nob(&prt->refc);
-    if (refc == 0)
-	erts_port_free(prt);	
-#endif
 }
 
 ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc)
 {
-#ifdef ERTS_SMP
-    int referred = erts_ptab_add_test_refc(&prt->common, add_refc);
+    int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc);
     if (!referred)
 	erts_port_free(prt);
-#else
-    int refc = erts_atomic32_add_read_nob(&prt->refc, add_refc);
-    if (refc == 0)
-	erts_port_free(prt);	
-#endif
+}
+
+ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
+{
+    return erts_ptab_atmc_read_refc(&prt->common);
 }
 
 ERTS_GLB_INLINE int
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 2aa0a27197..c701737e26 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1646,6 +1646,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_aint32_t state;
     int active;
     Uint64 start_time = 0;
+    ErtsSchedulerData *esdp = runq->scheduler;
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
 
@@ -1662,7 +1663,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     *curr_port_pp = pp;
     
     if (erts_sched_stat.enabled) {
-	ErtsSchedulerData *esdp = erts_get_scheduler_data();
 	Uint old = ERTS_PORT_SCHED_ID(pp, esdp->no);
 	int migrated = old && old != esdp->no;
 
@@ -1718,11 +1718,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	switch (ptp->type) {
 	case ERTS_PORT_TASK_TIMEOUT:
 	    reset_handle(ptp);
-	    reds = ERTS_PORT_REDS_TIMEOUT;
-	    if (!(state & ERTS_PORT_SFLGS_DEAD)) {
-                DTRACE_DRIVER(driver_timeout, pp);
-		(*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data);
-            }
+	    if (!ERTS_PTMR_IS_TIMED_OUT(pp))
+		reds = 0;
+	    else {
+		ERTS_PTMR_CLEAR(pp);
+		reds = ERTS_PORT_REDS_TIMEOUT;
+		if (!(state & ERTS_PORT_SFLGS_DEAD)) {
+		    DTRACE_DRIVER(driver_timeout, pp);
+		    (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data);
+		}
+	    }
 	    break;
 	case ERTS_PORT_TASK_INPUT:
 	    reds = ERTS_PORT_REDS_INPUT;
@@ -1879,7 +1884,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     runq->scheduler->reductions += reds;
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
-    ERTS_PORT_REDUCTIONS_EXECUTED(runq, reds);
+    ERTS_PORT_REDUCTIONS_EXECUTED(esdp, runq, reds);
 
     return res;
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f74a2ee54c..588bc75a43 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -44,8 +44,10 @@
 #include "dtrace-wrapper.h"
 #include "erl_ptab.h"
 #include "erl_bif_unique.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
 
-
+#define ERTS_CHECK_TIME_REDS CONTEXT_REDS
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
 #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2)
 
@@ -463,7 +465,7 @@ static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg);
 static void aux_work_timeout(void *unused);
 static void aux_work_timeout_early_init(int no_schedulers);
 static void aux_work_timeout_late_init(void);
-static void setup_aux_work_timer(void);
+static void setup_aux_work_timer(ErtsSchedulerData *esdp);
 
 static int execute_sys_tasks(Process *c_p,
 			     erts_aint32_t *statep,
@@ -493,6 +495,8 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_DD;
     valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR;
+    valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
+    valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
 #endif
 #if HAVE_ERTS_MSEG
@@ -610,13 +614,11 @@ erts_pre_init_process(void)
 #endif
 }
 
-#ifdef ERTS_SMP
 static void
 release_process(void *vproc)
 {
-    erts_smp_proc_dec_refc((Process *) vproc);
+    erts_proc_dec_refc((Process *) vproc);
 }
-#endif
 
 /* initialize the scheduler */
 void
@@ -632,16 +634,18 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab)
 
     erts_ptab_init_table(&erts_proc,
 			 ERTS_ALC_T_PROC_TABLE,
-#ifdef ERTS_SMP
 			 release_process,
-#else
-			 NULL,
-#endif
 			 (ErtsPTabElementCommon *) &erts_invalid_process.common,
 			 proc_tab_size,
 			 sizeof(Process),
 			 "process_table",
-			 legacy_proc_tab);
+			 legacy_proc_tab,
+#ifdef ERTS_SMP
+			 1
+#else
+			 0
+#endif
+	);
 
     last_reductions = 0;
     last_exact_reductions = 0;
@@ -1035,7 +1039,7 @@ reply_sched_wall_time(void *vswtrp)
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
 
-    erts_smp_proc_dec_refc(rp);
+    erts_proc_dec_refc(rp);
 
     if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0)
 	swtreq_free(vswtrp);
@@ -1067,7 +1071,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable)
     erts_smp_atomic32_init_nob(&swtrp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
-    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+    erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
 
 #ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
@@ -1128,7 +1132,7 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
     xplocks &= ~plocks;
     if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) {
 	if (xplocks & ERTS_PROC_LOCK_MAIN) {
-	    erts_smp_proc_inc_refc(p);
+	    erts_proc_inc_refc(p);
 	    erts_smp_proc_unlock(p, plocks);
 	    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL);
 	    refc = 1;
@@ -1144,7 +1148,7 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
     if (xplocks)
 	erts_smp_proc_unlock(p, xplocks);
     if (refc)
-	erts_smp_proc_dec_refc(p);
+	erts_proc_dec_refc(p);
     ASSERT(p->psd);
     if (p->psd != psd)
 	erts_free(ERTS_ALC_T_PSD, psd);
@@ -1771,6 +1775,101 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 }
 
+/*
+ * Canceled timers
+ */
+
+void
+erts_notify_canceled_timer(ErtsSchedulerData *esdp, int rsid)
+{
+    ASSERT(esdp && esdp == erts_get_scheduler_data());
+    if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp))
+	schedule_aux_work_wakeup(&esdp->aux_work_data,
+				 rsid,
+				 ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+    else
+	set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(rsid-1),
+				       ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    int need_thr_progress = 0;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    int more_work = 0;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+    erts_handle_canceled_timers((void *) awdp->esdp,
+				&need_thr_progress,
+				&wakeup,
+				&more_work);
+    if (more_work) {
+	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS)
+	    & ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) {
+	    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+	    aux_work &= ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
+	}
+	return aux_work;
+    }
+
+    if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later(awdp->esdp);
+	awdp->cncld_tmrs.thr_prgr = wakeup;
+	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+	haw_thr_prgr_soft_wakeup(awdp, wakeup);
+    }
+    return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS;
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_canceled_timers_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsSchedulerSleepInfo *ssi;
+    int need_thr_progress;
+    int more_work;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+    if (!erts_thr_progress_has_reached_this(current, awdp->cncld_tmrs.thr_prgr))
+	return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
+
+    ssi = awdp->ssi;
+    need_thr_progress = 0;
+    more_work = 0;
+
+    erts_handle_canceled_timers((void *) awdp->esdp,
+				&need_thr_progress,
+				&wakeup,
+				&more_work);
+    if (more_work) {
+	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+	return ((aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR)
+		| ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+    }
+
+    if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later(awdp->esdp);
+	awdp->cncld_tmrs.thr_prgr = wakeup;
+	haw_thr_prgr_soft_wakeup(awdp, wakeup);
+    }
+    else {
+	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+    }
+
+    return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
+}
+
 /*
  * Handle scheduled thread progress later operations.
  */
@@ -1869,7 +1968,7 @@ completed_dealloc(void *vproc)
 {
     if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) {
 	erts_resume((Process *) vproc, (ErtsProcLocks) 0);
-	erts_smp_proc_dec_refc((Process *) vproc);
+	erts_proc_dec_refc((Process *) vproc);
     }
 }
 
@@ -1918,7 +2017,7 @@ erts_debug_wait_deallocations(Process *c_p)
 				      count,
 				      0)) {
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
-	erts_smp_proc_inc_refc(c_p);
+	erts_proc_inc_refc(c_p);
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
@@ -2033,7 +2132,7 @@ static ERTS_INLINE erts_aint32_t
 handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO);
-    setup_aux_work_timer();
+    setup_aux_work_timer(awdp->esdp);
     return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO;
 }
 
@@ -2093,6 +2192,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 #ifdef ERTS_SMP
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP,
 		    handle_thr_prgr_later_op);
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS,
+		    handle_canceled_timers);
+    /* CNCLD_TMRS must be before CNCLD_TMRS_THR_PRGR */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR,
+		    handle_canceled_timers_thr_prgr);
 #endif
 
 #if ERTS_USE_ASYNC_READY_Q
@@ -2142,8 +2246,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 
 typedef struct {
     union {
-	ErlTimer data;
-	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))];
+	ErtsTWheelTimer data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsTWheelTimer))];
     } timer;
 
     int initialized;
@@ -2153,6 +2257,22 @@ typedef struct {
 
 static ErtsAuxWorkTmo *aux_work_tmo;
 
+static ERTS_INLINE void
+start_aux_work_timer(ErtsSchedulerData *esdp)
+{
+    ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp);
+    tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1);
+    tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1;
+    erts_twheel_init_timer(&aux_work_tmo->timer.data);
+    ASSERT(esdp);
+    erts_twheel_set_timer(esdp->timer_wheel,
+			  &aux_work_tmo->timer.data,
+			  aux_work_timeout,
+			  NULL,
+			  (void *) esdp,
+			  tmo);
+}
+
 static void
 aux_work_timeout_early_init(int no_schedulers)
 {
@@ -2185,18 +2305,12 @@ void
 aux_work_timeout_late_init(void)
 {
     aux_work_tmo->initialized = 1;
-    if (erts_atomic32_read_nob(&aux_work_tmo->refc)) {
-	erts_init_timer(&aux_work_tmo->timer.data);
-	erts_set_timer(&aux_work_tmo->timer.data,
-		       aux_work_timeout,
-		       NULL,
-		       NULL,
-		       1000);
-    }
+    if (erts_atomic32_read_nob(&aux_work_tmo->refc))
+	start_aux_work_timer(erts_get_scheduler_data());
 }
 
 static void
-aux_work_timeout(void *unused)
+aux_work_timeout(void *vesdp)
 {
     erts_aint32_t refc;
     int i;
@@ -2219,31 +2333,18 @@ aux_work_timeout(void *unused)
     if (refc != 1
 	|| 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) {
 	/* Setup next timeout... */
-	erts_set_timer(&aux_work_tmo->timer.data,
-		       aux_work_timeout,
-		       NULL,
-		       NULL,
-		       1000);
+	start_aux_work_timer((ErtsSchedulerData *) vesdp);
     }
 }
 
 static void
-setup_aux_work_timer(void)
+setup_aux_work_timer(ErtsSchedulerData *esdp)
 {
-#ifndef ERTS_SMP
-    if (!erts_get_scheduler_data())
+    if (!esdp || !esdp->timer_wheel)
 	set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0),
 				      ERTS_SSI_AUX_WORK_SET_TMO);
     else
-#endif
-    {
-	erts_init_timer(&aux_work_tmo->timer.data);
-	erts_set_timer(&aux_work_tmo->timer.data,
-		       aux_work_timeout,
-		       NULL,
-		       NULL,
-		       1000);
-    }
+	start_aux_work_timer(esdp);
 }
 
 erts_aint32_t
@@ -2274,7 +2375,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable)
 	if (refc == 1) {
 	    erts_atomic32_inc_acqb(&aux_work_tmo->refc);
 	    if (aux_work_tmo->initialized) 
-		setup_aux_work_timer();
+		setup_aux_work_timer(erts_get_scheduler_data());
 	}
     }
     return old;
@@ -2653,11 +2754,6 @@ aux_thread(void *unused)
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     int thr_prgr_active = 1;
-    ErtsTimerWheel *timer_wheel = erts_default_timer_wheel;
-    ErtsNextTimeoutRef nxt_tmo_ref = erts_get_next_timeout_reference(timer_wheel);
-
-    if (!timer_wheel)
-	ERTS_INTERNAL_ERROR("Missing aux timer wheel");
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -2681,7 +2777,6 @@ aux_thread(void *unused)
     sched_prep_spin_wait(ssi);
 
     while (1) {
-	ErtsMonotonicTime current_time;
 	erts_aint32_t flgs;
 
 	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
@@ -2693,56 +2788,28 @@ aux_thread(void *unused)
 		erts_thr_progress_leader_update(NULL);
 	}
 
-	if (aux_work) {
-	    current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time(nxt_tmo_ref)) {
-		if (!thr_prgr_active)
-		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
-		erts_bump_timers(timer_wheel, current_time);
-	    }
-	}
-	else {
-	    ErtsMonotonicTime timeout_time;
-	    timeout_time = erts_check_next_timeout_time(timer_wheel,
-							ERTS_SEC_TO_MONOTONIC(10*60));
-	    current_time = erts_get_monotonic_time();
-	    if (current_time >= timeout_time) {
-		if (!thr_prgr_active)
-		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
-	    }
-	    else {
-		if (thr_prgr_active)
-		    erts_thr_progress_active(NULL, thr_prgr_active = 0);
-		erts_thr_progress_prepare_wait(NULL);
+	if (!aux_work) {
+	    if (thr_prgr_active)
+		erts_thr_progress_active(NULL, thr_prgr_active = 0);
+	    erts_thr_progress_prepare_wait(NULL);
 
-		ERTS_SCHED_FAIR_YIELD();
+	    ERTS_SCHED_FAIR_YIELD();
 
-		flgs = sched_spin_wait(ssi, 0);
+	    flgs = sched_spin_wait(ssi, 0);
 
+	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		    int res;
+		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
-		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-			int res;
-			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
-			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			current_time = erts_get_monotonic_time();
-			do {
-			    Sint64 timeout;
-			    if (current_time >= timeout_time)
-				break;
-			    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
-							     - current_time
-							     - 1) + 1;
-			    res = erts_tse_twait(ssi->event, timeout);
-			    current_time = erts_get_monotonic_time();
-			} while (res == EINTR);
-		    }
+		    do {
+			res = erts_tse_wait(ssi->event);
+		    } while (res == EINTR);
 		}
-		erts_thr_progress_finalize_wait(NULL);
 	    }
-	    if (current_time >= timeout_time)
-		erts_bump_timers(timer_wheel, current_time);
+	    erts_thr_progress_finalize_wait(NULL);
 	}
 
 	flgs = sched_prep_spin_wait(ssi);
@@ -2825,7 +2892,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	    if (aux_work) {
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
-		current_time = erts_get_monotonic_time();
+		current_time = erts_get_monotonic_time(esdp);
 		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
 		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -2836,9 +2903,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    }
 	    else {
 		ErtsMonotonicTime timeout_time;
-		timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
-							    ERTS_SEC_TO_MONOTONIC(10*60));
-		current_time = erts_get_monotonic_time();
+		timeout_time = erts_check_next_timeout_time(esdp);
+		current_time = erts_get_monotonic_time(esdp);
 		if (current_time >= timeout_time) {
 		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -2864,7 +2930,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 			    int res;
 			    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 			    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			    current_time = erts_get_monotonic_time();
+			    current_time = erts_get_monotonic_time(esdp);
 			    do {
 				Sint64 timeout;
 				if (current_time >= timeout_time)
@@ -2873,7 +2939,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 								 - current_time
 								 - 1) + 1;
 				res = erts_tse_twait(ssi->event, timeout);
-				current_time = erts_get_monotonic_time();
+				current_time = erts_get_monotonic_time(esdp);
 			    } while (res == EINTR);
 			}
 		    }
@@ -2948,7 +3014,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
 	    erl_sys_schedule(1); /* Might give us something to do */
 
-	    current_time = erts_get_monotonic_time();
+	    current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 
@@ -3066,7 +3132,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	erl_sys_schedule(0);
 
 	{
-	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
+	    ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 	}
@@ -5275,6 +5341,7 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.completed_callback = NULL;
     awdp->dd.completed_arg = NULL;
+    awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST;
     awdp->later_op.size = 0;
     awdp->later_op.first = NULL;
@@ -5340,9 +5407,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->no = (Uint) num;
 #endif
 
-    esdp->timer_wheel = erts_default_timer_wheel;
-    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
-
     esdp->ssi = ssi;
     esdp->current_process = NULL;
     esdp->current_port = NULL;
@@ -5355,6 +5419,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->run_queue = runq;
     esdp->run_queue->scheduler = esdp;
 
+    esdp->last_monotonic_time = 0;
+    esdp->check_time_reds = 0;
+
     esdp->thr_id = (Uint32) num;
     erts_sched_bif_unique_init(esdp);
 
@@ -5918,13 +5985,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
     int check_emigration_need;
 #endif
 
-#ifdef ERTS_SMP
-    if ((p->static_flags & ERTS_STC_FLG_PREFER_SCHED)
-	&& p->preferred_run_queue != RUNQ_READ_RQ(&p->run_queue)) {
-	RUNQ_SET_RQ(&p->run_queue, p->preferred_run_queue);
-    }
-#endif
-
     a = state;
 
     while (1) {
@@ -6732,6 +6792,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	}
     }
 
+    (void) erts_get_monotonic_time(esdp);
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
@@ -6865,7 +6926,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		}
 
 		if (aux_work) {
-		    current_time = erts_get_monotonic_time();
+		    current_time = erts_get_monotonic_time(esdp);
 		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
 			if (!thr_prgr_active) {
 			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -6876,9 +6937,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		}
 		else {
 		    ErtsMonotonicTime timeout_time;
-		    timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
-								ERTS_SEC_TO_MONOTONIC(60*60));
-		    current_time = erts_get_monotonic_time();
+		    timeout_time = erts_check_next_timeout_time(esdp);
+		    current_time = erts_get_monotonic_time(esdp);
 
 		    if (current_time >= timeout_time) {
 			if (!thr_prgr_active) {
@@ -6904,7 +6964,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 					 | ERTS_SSI_FLG_SUSPENDED)) {
 				int res;
 
-				current_time = erts_get_monotonic_time();
+				current_time = erts_get_monotonic_time(esdp);
 				do {
 				    Sint64 timeout;
 				    if (current_time >= timeout_time)
@@ -6913,7 +6973,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 								     - current_time
 								     - 1) + 1;
 				    res = erts_tse_twait(ssi->event, timeout);
-				    current_time = erts_get_monotonic_time();
+				    current_time = erts_get_monotonic_time(esdp);
 				} while (res == EINTR);
 			    }
 			}
@@ -7741,8 +7801,8 @@ sched_thread_func(void *vesdp)
     ErtsSchedulerData *esdp = vesdp;
     Uint no = esdp->no;
 
-    esdp->timer_wheel = erts_create_timer_wheel((int) no);
-    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
+    erts_sched_init_time_sup(esdp);
+
 #ifdef ERTS_SMP
     ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
@@ -9111,7 +9171,7 @@ Process *schedule(Process *p, int calls)
 	schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */
 	proxy_p = NULL;
 
-	ERTS_PROC_REDUCTIONS_EXECUTED(rq,
+	ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
 				      (int) ERTS_PSFLGS_GET_USR_PRIO(state),
 				      reds,
 				      actual_reds);
@@ -9128,12 +9188,9 @@ Process *schedule(Process *p, int calls)
 	    ASSERT(esdp->free_process == p);
 	    esdp->free_process = NULL;
 #else
-	    state = erts_smp_atomic32_read_nob(&p->state);
-	    if (!(state & ERTS_PSFLG_IN_RUNQ))
-		erts_free_proc(p);
+	    erts_proc_dec_refc(p);
 #endif
 	}
-
 #ifdef ERTS_SMP
 	ASSERT(!esdp->free_process);
 #endif
@@ -9141,13 +9198,13 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-	{
-	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-		erts_smp_runq_unlock(rq);
-		erts_bump_timers(esdp->timer_wheel, current_time);
-		erts_smp_runq_lock(rq);
-	    }
+	if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
+	    (void) erts_get_monotonic_time(esdp);
+
+	if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+	    erts_smp_runq_unlock(rq);
+	    erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time);
+	    erts_smp_runq_lock(rq);
 	}
 	BM_STOP_TIMER(system);
 
@@ -9307,7 +9364,7 @@ Process *schedule(Process *p, int calls)
 	    erts_smp_runq_unlock(rq);
 	    erl_sys_schedule(1);
 
-	    current_time = erts_get_monotonic_time();
+	    current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 
@@ -9448,13 +9505,8 @@ Process *schedule(Process *p, int calls)
 				      | ERTS_PSFLG_PENDING_EXIT
 				      | ERTS_PSFLG_ACTIVE_SYS))
 			    == ERTS_PSFLG_SUSPENDED)) {
-			if (state & ERTS_PSFLG_FREE) {
-#ifdef ERTS_SMP
-			    erts_smp_proc_dec_refc(p);
-#else
-			    erts_free_proc(p);
-#endif
-			}
+			if (state & ERTS_PSFLG_FREE)
+			    erts_proc_dec_refc(p);
 			if (proxy_p) {
 			    free_proxy_proc(proxy_p);
 			    proxy_p = NULL;
@@ -9597,6 +9649,20 @@ Process *schedule(Process *p, int calls)
 	/* Never run a suspended process */
 	ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
 
+	ASSERT(erts_proc_read_refc(p) > 0);
+
+	if (ERTS_PTMR_IS_TIMED_OUT(p)) {
+	    BeamInstr** pi;
+#ifdef ERTS_SMP
+	    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+	    pi = (BeamInstr **) p->def_arg_reg;
+	    p->i = *pi;
+	    p->flags &= ~F_INSLPQUEUE;
+	    p->flags |= F_TIMO;
+	    ERTS_PTMR_CLEAR(p);
+	}
+
 	return p;
     }
 }
@@ -10513,6 +10579,8 @@ erts_free_proc(Process *p)
 #ifdef ERTS_SMP
     erts_proc_lock_fin(p);
 #endif
+    ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE);
+    ASSERT(0 == erts_proc_read_refc(p));
     erts_free(ERTS_ALC_T_PROC, (void *) p);
 }
 
@@ -10565,6 +10633,8 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
 	return NULL;
     }
 
+    ASSERT(erts_proc_read_refc(p) > 0);
+
     ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL);
     
     p->approx_started = erts_get_approx_time();
@@ -10614,10 +10684,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	    int ix = so->scheduler-1;
 	    ASSERT(0 <= ix && ix < erts_no_run_queues);
 	    rq = ERTS_RUNQ_IX(ix);
-	    if (!(so->flags & SPO_PREFER_SCHED)) {
-		/* Unsupported feature... */
-		state |= ERTS_PSFLG_BOUND;
-	    }
+	    /* Unsupported feature... */
+	    state |= ERTS_PSFLG_BOUND;
 	}
 	prio = (erts_aint32_t) so->priority;
     }
@@ -10625,9 +10693,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET)
 	      | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET));
 
-    if (so->flags & SPO_OFF_HEAP_MSGS)
-	state |= ERTS_PSFLG_OFF_HEAP_MSGS;
-
     if (!rq)
 	rq = erts_get_runq_proc(parent);
 
@@ -10651,12 +10716,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     heap_need = arg_size;
 
     p->flags = erts_default_process_flags;
-    if (so->flags & SPO_OFF_HEAP_MSGS)
-	p->flags |= F_OFF_HEAP_MSGS;
 
-#ifdef ERTS_SMP
-    p->preferred_run_queue = NULL;
-#endif
     p->static_flags = 0;
     if (so->flags & SPO_SYSTEM_PROC)
 	p->static_flags |= ERTS_STC_FLG_SYSTEM_PROC;
@@ -10664,12 +10724,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->min_heap_size  = so->min_heap_size;
 	p->min_vheap_size = so->min_vheap_size;
 	p->max_gen_gcs    = so->max_gen_gcs;
-	if (so->flags & SPO_PREFER_SCHED) {
-#ifdef ERTS_SMP
-	    p->preferred_run_queue = rq;
-#endif
-	    p->static_flags |= ERTS_STC_FLG_PREFER_SCHED;
-	}
     } else {
 	p->min_heap_size  = H_MIN_SIZE;
 	p->min_vheap_size = BIN_VH_MIN_SIZE;
@@ -10678,9 +10732,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->schedule_count = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
     
-    p->initial[INITIAL_MOD] = mod;
-    p->initial[INITIAL_FUN] = func;
-    p->initial[INITIAL_ARI] = (Uint) arity;
+    p->u.initial[INITIAL_MOD] = mod;
+    p->u.initial[INITIAL_FUN] = func;
+    p->u.initial[INITIAL_ARI] = (Uint) arity;
 
     /*
      * Must initialize binary lists here before copying binaries to process.
@@ -10721,7 +10775,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
     /* No need to initialize p->fcalls. */
 
-    p->current = p->initial+INITIAL_MOD;
+    p->current = p->u.initial+INITIAL_MOD;
 
     p->i = (BeamInstr *) beam_apply;
     p->cp = (BeamInstr *) beam_apply+1;
@@ -10744,11 +10798,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->ftrace = NIL;
     p->reds = 0;
 
-#ifdef ERTS_SMP
-    p->common.u.alive.ptimer = NULL;
-#else
-    erts_init_timer(&p->common.u.alive.tm);
-#endif
+    ERTS_PTMR_INIT(p);
 
     p->common.u.alive.reg = NULL;
     ERTS_P_LINKS(p) = NULL;
@@ -10779,7 +10829,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg_inq.last = &p->msg_inq.first;
     p->msg_inq.len = 0;
 #endif
-    p->u.bif_timers = NULL;
+    p->bif_timers = NULL;
+    p->accessor_bif_timers = NULL;
     p->mbuf = NULL;
     p->mbuf_sz = 0;
     p->psd = NULL;
@@ -10937,11 +10988,7 @@ void erts_init_empty_process(Process *p)
     p->bin_old_vheap = 0;
     p->sys_task_qs = NULL;
     p->bin_vheap_mature = 0;
-#ifdef ERTS_SMP
-    p->common.u.alive.ptimer = NULL;
-#else
-    erts_init_timer(&p->common.u.alive.tm);
-#endif
+    ERTS_PTMR_INIT(p);
     p->next = NULL;
     p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
@@ -10962,14 +11009,15 @@ void erts_init_empty_process(Process *p)
     p->msg.last = &p->msg.first;
     p->msg.save = &p->msg.first;
     p->msg.len = 0;
-    p->u.bif_timers = NULL;
+    p->bif_timers = NULL;
+    p->accessor_bif_timers = NULL;
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
     p->seq_trace_lastcnt = 0;
     p->seq_trace_token = NIL;
-    p->initial[0] = 0;
-    p->initial[1] = 0;
-    p->initial[2] = 0;
+    p->u.initial[0] = 0;
+    p->u.initial[1] = 0;
+    p->u.initial[2] = 0;
     p->catches = 0;
     p->cp = NULL;
     p->i = NULL;
@@ -11017,7 +11065,6 @@ void erts_init_empty_process(Process *p)
     p->pending_suspenders = NULL;
     p->pending_exit.reason = THE_NON_VALUE;
     p->pending_exit.bp = NULL;
-    p->preferred_run_queue = NULL;
     erts_proc_lock_init(p);
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
@@ -11057,7 +11104,8 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->suspend_monitors == NULL);
     ASSERT(p->msg.first == NULL);
     ASSERT(p->msg.len == 0);
-    ASSERT(p->u.bif_timers == NULL);
+    ASSERT(p->bif_timers == NULL);
+    ASSERT(p->accessor_bif_timers == NULL);
     ASSERT(p->dictionary == NULL);
     ASSERT(p->catches == 0);
     ASSERT(p->cp == NULL);
@@ -11221,7 +11269,6 @@ set_proc_exiting(Process *p,
      */
     p->freason = EXTAG_EXIT;
     KILL_CATCHES(p);
-    cancel_timer(p);
     p->i = (BeamInstr *) beam_exit;
 
     if (enqueue)
@@ -11919,6 +11966,7 @@ erts_do_exit_process(Process* p, Eterm reason)
 {
     p->arity = 0;		/* No live registers */
     p->fvalue = reason;
+    
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(process_exit)) {
@@ -11973,20 +12021,20 @@ erts_do_exit_process(Process* p, Eterm reason)
     ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS)
 	   == F_INITIAL_TRACE_FLAGS);
 
-    cancel_timer(p);		/* Always cancel timer just in case */
-
-    if (p->u.bif_timers)
-	erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL);
+    ASSERT(erts_proc_read_refc(p) > 0);
+    if (ERTS_PTMR_IS_SET(p)) {
+	erts_cancel_proc_timer(p);
+	ASSERT(erts_proc_read_refc(p) > 0);
+    }
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 
     /*
-     * The p->u.bif_timers of this process can *not* be used anymore;
+     * p->u.initial of this process can *not* be used anymore;
      * will be overwritten by misc termination data.
      */
     p->u.terminate = NULL;
 
-
     erts_continue_exit_process(p);
 }
 
@@ -12011,6 +12059,27 @@ erts_continue_exit_process(Process *p)
 
     ASSERT(ERTS_PROC_IS_EXITING(p));
 
+    ASSERT(erts_proc_read_refc(p) > 0);
+    if (p->bif_timers) {
+	if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) {
+	    ASSERT(erts_proc_read_refc(p) > 0);
+	    goto yield;
+	}
+	ASSERT(erts_proc_read_refc(p) > 0);
+	p->bif_timers = NULL;
+    }
+
+    if (p->accessor_bif_timers) {
+	if (erts_detach_accessor_bif_timers(p,
+					    p->accessor_bif_timers,
+					    &p->u.terminate)) {
+	    ASSERT(erts_proc_read_refc(p) > 0);
+	    goto yield;
+	}
+	ASSERT(erts_proc_read_refc(p) > 0);
+	p->accessor_bif_timers = NULL;
+    }
+
 #ifdef ERTS_SMP
     if (p->flags & F_HAVE_BLCKD_MSCHED) {
 	ErtsSchedSuspendResult ssr;
@@ -12104,6 +12173,8 @@ erts_continue_exit_process(Process *p)
 
 	p->scheduler_data->current_process = NULL;
 	p->scheduler_data->free_process = p;
+#else
+	erts_proc_inc_refc(p); /* Decremented in schedule() */
 #endif
 
 	/* Time of death! */
@@ -12122,29 +12193,23 @@ erts_continue_exit_process(Process *p)
     {
 	/* Inactivate and notify free */
 	erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state);
-#ifdef ERTS_SMP
 	int refc_inced = 0;
-#endif
 	while (1) {
 	    n = e = a;
 	    ASSERT(a & ERTS_PSFLG_EXITING);
 	    n |= ERTS_PSFLG_FREE;
 	    n &= ~ERTS_PSFLG_ACTIVE;
-#ifdef ERTS_SMP
 	    if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) {
-		erts_smp_proc_inc_refc(p);
+		erts_proc_inc_refc(p);
 		refc_inced = 1;
 	    }
-#endif
 	    a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	    if (a == e)
 		break;
 	}
 
-#ifdef ERTS_SMP
 	if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ))
-	    erts_smp_proc_dec_refc(p);
-#endif
+	    erts_proc_dec_refc(p);
     }
     
     dep = ((p->flags & F_DISTRIBUTION)
@@ -12235,64 +12300,6 @@ erts_continue_exit_process(Process *p)
 
 }
 
-/* Callback for process timeout */
-static void
-timeout_proc(Process* p)
-{
-    erts_aint32_t state;
-    BeamInstr** pi = (BeamInstr **) p->def_arg_reg;
-    p->i = *pi;
-    p->flags |= F_TIMO;
-    p->flags &= ~F_INSLPQUEUE;
-
-    state = erts_smp_atomic32_read_acqb(&p->state);
-    if (!(state & ERTS_PSFLG_ACTIVE))
-	schedule_process(p, state, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-}
-
-
-void
-cancel_timer(Process* p)
-{
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
-    p->flags &= ~(F_INSLPQUEUE|F_TIMO);
-#ifdef ERTS_SMP
-    erts_cancel_smp_ptimer(p->common.u.alive.ptimer);
-#else
-    erts_cancel_timer(&p->common.u.alive.tm);
-#endif
-}
-
-/*
- * Insert a process into the time queue, with a timeout 'timeout' in ms.
- */
-void
-set_timer(Process* p, Uint timeout)
-{
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
-
-    /* check for special case timeout=0 DONT ADD TO time queue */
-    if (timeout == 0) {
-	p->flags |= F_TIMO;
-	return;
-    }
-    p->flags |= F_INSLPQUEUE;
-    p->flags &= ~F_TIMO;
-
-#ifdef ERTS_SMP
-    erts_create_smp_ptimer(&p->common.u.alive.ptimer,
-			   p->common.id,
-			   (ErlTimeoutProc) timeout_proc,
-			   timeout);
-#else
-    erts_set_timer(&p->common.u.alive.tm,
-		  (ErlTimeoutProc) timeout_proc,
-		  NULL,
-		  (void*) p,
-		  timeout);
-#endif
-}
-
 /*
  * Stack dump functions follow.
  */
@@ -12450,6 +12457,10 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
                 erts_print(to, to_arg, "FIX_ALLOC_LOWER_LIM"); break;
             case ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP:
                 erts_print(to, to_arg, "THR_PRGR_LATER_OP"); break;
+            case ERTS_SSI_AUX_WORK_CNCLD_TMRS:
+                erts_print(to, to_arg, "CANCELED_TIMERS"); break;
+            case ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR:
+                erts_print(to, to_arg, "CANCELED_TIMERS_THR_PRGR"); break;
             case ERTS_SSI_AUX_WORK_ASYNC_READY:
                 erts_print(to, to_arg, "ASYNC_READY"); break;
             case ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN:
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 743711cc3b..b1c30e7652 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -52,7 +52,7 @@ typedef struct process Process;
 #include "erl_node_container_utils.h"
 #include "erl_node_tables.h"
 #include "erl_monitors.h"
-#include "erl_bif_timer.h"
+#include "erl_hl_timer.h"
 #include "erl_time.h"
 #include "erl_atom_table.h"
 #include "external.h"
@@ -278,16 +278,18 @@ typedef enum {
 #define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC	(((erts_aint32_t) 1) << 3)
 #define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM	(((erts_aint32_t) 1) << 4)
 #define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP	(((erts_aint32_t) 1) << 5)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 6)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 7)
-#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 8)
-#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 9)
-#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 10)
-#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 11)
-#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 12)
-#define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 13)
-
-#define ERTS_SSI_AUX_WORK_MAX                                           14
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS		(((erts_aint32_t) 1) << 6)
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR	(((erts_aint32_t) 1) << 7)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 8)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 9)
+#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 10)
+#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 11)
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 12)
+#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 13)
+#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 14)
+#define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 15)
+
+#define ERTS_SSI_AUX_WORK_MAX                                           16
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
@@ -463,19 +465,21 @@ typedef union {
 
 extern ErtsAlignedRunQueue *erts_aligned_run_queues;
 
-#define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS)	\
+#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
 do {								\
     (RQ)->procs.reductions += (AREDS);				\
     (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
     (RQ)->check_balance_reds -= (REDS);				\
     (RQ)->wakeup_other_reds += (AREDS);				\
+    (SD)->check_time_reds += (AREDS);				\
 } while (0)
 
-#define ERTS_PORT_REDUCTIONS_EXECUTED(RQ, REDS)			\
+#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
 do {								\
     (RQ)->ports.info.reds += (REDS);				\
     (RQ)->check_balance_reds -= (REDS);				\
     (RQ)->wakeup_other_reds += (REDS);				\
+    (SD)->check_time_reds += (REDS);				\
 } while (0)
 
 typedef struct {
@@ -514,6 +518,9 @@ typedef struct {
 	void (*completed_callback)(void *);
 	void (*completed_arg)(void *);
     } dd;
+    struct {
+	ErtsThrPrgrVal thr_prgr;
+    } cncld_tmrs;
     struct {
 	ErtsThrPrgrVal thr_prgr;
 	UWord size;
@@ -566,6 +573,7 @@ struct ErtsSchedulerData_ {
 
     ErtsTimerWheel *timer_wheel;
     ErtsNextTimeoutRef next_tmo_ref;
+    ErtsHLTimerService *timer_service;
 #ifdef ERTS_SMP
     ethr_tid tid;		/* Thread id */
     struct erl_bits_state erl_bits_state; /* erl_bits.c state */
@@ -592,6 +600,9 @@ struct ErtsSchedulerData_ {
     ErtsAuxWorkData aux_work_data;
     ErtsAtomCacheMap atom_cache_map;
 
+    ErtsMonotonicTime last_monotonic_time;
+    int check_time_reds;
+
     Uint32 thr_id;
     Uint64 unique;
     Uint64 ref;
@@ -913,10 +924,8 @@ struct process {
 
     ErlMessageQueue msg;	/* Message queue */
 
-    union {
-	ErtsBifTimer *bif_timers;	/* Bif timers aiming at this process */
-	void *terminate;
-    } u;
+    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
+    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
 
     ProcDict  *dictionary;       /* Process dictionary, may be NULL */
 
@@ -927,9 +936,12 @@ struct process {
 #ifdef USE_VM_PROBES
     Eterm dt_utag;              /* Place to store the dynamc trace user tag */
     Uint dt_utag_flags;         /* flag field for the dt_utag */
-#endif       
-    BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
+#endif
+    union {
+	void *terminate;
+	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
 				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
+    } u;
     BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
 				 * module(0), function(1), arity(2)
 				 * (module and functions are tagged atoms;
@@ -975,7 +987,6 @@ struct process {
     ErtsSchedulerData *scheduler_data;
     Eterm suspendee;
     ErtsPendingSuspend *pending_suspenders;
-    ErtsRunQueue *preferred_run_queue;
     erts_smp_atomic_t run_queue;
 #ifdef HIPE
     struct hipe_process_state_smp hipe_smp;
@@ -1085,15 +1096,14 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
-#define ERTS_PSFLG_OFF_HEAP_MSGS	ERTS_PSFLG_BIT(18)
 #ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(19)
-#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(20)
-#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(21)
-#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(22)
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 23)
+#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
+#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
+#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
 #else
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 19)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
 #endif
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
@@ -1112,7 +1122,6 @@ void erts_check_for_holes(Process* p);
  * Static flags that do not change after process creation.
  */
 #define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)
-#define ERTS_STC_FLG_PREFER_SCHED	(((Uint32) 1) << 1)
 
 /* The sequential tracing token is a tuple of size 5:
  *
@@ -1141,9 +1150,7 @@ void erts_check_for_holes(Process* p);
 #define SPO_LINK 1
 #define SPO_USE_ARGS 2
 #define SPO_MONITOR 4
-#define SPO_OFF_HEAP_MSGS 8
-#define SPO_SYSTEM_PROC 16
-#define SPO_PREFER_SCHED 32
+#define SPO_SYSTEM_PROC 8
 
 /*
  * The following struct contains options for a process to be spawned.
@@ -1231,7 +1238,6 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC */
-#define F_OFF_HEAP_MSGS      (1 << 12)
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
@@ -1267,8 +1273,6 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #  define F_INITIAL_TRACE_FLAGS 0
 #endif
 
-
-
 #define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
 		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
 		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
@@ -1302,12 +1306,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
 #define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)
 
-#define CANCEL_TIMER(p) \
-    do { \
-	if ((p)->flags & (F_INSLPQUEUE)) \
-	    cancel_timer(p); \
-	else \
-	    (p)->flags &= ~F_TIMO; \
+#define CANCEL_TIMER(P)					\
+    do {						\
+	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
+	    if ((P)->flags & F_INSLPQUEUE)		\
+		erts_cancel_proc_timer((P));		\
+	    else					\
+		(P)->flags &= ~F_TIMO;			\
+	}						\
     } while (0)
 
 #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
@@ -1594,6 +1600,9 @@ Eterm erts_multi_scheduling_blockers(Process *);
 void erts_start_schedulers(void);
 void erts_alloc_notify_delayed_dealloc(int);
 void erts_alloc_ensure_handle_delayed_dealloc_call(int);
+#ifdef ERTS_SMP
+void erts_notify_canceled_timer(ErtsSchedulerData *, int);
+#endif
 void erts_smp_notify_check_children_needed(void);
 #endif
 #if ERTS_USE_ASYNC_READY_Q
@@ -1628,8 +1637,6 @@ void erts_schedule_misc_op(void (*)(void *), void *);
 Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
 void erts_do_exit_process(Process*, Eterm);
 void erts_continue_exit_process(Process *);
-void set_timer(Process*, Uint);
-void cancel_timer(Process*);
 /* Begin System profile */
 Uint erts_runnable_process_count(void);
 /* End System profile */
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 82cc68222d..fff267ff2a 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -103,6 +103,7 @@ static struct {
     Sint16 proc_lock_main;
     Sint16 proc_lock_link;
     Sint16 proc_lock_msgq;
+    Sint16 proc_lock_btm;
     Sint16 proc_lock_status;
 } lc_id;
 #endif
@@ -145,6 +146,7 @@ erts_init_proc_lock(int cpus)
     lc_id.proc_lock_main	= erts_lc_get_lock_order_id("proc_main");
     lc_id.proc_lock_link	= erts_lc_get_lock_order_id("proc_link");
     lc_id.proc_lock_msgq	= erts_lc_get_lock_order_id("proc_msgq");
+    lc_id.proc_lock_btm		= erts_lc_get_lock_order_id("proc_btm");
     lc_id.proc_lock_status	= erts_lc_get_lock_order_id("proc_status");
 #endif
 }
@@ -707,7 +709,7 @@ proc_safelock(int is_managed,
 	    need_locks1 |= unlock_locks;
 	    if (!is_managed && !have_locks1) {
 		refc1 = 1;
-		erts_smp_proc_inc_refc(p1);
+		erts_proc_inc_refc(p1);
 	    }
 	    erts_smp_proc_unlock(p1, unlock_locks);
 	}
@@ -717,7 +719,7 @@ proc_safelock(int is_managed,
 	    need_locks2 |= unlock_locks;
 	    if (!is_managed && !have_locks2) {
 		refc2 = 1;
-		erts_smp_proc_inc_refc(p2);
+		erts_proc_inc_refc(p2);
 	    }
 	    erts_smp_proc_unlock(p2, unlock_locks);
 	}
@@ -798,9 +800,9 @@ proc_safelock(int is_managed,
 
     if (!is_managed) {
 	if (refc1)
-	    erts_smp_proc_dec_refc(p1);
+	    erts_proc_dec_refc(p1);
 	if (refc2)
-	    erts_smp_proc_dec_refc(p2);
+	    erts_proc_dec_refc(p2);
     }
 }
 
@@ -861,8 +863,8 @@ erts_pid2proc_opt(Process *c_p,
 	    return NULL;
 	need_locks &= ~c_p_have_locks;
 	if (!need_locks) {
-	    if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
-		erts_smp_proc_inc_refc(c_p);
+	    if (flags & ERTS_P2P_FLG_INC_REFC)
+		erts_proc_inc_refc(c_p);
 	    return c_p;
 	}
     }
@@ -875,8 +877,8 @@ erts_pid2proc_opt(Process *c_p,
 	if (proc->common.id != pid)
 	    proc = NULL;
 	else if (!need_locks) {
-	    if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
-		erts_smp_proc_inc_refc(proc);
+	    if (flags & ERTS_P2P_FLG_INC_REFC)
+		erts_proc_inc_refc(proc);
 	}
 	else {
 	    int busy;
@@ -916,8 +918,8 @@ erts_pid2proc_opt(Process *c_p,
 #endif
 
 	    if (!busy) {
-		if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
-		    erts_smp_proc_inc_refc(proc);
+		if (flags & ERTS_P2P_FLG_INC_REFC)
+		    erts_proc_inc_refc(proc);
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
 	    	/* all is great */
@@ -932,8 +934,8 @@ erts_pid2proc_opt(Process *c_p,
 		    proc = ERTS_PROC_LOCK_BUSY;
 		else {
 		    int managed;
-		    if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
-			erts_smp_proc_inc_refc(proc);
+		    if (flags & ERTS_P2P_FLG_INC_REFC)
+			erts_proc_inc_refc(proc);
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
 		    erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
@@ -941,7 +943,7 @@ erts_pid2proc_opt(Process *c_p,
 
 		    managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED;
 		    if (!managed) {
-			erts_smp_proc_inc_refc(proc);
+			erts_proc_inc_refc(proc);
 			erts_thr_progress_unmanaged_continue(dhndl);
 			dec_refc_proc = proc;
 
@@ -978,14 +980,14 @@ erts_pid2proc_opt(Process *c_p,
 
 	erts_smp_proc_unlock(proc, need_locks);
 
-	if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
+	if (flags & ERTS_P2P_FLG_INC_REFC)
 	    dec_refc_proc = proc;
 	proc = NULL;
 
     }
 
     if (dec_refc_proc)
-	erts_smp_proc_dec_refc(dec_refc_proc);
+	erts_proc_dec_refc(dec_refc_proc);
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG)
     ERTS_LC_ASSERT(!proc
@@ -1037,6 +1039,11 @@ erts_proc_lock_init(Process *p)
     ethr_mutex_lock(&p->lock.msgq.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.msgq.lc);
+#endif
+    erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count);
+    ethr_mutex_lock(&p->lock.btm.mtx);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_trylock(1, &p->lock.btm.lc);
 #endif
     erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id,
 		    do_lock_count);
@@ -1045,7 +1052,6 @@ erts_proc_lock_init(Process *p)
     erts_lc_trylock(1, &p->lock.status.lc);
 #endif
 #endif
-    erts_atomic32_init_nob(&p->lock.refc, 1);
 #ifdef ERTS_PROC_LOCK_DEBUG
     for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
 	erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1);
@@ -1064,6 +1070,7 @@ erts_proc_lock_fin(Process *p)
     erts_mtx_destroy(&p->lock.main);
     erts_mtx_destroy(&p->lock.link);
     erts_mtx_destroy(&p->lock.msgq);
+    erts_mtx_destroy(&p->lock.btm);
     erts_mtx_destroy(&p->lock.status);
 #endif
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
@@ -1079,17 +1086,20 @@ void erts_lcnt_proc_lock_init(Process *p) {
     if (p->common.id != ERTS_INVALID_PID) {
 	erts_lcnt_init_lock_x(&(p->lock.lcnt_main),   "proc_main",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
 	erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq),   "proc_msgq",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
+	erts_lcnt_init_lock_x(&(p->lock.lcnt_btm),    "proc_btm",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
 	erts_lcnt_init_lock_x(&(p->lock.lcnt_link),   "proc_link",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
 	erts_lcnt_init_lock_x(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK, p->common.id);
     } else {
 	erts_lcnt_init_lock(&(p->lock.lcnt_main),   "proc_main",   ERTS_LCNT_LT_PROCLOCK);
 	erts_lcnt_init_lock(&(p->lock.lcnt_msgq),   "proc_msgq",   ERTS_LCNT_LT_PROCLOCK);
+	erts_lcnt_init_lock(&(p->lock.lcnt_btm),    "proc_btm",   ERTS_LCNT_LT_PROCLOCK);
 	erts_lcnt_init_lock(&(p->lock.lcnt_link),   "proc_link",   ERTS_LCNT_LT_PROCLOCK);
 	erts_lcnt_init_lock(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK);
     }
     } else {
 	sys_memzero(&(p->lock.lcnt_main), sizeof(p->lock.lcnt_main));
 	sys_memzero(&(p->lock.lcnt_msgq), sizeof(p->lock.lcnt_msgq));
+	sys_memzero(&(p->lock.lcnt_btm), sizeof(p->lock.lcnt_btm));
 	sys_memzero(&(p->lock.lcnt_link), sizeof(p->lock.lcnt_link));
 	sys_memzero(&(p->lock.lcnt_status), sizeof(p->lock.lcnt_status));
     }
@@ -1099,6 +1109,7 @@ void erts_lcnt_proc_lock_init(Process *p) {
 void erts_lcnt_proc_lock_destroy(Process *p) {
     erts_lcnt_destroy_lock(&(p->lock.lcnt_main));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq));
+    erts_lcnt_destroy_lock(&(p->lock.lcnt_btm));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_link));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_status));
 }
@@ -1111,6 +1122,9 @@ void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (locks & ERTS_PROC_LOCK_MSGQ) {
         erts_lcnt_lock(&(lock->lcnt_msgq));
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+        erts_lcnt_lock(&(lock->lcnt_btm));
+    }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	erts_lcnt_lock(&(lock->lcnt_link));
     }
@@ -1128,6 +1142,9 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, cha
     if (locks & ERTS_PROC_LOCK_MSGQ) {
         erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+        erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line);
+    }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line);
     }
@@ -1145,6 +1162,9 @@ void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (locks & ERTS_PROC_LOCK_MSGQ) {
         erts_lcnt_lock_unaquire(&(lock->lcnt_msgq));
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+        erts_lcnt_lock_unaquire(&(lock->lcnt_btm));
+    }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	erts_lcnt_lock_unaquire(&(lock->lcnt_link));
     }
@@ -1162,6 +1182,9 @@ void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (locks & ERTS_PROC_LOCK_MSGQ) {
         erts_lcnt_unlock(&(lock->lcnt_msgq));
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+        erts_lcnt_unlock(&(lock->lcnt_btm));
+    }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	erts_lcnt_unlock(&(lock->lcnt_link));
     }
@@ -1178,6 +1201,9 @@ void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res
     if (locks & ERTS_PROC_LOCK_MSGQ) {
         erts_lcnt_trylock(&(lock->lcnt_msgq), res);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+        erts_lcnt_trylock(&(lock->lcnt_btm), res);
+    }
     if (locks & ERTS_PROC_LOCK_LINK) {
 	erts_lcnt_trylock(&(lock->lcnt_link), res);
     }
@@ -1235,6 +1261,10 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_lock_x(&lck,file,line);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	lck.id = lc_id.proc_lock_btm;
+	erts_lc_lock_x(&lck,file,line);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_lock_x(&lck,file,line);
@@ -1260,6 +1290,10 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_trylock_x(locked, &lck, file, line);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	lck.id = lc_id.proc_lock_btm;
+	erts_lc_trylock_x(locked, &lck, file, line);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_trylock_x(locked, &lck, file, line);
@@ -1276,6 +1310,10 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks)
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_unlock(&lck);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	lck.id = lc_id.proc_lock_btm;
+	erts_lc_unlock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_unlock(&lck);
@@ -1303,6 +1341,10 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_might_unlock(&lck);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	lck.id = lc_id.proc_lock_btm;
+	erts_lc_might_unlock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_might_unlock(&lck);
@@ -1322,6 +1364,8 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 	erts_lc_might_unlock(&p->lock.link.lc);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_lc_might_unlock(&p->lock.msgq.lc);
+    if (locks & ERTS_PROC_LOCK_BTM)
+	erts_lc_might_unlock(&p->lock.btm.lc);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_might_unlock(&p->lock.status.lc);
 #endif
@@ -1347,6 +1391,10 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_require_lock(&lck, file, line);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	lck.id = lc_id.proc_lock_btm;
+	erts_lc_require_lock(&lck, file, line);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_require_lock(&lck, file, line);
@@ -1358,6 +1406,8 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 	erts_lc_require_lock(&p->lock.link.lc, file, line);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_lc_require_lock(&p->lock.msgq.lc, file, line);
+    if (locks & ERTS_PROC_LOCK_BTM)
+	erts_lc_require_lock(&p->lock.btm.lc, file, line);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_require_lock(&p->lock.status.lc, file, line);
 #endif
@@ -1374,6 +1424,10 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_unrequire_lock(&lck);
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	lck.id = lc_id.proc_lock_btm;
+	erts_lc_unrequire_lock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_unrequire_lock(&lck);
@@ -1393,6 +1447,8 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
 	erts_lc_unrequire_lock(&p->lock.link.lc);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_lc_unrequire_lock(&p->lock.msgq.lc);
+    if (locks & ERTS_PROC_LOCK_BTM)
+	erts_lc_unrequire_lock(&p->lock.btm.lc);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_unrequire_lock(&p->lock.status.lc);
 #endif
@@ -1414,6 +1470,8 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
 	    lck.id = lc_id.proc_lock_link;
 	else if (locks & ERTS_PROC_LOCK_MSGQ)
 	    lck.id = lc_id.proc_lock_msgq;
+	else if (locks & ERTS_PROC_LOCK_BTM)
+	    lck.id = lc_id.proc_lock_btm;
 	else if (locks & ERTS_PROC_LOCK_STATUS)
 	    lck.id = lc_id.proc_lock_status;
 	else
@@ -1448,7 +1506,8 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 {
     int have_locks_len = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
@@ -1464,18 +1523,24 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
 	have_locks[have_locks_len++].extra = p->common.id;
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_btm;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	have_locks[have_locks_len].id = lc_id.proc_lock_status;
 	have_locks[have_locks_len++].extra = p->common.id;
     }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t have_locks[4];
+    erts_lc_lock_t have_locks[5];
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
     if (locks & ERTS_PROC_LOCK_LINK)
 	have_locks[have_locks_len++] = p->lock.link.lc;
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	have_locks[have_locks_len++] = p->lock.msgq.lc;
+    if (locks & ERTS_PROC_LOCK_BTM)
+	have_locks[have_locks_len++] = p->lock.btm.lc;
     if (locks & ERTS_PROC_LOCK_STATUS)
 	have_locks[have_locks_len++] = p->lock.status.lc;
 #endif
@@ -1488,11 +1553,11 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
     int have_locks_len = 0;
     int have_not_locks_len = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
-    erts_lc_lock_t have_not_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_not_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT};
@@ -1521,6 +1586,14 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_msgq;
 	have_not_locks[have_not_locks_len++].extra = p->common.id;
     }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_btm;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    else {
+	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_btm;
+	have_not_locks[have_not_locks_len++].extra = p->common.id;
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	have_locks[have_locks_len].id = lc_id.proc_lock_status;
 	have_locks[have_locks_len++].extra = p->common.id;
@@ -1530,8 +1603,8 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_not_locks[have_not_locks_len++].extra = p->common.id;
     }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t have_locks[4];
-    erts_lc_lock_t have_not_locks[4];
+    erts_lc_lock_t have_locks[5];
+    erts_lc_lock_t have_not_locks[5];
 
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
@@ -1545,6 +1618,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len++] = p->lock.msgq.lc;
     else
 	have_not_locks[have_not_locks_len++] = p->lock.msgq.lc;
+    if (locks & ERTS_PROC_LOCK_BTM)
+	have_locks[have_locks_len++] = p->lock.btm.lc;
+    else
+	have_not_locks[have_not_locks_len++] = p->lock.btm.lc;
     if (locks & ERTS_PROC_LOCK_STATUS)
 	have_locks[have_locks_len++] = p->lock.status.lc;
     else
@@ -1558,10 +1635,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 ErtsProcLocks
 erts_proc_lc_my_proc_locks(Process *p)
 {
-    int resv[4];
+    int resv[5];
     ErtsProcLocks res = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t locks[4] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
+    erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
 						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_link,
@@ -1570,17 +1647,21 @@ erts_proc_lc_my_proc_locks(Process *p)
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq,
 						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK),
+			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_btm,
+						 p->common.id,
+						 ERTS_LC_FLG_LT_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_status,
 						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK)};
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t locks[4] = {p->lock.main.lc,
+    erts_lc_lock_t locks[5] = {p->lock.main.lc,
 			       p->lock.link.lc,
 			       p->lock.msgq.lc,
+			       p->lock.btm.lc,
 			       p->lock.status.lc};
 #endif
 
-    erts_lc_have_locks(resv, locks, 4);
+    erts_lc_have_locks(resv, locks, 5);
     if (resv[0])
 	res |= ERTS_PROC_LOCK_MAIN;
     if (resv[1])
@@ -1588,6 +1669,8 @@ erts_proc_lc_my_proc_locks(Process *p)
     if (resv[2])
 	res |= ERTS_PROC_LOCK_MSGQ;
     if (resv[3])
+	res |= ERTS_PROC_LOCK_BTM;
+    if (resv[4])
 	res |= ERTS_PROC_LOCK_STATUS;
 
     return res;
@@ -1596,13 +1679,14 @@ erts_proc_lc_my_proc_locks(Process *p)
 void
 erts_proc_lc_chk_no_proc_locks(char *file, int line)
 {
-    int resv[4];
-    int ids[4] = {lc_id.proc_lock_main,
+    int resv[5];
+    int ids[5] = {lc_id.proc_lock_main,
 		  lc_id.proc_lock_link,
 		  lc_id.proc_lock_msgq,
+		  lc_id.proc_lock_btm,
 		  lc_id.proc_lock_status};
-    erts_lc_have_lock_ids(resv, ids, 4);
-    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3])) {
+    erts_lc_have_lock_ids(resv, ids, 5);
+    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) {
 	erts_lc_fail("%s:%d: Thread has process locks locked when expected "
 		     "not to have any process locks locked",
 		     file, line);
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 052d992d3f..8957e7773b 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -65,7 +65,7 @@
 
 #endif
 
-#define ERTS_PROC_LOCK_MAX_BIT 3
+#define ERTS_PROC_LOCK_MAX_BIT 4
 
 typedef erts_aint32_t ErtsProcLocks;
 
@@ -81,17 +81,18 @@ typedef struct erts_proc_lock_t_ {
     erts_lcnt_lock_t lcnt_main;
     erts_lcnt_lock_t lcnt_link;
     erts_lcnt_lock_t lcnt_msgq;
+    erts_lcnt_lock_t lcnt_btm;
     erts_lcnt_lock_t lcnt_status;
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     erts_mtx_t main;
     erts_mtx_t link;
     erts_mtx_t msgq;
+    erts_mtx_t btm;
     erts_mtx_t status;
 #else
 #  error "no implementation"
 #endif
-    erts_atomic32_t refc;
 #ifdef ERTS_PROC_LOCK_DEBUG
     erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
 #endif
@@ -120,10 +121,16 @@ typedef struct erts_proc_lock_t_ {
  * Message queue lock:
  *   Protects the following fields in the process structure:
  *   * msg_inq
- *   * bif_timers
  */
 #define ERTS_PROC_LOCK_MSGQ		(((ErtsProcLocks) 1) << 2)
 
+/*
+ * Bif timer lock:
+ *   Protects the following fields in the process structure:
+ *   * bif_timers
+ */
+#define ERTS_PROC_LOCK_BTM		(((ErtsProcLocks) 1) << 3)
+
 /*
  * Status lock:
  *   Protects the following fields in the process structure:
@@ -463,6 +470,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	if (erts_mtx_trylock(&p->lock.msgq) == EBUSY)
 	    goto busy_msgq;
+    if (locks & ERTS_PROC_LOCK_BTM)
+	if (erts_mtx_trylock(&p->lock.btm) == EBUSY)
+	    goto busy_btm;
     if (locks & ERTS_PROC_LOCK_STATUS)
 	if (erts_mtx_trylock(&p->lock.status) == EBUSY)
 	    goto busy_status;
@@ -470,6 +480,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
     return 0;
 
 busy_status:
+    if (locks & ERTS_PROC_LOCK_BTM)
+	erts_mtx_unlock(&p->lock.btm);
+busy_btm:
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_mtx_unlock(&p->lock.msgq);
 busy_msgq:
@@ -549,6 +562,8 @@ erts_smp_proc_lock__(Process *p,
 	erts_mtx_lock(&p->lock.link);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_mtx_lock(&p->lock.msgq);
+    if (locks & ERTS_PROC_LOCK_BTM)
+	erts_mtx_lock(&p->lock.btm);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_mtx_lock(&p->lock.status);
 
@@ -638,6 +653,8 @@ erts_smp_proc_unlock__(Process *p,
 
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_mtx_unlock(&p->lock.status);
+    if (locks & ERTS_PROC_LOCK_BTM)
+	erts_mtx_unlock(&p->lock.btm);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_mtx_unlock(&p->lock.msgq);
     if (locks & ERTS_PROC_LOCK_LINK)
@@ -752,9 +769,10 @@ ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks);
 ERTS_GLB_INLINE void erts_smp_proc_unlock(Process *, ErtsProcLocks);
 ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks);
 
-ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *);
-ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *);
-ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *, Sint32);
+ERTS_GLB_INLINE void erts_proc_inc_refc(Process *);
+ERTS_GLB_INLINE void erts_proc_dec_refc(Process *);
+ERTS_GLB_INLINE void erts_proc_add_refc(Process *, Sint);
+ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -814,28 +832,59 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
 #endif
 }
 
-ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p)
+ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p)
 {
+    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
 #ifdef ERTS_SMP
+    erts_ptab_atmc_inc_refc(&p->common);
+#else
     erts_ptab_inc_refc(&p->common);
 #endif
 }
 
-ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p)
+ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
 {
+    Sint referred;
+    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
 #ifdef ERTS_SMP
-    int referred = erts_ptab_dec_test_refc(&p->common);
-    if (!referred)
-	erts_free_proc(p);
+    referred = erts_ptab_atmc_dec_test_refc(&p->common);
+#else
+    referred = erts_ptab_dec_test_refc(&p->common);
 #endif
+    if (!referred) {
+	ASSERT(ERTS_PROC_IS_EXITING(p));
+	ASSERT(ERTS_AINT_NULL
+	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
+					    internal_pid_index(p->common.id)));
+	erts_free_proc(p);
+    }
 }
 
-ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc)
+ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 {
+    Sint referred;
+    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
 #ifdef ERTS_SMP
-    int referred = erts_ptab_add_test_refc(&p->common, add_refc);
-    if (!referred)
+    referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc);
+#else
+    referred = erts_ptab_add_test_refc(&p->common, add_refc);
+#endif
+    if (!referred) {
+	ASSERT(ERTS_PROC_IS_EXITING(p));
+	ASSERT(ERTS_AINT_NULL
+	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
+					    internal_pid_index(p->common.id)));
 	erts_free_proc(p);
+    }
+}
+
+ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p)
+{
+    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
+#ifdef ERTS_SMP
+    return erts_ptab_atmc_read_refc(&p->common);
+#else
+    return erts_ptab_read_refc(&p->common);
 #endif
 }
 
@@ -868,7 +917,7 @@ void erts_proc_safelock(Process *a_proc,
 
 #define ERTS_P2P_FLG_ALLOW_OTHER_X	(1 <<  0)
 #define ERTS_P2P_FLG_TRY_LOCK		(1 <<  1)
-#define ERTS_P2P_FLG_SMP_INC_REFC	(1 <<  2)
+#define ERTS_P2P_FLG_INC_REFC		(1 <<  2)
 
 #define ERTS_PROC_LOCK_BUSY ((Process *) &erts_invalid_process)
 
@@ -928,11 +977,14 @@ erts_pid2proc_opt(Process *c_p_unused,
 		  int flags)
 {
     Process *proc = erts_proc_lookup_raw(pid);
-    return ((!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
-	     && proc
-	     && ERTS_PROC_IS_EXITING(proc))
-	    ? NULL
-	    : proc);
+    if (!proc)
+	return NULL;
+    if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
+	&& ERTS_PROC_IS_EXITING(proc))
+	return NULL;
+    if (flags & ERTS_P2P_FLG_INC_REFC)
+	erts_proc_inc_refc(proc);
+    return proc;
 }
 #endif /* !ERTS_SMP */
 
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index 02943ee683..c688db98d8 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -360,7 +360,8 @@ erts_ptab_init_table(ErtsPTab *ptab,
 		     int size,
 		     UWord element_size,
 		     char *name,
-		     int legacy)
+		     int legacy,
+		     int atomic_refc)
 {
     size_t tab_sz, alloc_sz;
     Uint32 bits, cl, cli, ix, ix_per_cache_line, tab_cache_lines; 
@@ -415,6 +416,8 @@ erts_ptab_init_table(ErtsPTab *ptab,
     ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id);
     ptab->r.o.release_element = release_element;
 
+    ptab->r.o.atomic_refc = atomic_refc;
+
     if (legacy) {
 	ptab->r.o.free_id_data = NULL;
 	ptab->r.o.dix_cl_mask = 0;
@@ -533,9 +536,10 @@ erts_ptab_new_element(ErtsPTab *ptab,
 
 	init_ptab_el(init_arg, (Eterm) data);
 
-#ifdef ERTS_SMP
-	erts_smp_atomic32_init_nob(&ptab_el->refc, 1);
-#endif
+	if (ptab->r.o.atomic_refc)
+	    erts_atomic_init_nob(&ptab_el->refc.atmc, 1);
+	else
+	    ptab_el->refc.sint = 1;
 
 	pix = erts_ptab_data2pix(ptab, (Eterm) data);
 
@@ -608,9 +612,10 @@ erts_ptab_new_element(ErtsPTab *ptab,
 
 	init_ptab_el(init_arg, data);
 
-#ifdef ERTS_SMP
-	erts_smp_atomic32_init_nob(&ptab_el->refc, 1);
-#endif
+	if (ptab->r.o.atomic_refc)
+	    erts_atomic_init_nob(&ptab_el->refc.atmc, 1);
+	else
+	    ptab_el->refc.sint = 1;
 
 	/* Move into slot reserved */
 #ifdef DEBUG
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index 876241159b..102d41e07f 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -51,11 +51,13 @@
 
 typedef struct {
     Eterm id;
-#ifdef ERTS_SMP
-    erts_atomic32_t refc;
-#endif
+    union {
+	erts_atomic_t atmc;
+	Sint sint;
+    } refc;
     Eterm tracer_proc;
     Uint trace_flags;
+    erts_smp_atomic_t timer;
     union {
 	/* --- While being alive --- */
 	struct {
@@ -63,11 +65,6 @@ typedef struct {
 	    struct reg_proc *reg;
 	    ErtsLink *links;
 	    ErtsMonitor *monitors;
-#ifdef ERTS_SMP
-	    ErtsSmpPTimer *ptimer;
-#else
-	    ErlTimer tm;
-#endif
 	} alive;
 
 	/* --- While being released --- */
@@ -111,6 +108,7 @@ typedef struct {
     Eterm invalid_data;
     void (*release_element)(void *);
     UWord element_size;
+    int atomic_refc;
 } ErtsPTabReadOnlyData;
 
 typedef struct {
@@ -181,7 +179,8 @@ void erts_ptab_init_table(ErtsPTab *ptab,
 			  int size,
 			  UWord element_size,
 			  char *name,
-			  int legacy);
+			  int legacy,
+			  int atomic_refc);
 int erts_ptab_new_element(ErtsPTab *ptab,
 			  ErtsPTabElementCommon *ptab_el,
 			  void *init_arg,
@@ -206,9 +205,15 @@ ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix);
 ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix);
 ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix);
 ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el);
-ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el);
-ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
-					    Sint32 add_refc);
+ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el);
+ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
+					     Sint add_refc);
+ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el);
+ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el);
+ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el);
+ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el,
+						  Sint add_refc);
+ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el);
 ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab);
 ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab);
 ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab);
@@ -365,50 +370,65 @@ ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix)
     return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]);
 }
 
-ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el)
+ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el)
 {
-#ifdef ERTS_SMP
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_aint32_t refc = erts_atomic32_inc_read_nob(&ptab_el->refc);
-    ERTS_SMP_LC_ASSERT(refc > 1);
+    erts_aint_t refc = erts_atomic_inc_read_nob(&ptab_el->refc.atmc);
+    ERTS_LC_ASSERT(refc > 1);
 #else
-    erts_atomic32_inc_nob(&ptab_el->refc);
-#endif
+    erts_atomic_inc_nob(&ptab_el->refc.atmc);
 #endif
 }
 
-ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el)
+ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el)
 {
-#ifdef ERTS_SMP
-    erts_aint32_t refc = erts_atomic32_dec_read_nob(&ptab_el->refc);
+    erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc);
     ERTS_SMP_LC_ASSERT(refc >= 0);
-    return (int) refc;
-#else
-    return 0;
+#ifdef ERTS_SMP
+    if (refc == 0)
+	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
 #endif
+    return (Sint) refc;
 }
 
-ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
-					    Sint32 add_refc)
+ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el,
+						  Sint add_refc)
 {
-#ifdef ERTS_SMP
-    erts_aint32_t refc;
+    erts_aint_t refc = erts_atomic_add_read_mb(&ptab_el->refc.atmc,
+					       (erts_aint_t) add_refc);
+    ERTS_SMP_LC_ASSERT(refc >= 0);
+    return (Sint) refc;
+}
 
-#ifndef ERTS_ENABLE_LOCK_CHECK
-    if (add_refc >= 0) {
-	erts_atomic32_add_nob(&ptab_el->refc,
-			      (erts_aint32_t) add_refc);
-	return 1;
-    }
-#endif
+ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el)
+{
+    return (Sint) erts_atomic_read_nob(&ptab_el->refc.atmc);
+}
+
+ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el)
+{
+    ptab_el->refc.sint++;
+    ASSERT(ptab_el->refc.sint > 1);
+}
 
-    refc = erts_atomic32_add_read_nob(&ptab_el->refc,
-				      (erts_aint32_t) add_refc);
+ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el)
+{
+    Sint refc = --ptab_el->refc.sint;
     ERTS_SMP_LC_ASSERT(refc >= 0);
-    return (int) refc;
-#else
-    return 0;
-#endif
+    return refc;
+}
+
+ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
+					     Sint add_refc)
+{
+    ptab_el->refc.sint += add_refc;
+    ERTS_SMP_LC_ASSERT(ptab_el->refc.sint >= 0);
+    return (Sint) ptab_el->refc.sint;
+}
+
+ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el)
+{
+    return ptab_el->refc.sint;
 }
 
 ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab)
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 4c9b00d2ee..78e0964e8b 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1360,6 +1360,7 @@ erts_thr_progress_fatal_error_wait(SWord timeout) {
     erts_aint32_t bc;
     SWord time_left = timeout;
     ErtsMonotonicTime timeout_time;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     /*
      * Counting poll intervals may give us a too long timeout
@@ -1367,7 +1368,7 @@ erts_thr_progress_fatal_error_wait(SWord timeout) {
      * this. In case we havn't got time correction this may
      * however fail too...
      */
-    timeout_time = erts_get_monotonic_time();
+    timeout_time = erts_get_monotonic_time(esdp);
     timeout_time += ERTS_MSEC_TO_MONOTONIC((ErtsMonotonicTime) timeout);
 
     while (1) {
@@ -1378,7 +1379,7 @@ erts_thr_progress_fatal_error_wait(SWord timeout) {
 	    break; /* Succefully blocked all managed threads */
 	if (time_left <= 0)
 	    break; /* Timeout */
-	if (timeout_time <= erts_get_monotonic_time())
+	if (timeout_time <= erts_get_monotonic_time(esdp))
 	    break; /* Timeout */
     }
 }
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 5af3c21d40..4560cd23af 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -20,6 +20,13 @@
 #ifndef ERL_TIME_H__
 #define ERL_TIME_H__
 
+/* timer wheel size NEED to be a power of 2 */
+#ifdef SMALL_MEMORY
+#define ERTS_TIW_SIZE (1 << 13)
+#else
+#define ERTS_TIW_SIZE (1 << 16)
+#endif
+
 #if defined(DEBUG) || 0
 #define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B)
 #else
@@ -33,52 +40,10 @@ typedef enum {
 } ErtsTimeWarpMode;
 
 typedef struct ErtsTimerWheel_ ErtsTimerWheel;
-typedef erts_atomic64_t * ErtsNextTimeoutRef;
-extern ErtsTimerWheel *erts_default_timer_wheel;
+typedef ErtsMonotonicTime * ErtsNextTimeoutRef;
 
 extern SysTimeval erts_first_emu_time;
 
-/*
-** Timer entry:
-*/
-typedef struct erl_timer {
-    struct erl_timer* next;	/* next entry tiw slot or chain */
-    struct erl_timer* prev;	/* prev entry tiw slot or chain */
-    erts_smp_atomic_t wheel;
-    ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
-    /* called when timeout */
-    void (*timeout)(void*);
-    /* called when cancel (may be NULL) */
-    void (*cancel)(void*);
-    void* arg;        /* argument to timeout/cancel procs */
-    int slot;			/* slot in timer wheel */
-} ErlTimer;
-
-typedef void (*ErlTimeoutProc)(void*);
-typedef void (*ErlCancelProc)(void*);
-
-#ifdef ERTS_SMP
-/*
- * Process and port timer
- */
-typedef union ErtsSmpPTimer_ ErtsSmpPTimer;
-union ErtsSmpPTimer_ {
-    struct {
-	ErlTimer tm;
-	Eterm id;
-	void (*timeout_func)(void*);
-	ErtsSmpPTimer **timer_ref;
-	Uint32 flags;
-    } timer;
-    ErtsSmpPTimer *next;
-};
-
-void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
-			    Eterm id,
-			    ErlTimeoutProc timeout_func,
-			    Uint timeout);
-void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer);
-#endif
 
 void erts_monitor_time_offset(Eterm id, Eterm ref);
 int erts_demonitor_time_offset(Eterm ref);
@@ -86,14 +51,8 @@ int erts_demonitor_time_offset(Eterm ref);
 int erts_init_time_sup(int, ErtsTimeWarpMode);
 void erts_late_init_time_sup(void);
 
-/* timer-wheel api */
-
-ErtsTimerWheel *erts_create_timer_wheel(int);
 ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *);
 void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode);
-void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
-void erts_cancel_timer(ErlTimer*);
-Uint erts_time_left(ErlTimer *);
 void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime);
 Uint erts_timer_wheel_memory_size(void);
 
@@ -101,27 +60,6 @@ Uint erts_timer_wheel_memory_size(void);
 void erts_p_slpq(void);
 #endif
 
-ErtsMonotonicTime erts_check_next_timeout_time(ErtsTimerWheel *,
-					       ErtsMonotonicTime);
-
-ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p);
-ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p)
-{
-    erts_smp_atomic_init_nob(&p->wheel, (erts_aint_t) NULL);
-}
-
-ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
-{
-    return (ErtsMonotonicTime) erts_atomic64_read_acqb((erts_atomic64_t *) nxt_tmo_ref);
-}
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-
-
 /* time_sup */
 
 #if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
@@ -154,6 +92,7 @@ ErtsTimeOffsetState erts_time_offset_state(void);
 ErtsTimeOffsetState erts_finalize_time_offset(void); 
 struct process;
 Eterm erts_get_monotonic_start_time(struct process *c_p);
+Eterm erts_get_monotonic_end_time(struct process *c_p);
 Eterm erts_monotonic_time_source(struct process*c_p);
 Eterm erts_system_time_source(struct process*c_p);
 
@@ -163,8 +102,20 @@ Eterm erts_system_time_source(struct process*c_p);
 #define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution)
 #endif
 
+#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000))
+
 struct erts_time_sup_read_only__ {
     ErtsMonotonicTime monotonic_time_unit;
+#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
+    ErtsMonotonicTime start;
+    struct {
+	ErtsMonotonicTime native;
+	ErtsMonotonicTime nsec;
+	ErtsMonotonicTime usec;
+	ErtsMonotonicTime msec;
+	ErtsMonotonicTime sec;
+    } start_offset;
+#endif
 #ifndef SYS_CLOCK_RESOLUTION
     ErtsMonotonicTime clktck_resolution;
 #endif
@@ -220,6 +171,16 @@ erts_time_unit_conversion(Uint64 value,
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+/*
+ * Range of monotonic time internally
+ */
+
+#define ERTS_MONOTONIC_BEGIN						\
+    ERTS_MONOTONIC_TIME_UNIT
+#define ERTS_MONOTONIC_END						\
+    ((ERTS_MONOTONIC_TIME_MAX / ERTS_MONOTONIC_TIME_UNIT)		\
+     * ERTS_MONOTONIC_TIME_UNIT)
+
 #if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
 
 /*
@@ -231,9 +192,6 @@ erts_time_unit_conversion(Uint64 value,
 #  error Compile time time unit needs to be at least 1000000
 #endif
 
-#define ERTS_MONOTONIC_TIME_UNIT \
-    ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT)
-
 #if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000
 /* Nano-second time unit */
 
@@ -264,6 +222,66 @@ erts_time_unit_conversion(Uint64 value,
 #error Missing implementation for monotonic time unit
 #endif
 
+#define ERTS_MONOTONIC_TIME_UNIT \
+    ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT)
+
+/*
+ * NOTE! ERTS_MONOTONIC_TIME_START_EXTERNAL *need* to be a multiple
+ *       of ERTS_MONOTONIC_TIME_UNIT.
+ */
+
+#ifdef ARCH_32
+/*
+ * Want to use a big-num of arity 2 as long as possible (584 years
+ * in the nano-second time unit case).
+ */
+#define ERTS_MONOTONIC_TIME_START_EXTERNAL		\
+    (((((((ErtsMonotonicTime) 1) << 32)-1)	\
+       / ERTS_MONOTONIC_TIME_UNIT)		\
+      * ERTS_MONOTONIC_TIME_UNIT)		\
+     + ERTS_MONOTONIC_TIME_UNIT)
+
+#else /* ARCH_64 */
+
+#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000
+
+/*
+ * Using micro second time unit or lower. Start at zero since
+ * time will remain an immediate for a very long time anyway
+ * (1827 years in the 10 micro second case)...
+ */
+#define ERTS_MONOTONIC_TIME_START_EXTERNAL ((ErtsMonotonicTime) 0)
+
+#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 10*1000*1000 */
+
+/*
+ * Want to use an immediate as long as possible (36 years in the
+ * nano-second time unit case).
+*/
+#define ERTS_MONOTONIC_TIME_START_EXTERNAL 		\
+    ((((ErtsMonotonicTime) MIN_SMALL)		\
+      / ERTS_MONOTONIC_TIME_UNIT)		\
+     * ERTS_MONOTONIC_TIME_UNIT)
+
+#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */
+
+#endif /* ARCH_64 */
+
+/*
+ * Offsets from internal monotonic time to external monotonic time
+ */
+
+#define ERTS_MONOTONIC_OFFSET_NATIVE \
+    (ERTS_MONOTONIC_TIME_START_EXTERNAL - ERTS_MONOTONIC_BEGIN)
+#define ERTS_MONOTONIC_OFFSET_NSEC					\
+    ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
+#define ERTS_MONOTONIC_OFFSET_USEC					\
+    ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
+#define ERTS_MONOTONIC_OFFSET_MSEC					\
+    ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
+#define ERTS_MONOTONIC_OFFSET_SEC					\
+    ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
+
 #define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \
     ((MON) / (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION))
 #define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \
@@ -271,8 +289,23 @@ erts_time_unit_conversion(Uint64 value,
 
 #else /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
 
+/*
+ * Initialized in erts_init_sys_time_sup()
+ */
 #define ERTS_MONOTONIC_TIME_UNIT (erts_time_sup__.r.o.monotonic_time_unit)
 
+/*
+ * Offsets from internal monotonic time to external monotonic time
+ *
+ * Initialized in erts_init_time_sup()...
+ */
+#define ERTS_MONOTONIC_TIME_START_EXTERNAL (erts_time_sup__.r.o.start)
+#define ERTS_MONOTONIC_OFFSET_NATIVE (erts_time_sup__.r.o.start_offset.native)
+#define ERTS_MONOTONIC_OFFSET_NSEC (erts_time_sup__.r.o.start_offset.nsec)
+#define ERTS_MONOTONIC_OFFSET_USEC (erts_time_sup__.r.o.start_offset.usec)
+#define ERTS_MONOTONIC_OFFSET_MSEC (erts_time_sup__.r.o.start_offset.msec)
+#define ERTS_MONOTONIC_OFFSET_SEC (erts_time_sup__.r.o.start_offset.sec)
+
 #define ERTS_CONV_FROM_MON_UNIT___(M, TO)				\
     ((ErtsMonotonicTime)						\
      erts_time_unit_conversion((Uint64) (M),				\
@@ -310,6 +343,10 @@ erts_time_unit_conversion(Uint64 value,
 
 #endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
 
+#define ERTS_MONOTONIC_TIME_END_EXTERNAL				\
+    (ERTS_MONOTONIC_TIME_START_EXTERNAL					\
+     + (ERTS_MONOTONIC_END - ERTS_MONOTONIC_BEGIN))
+
 #define ERTS_MSEC_TO_CLKTCKS__(MON) \
     ((MON) * (ERTS_CLKTCK_RESOLUTION/1000))
 #define ERTS_CLKTCKS_TO_MSEC__(TCKS) \
@@ -355,3 +392,65 @@ erts_time_unit_conversion(Uint64 value,
      ERTS_CLKTCKS_TO_MSEC__((X)))
 
 #endif /* ERL_TIME_H__ */
+
+/* timer-wheel api */
+#if defined(ERTS_WANT_TIMER_WHEEL_API) && !defined(ERTS_GOT_TIMER_WHEEL_API)
+#define ERTS_GOT_TIMER_WHEEL_API
+
+#include "erl_thr_progress.h"
+#include "erl_process.h"
+
+void erts_sched_init_time_sup(ErtsSchedulerData *esdp);
+
+
+#define ERTS_TWHEEL_SLOT_AT_ONCE -1
+#define ERTS_TWHEEL_SLOT_INACTIVE -2
+
+/*
+** Timer entry:
+*/
+typedef struct erl_timer {
+    struct erl_timer* next;	/* next entry tiw slot or chain */
+    struct erl_timer* prev;	/* prev entry tiw slot or chain */
+    union {
+	struct {
+	    void (*timeout)(void*); /* called when timeout */
+	    void (*cancel)(void*);  /* called when cancel (may be NULL) */
+	    void* arg;              /* argument to timeout/cancel procs */
+	} func;
+	ErtsThrPrgrLaterOp cleanup;
+    } u;
+    ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
+    int slot;
+} ErtsTWheelTimer;
+
+typedef void (*ErlTimeoutProc)(void*);
+typedef void (*ErlCancelProc)(void*);
+
+void erts_twheel_set_timer(ErtsTimerWheel *tiw,
+			   ErtsTWheelTimer *p, ErlTimeoutProc timeout,
+			   ErlCancelProc cancel, void *arg,
+			   ErtsMonotonicTime timeout_pos);
+void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p);
+ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp);
+
+ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *);
+
+ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p);
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p)
+{
+    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+}
+
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
+{
+    return *((ErtsMonotonicTime *) nxt_tmo_ref);
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* timer wheel api */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 9d572c0b0a..1139cb9c97 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -30,6 +30,8 @@
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
  
 static erts_smp_mtx_t erts_timeofday_mtx;
 static erts_smp_mtx_t erts_get_time_mtx;
@@ -57,76 +59,6 @@ static int time_sup_initialized = 0;
 static void
 schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset);
 
-/*
- * NOTE! ERTS_MONOTONIC_TIME_START *need* to be a multiple
- *       of ERTS_MONOTONIC_TIME_UNIT.
- */
-
-#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
-
-#ifdef ARCH_32
-/*
- * Want to use a big-num of arity 2 as long as possible (584 years
- * in the nano-second time unit case).
- */
-#define ERTS_MONOTONIC_TIME_START		\
-    (((((((ErtsMonotonicTime) 1) << 32)-1)	\
-       / ERTS_MONOTONIC_TIME_UNIT)		\
-      * ERTS_MONOTONIC_TIME_UNIT)		\
-     + ERTS_MONOTONIC_TIME_UNIT)
-
-#else /* ARCH_64 */
-
-#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000
-
-/*
- * Using micro second time unit or lower. Start at zero since
- * time will remain an immediate for a very long time anyway
- * (1827 years in the 10 micro second case)...
- */
-#define ERTS_MONOTONIC_TIME_START ((ErtsMonotonicTime) 0)
-
-#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */
-
-/*
- * Want to use an immediate as long as possible (36 years in the
- * nano-second time unit case).
-*/
-#define ERTS_MONOTONIC_TIME_START 		\
-    ((((ErtsMonotonicTime) MIN_SMALL)		\
-      / ERTS_MONOTONIC_TIME_UNIT)		\
-     * ERTS_MONOTONIC_TIME_UNIT)
-
-#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */
-
-#endif /* ARCH_64 */
-
-#define ERTS_MONOTONIC_OFFSET_NATIVE \
-    (ERTS_MONOTONIC_TIME_START - ERTS_MONOTONIC_TIME_UNIT)
-#define ERTS_MONOTONIC_OFFSET_NSEC					\
-    ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
-#define ERTS_MONOTONIC_OFFSET_USEC					\
-    ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
-#define ERTS_MONOTONIC_OFFSET_MSEC					\
-    ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
-#define ERTS_MONOTONIC_OFFSET_SEC					\
-    ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
-
-#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
-
-/*
- * Initialized in erts_init_time_sup()...
- */
-
-#define ERTS_MONOTONIC_TIME_START (time_sup.r.o.start)
-#define ERTS_MONOTONIC_OFFSET_NATIVE (time_sup.r.o.start_offset.native)
-#define ERTS_MONOTONIC_OFFSET_NSEC (time_sup.r.o.start_offset.nsec)
-#define ERTS_MONOTONIC_OFFSET_USEC (time_sup.r.o.start_offset.usec)
-#define ERTS_MONOTONIC_OFFSET_MSEC (time_sup.r.o.start_offset.msec)
-#define ERTS_MONOTONIC_OFFSET_SEC (time_sup.r.o.start_offset.sec)
-
-#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
-
 struct time_sup_read_only__ {
     ErtsMonotonicTime (*get_time)(void);
     int correction;
@@ -146,16 +78,6 @@ struct time_sup_read_only__ {
     int os_system_time_locked;
     Uint64 os_system_time_resolution;
     Uint64 os_system_time_extended;
-#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
-    ErtsMonotonicTime start;
-    struct {
-	ErtsMonotonicTime native;
-	ErtsMonotonicTime nsec;
-	ErtsMonotonicTime usec;
-	ErtsMonotonicTime msec;
-	ErtsMonotonicTime sec;
-    } start_offset;
-#endif
     struct {
 	ErtsMonotonicTime large_diff;
 	ErtsMonotonicTime small_diff;
@@ -211,7 +133,7 @@ struct time_sup_infrequently_changed__ {
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
     struct {
 	erts_smp_rwmtx_t rwmtx;
-	ErlTimer timer;
+	ErtsTWheelTimer timer;
 	ErtsMonotonicCorrectionData cdata;
     } parmon;
     ErtsMonotonicTime minit;
@@ -273,6 +195,17 @@ get_time_offset(void)
     return (ErtsMonotonicTime) erts_smp_atomic64_read_acqb(&time_sup.inf.c.offset);
 }
 
+static ERTS_INLINE void
+update_last_mtime(ErtsSchedulerData *esdp, ErtsMonotonicTime mtime)
+{
+    if (!esdp)
+	esdp = erts_get_scheduler_data();
+    if (esdp) {
+	ASSERT(mtime >= esdp->last_monotonic_time);
+	esdp->last_monotonic_time = mtime;
+	esdp->check_time_reds = 0;
+    }
+}
 
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
 
@@ -411,15 +344,26 @@ print_correction(int change,
 
 #endif
 
+static ERTS_INLINE ErtsMonotonicTime
+get_timeout_pos(ErtsMonotonicTime now, ErtsMonotonicTime tmo)
+{
+    ErtsMonotonicTime tpos;
+    tpos = ERTS_MONOTONIC_TO_CLKTCKS(now - 1);
+    tpos += ERTS_MSEC_TO_CLKTCKS(tmo);
+    tpos += 1;
+    return tpos;
+}
+
 static void
-check_time_correction(void *idap)
+check_time_correction(void *vesdp)
 {
-    UWord init_drift_adj = (UWord) idap;
+    int init_drift_adj = !vesdp;
+    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
     ErtsMonotonicCorrectionData cdata;
     ErtsMonotonicCorrection new_correction;
     ErtsMonotonicCorrectionInstance *cip;
     ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime,
-	erl_stime, time_offset;
+	erl_stime, time_offset, timeout_pos;
     Uint timeout;
     int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time;
     int set_new_correction = 0, begin_short_intervals = 0;
@@ -681,6 +625,8 @@ check_time_correction(void *idap)
 	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK);
     }
 
+    timeout_pos = get_timeout_pos(erl_mtime, timeout);
+
 #ifdef ERTS_TIME_CORRECTION_PRINT
     print_correction(set_new_correction,
 		     sdiff,
@@ -723,11 +669,15 @@ check_time_correction(void *idap)
 	erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx);
     }
 
-    erts_set_timer(&time_sup.inf.c.parmon.timer,
-		   check_time_correction,
-		   NULL,
-		   NULL,
-		   timeout);
+    if (!esdp)
+	esdp = erts_get_scheduler_data();
+
+    erts_twheel_set_timer(esdp->timer_wheel,
+			  &time_sup.inf.c.parmon.timer,
+			  check_time_correction,
+			  NULL,
+			  (void *) esdp,
+			  timeout_pos);
 }
 
 static ErtsMonotonicTime get_os_corrected_time(void)
@@ -737,10 +687,11 @@ static ErtsMonotonicTime get_os_corrected_time(void)
 }
 
 static void
-check_time_offset(void *unused)
+check_time_offset(void *vesdp)
 {
+    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
     ErtsMonotonicTime sdiff, os_mtime, erl_mtime, os_stime,
-	erl_stime, time_offset;
+	erl_stime, time_offset, timeout, timeout_pos;
 
     ASSERT(time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE);
 
@@ -769,15 +720,19 @@ check_time_offset(void *unused)
 		      ERTS_MONOTONIC_TO_NSEC(sdiff));
 #endif
 
-    erts_set_timer(&time_sup.inf.c.parmon.timer,
-		   check_time_offset,
-		   NULL,
-		   NULL,
-		   ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK));
+    timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK);
+    timeout_pos = get_timeout_pos(erl_mtime, timeout);
+
+    erts_twheel_set_timer(esdp->timer_wheel,
+			  &time_sup.inf.c.parmon.timer,
+			  check_time_offset,
+			  NULL,
+			  vesdp,
+			  timeout_pos);
 }
 
 static void
-init_check_time_correction(void *quick_init_drift)
+init_check_time_correction(void *vesdp)
 {
     ErtsMonotonicDriftData *ddp;
     ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff,
@@ -821,7 +776,7 @@ init_check_time_correction(void *quick_init_drift)
     ddp->ix = 0;
     ddp->dirty_counter = time_sup.r.o.drift_adj.intervals;
 
-    check_time_correction(quick_init_drift);
+    check_time_correction(vesdp);
 }
 
 static ErtsMonotonicTime
@@ -850,14 +805,14 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep)
 }
 
 static void
-late_init_time_correction(void)
+late_init_time_correction(ErtsSchedulerData *esdp)
 {
-    Uint timeout;
-    Uint quick_init_drift_adj;
+    int quick_init_drift_adj;
     void (*check_func)(void *);
+    ErtsMonotonicTime timeout, timeout_pos;
 
     quick_init_drift_adj =
-	(Uint) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error) == 0;
+	ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error) == 0;
 
     if (quick_init_drift_adj)
 	timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK/10);
@@ -866,17 +821,25 @@ late_init_time_correction(void)
 
     if (!time_sup.r.o.os_corrected_monotonic_time)
 	check_func = init_check_time_correction;
-    else if (time_sup.r.o.get_time == get_os_corrected_time)
+    else if (time_sup.r.o.get_time == get_os_corrected_time) {
+	quick_init_drift_adj = 0;
 	check_func = check_time_offset;
+    }
     else
 	check_func = check_time_correction;
 
-    erts_init_timer(&time_sup.inf.c.parmon.timer);
-    erts_set_timer(&time_sup.inf.c.parmon.timer,
-		   check_func,
-		   NULL,
-		   (void *) quick_init_drift_adj,
-		   timeout);
+    timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp),
+				  timeout);
+
+    erts_twheel_init_timer(&time_sup.inf.c.parmon.timer);
+    erts_twheel_set_timer(esdp->timer_wheel,
+			  &time_sup.inf.c.parmon.timer,
+			  check_func,
+			  NULL,
+			  (quick_init_drift_adj
+			   ? NULL
+			   : esdp),
+			  timeout_pos);
 }
 
 #endif /* ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT */
@@ -987,6 +950,8 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     ErtsMonotonicTime abs_native_offset, native_offset;
 #endif
 
+    erts_hl_timer_init();
+
     ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX);
 
     erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday");
@@ -1003,51 +968,55 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
 
+    /*
+     * NOTE! erts_time_sup__.r.o.start *need* to be a multiple
+     *       of ERTS_MONOTONIC_TIME_UNIT.
+     */
+
 #ifdef ARCH_32
-    time_sup.r.o.start = ((((ErtsMonotonicTime) 1) << 32)-1);
-    time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
-    time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
-    time_sup.r.o.start += ERTS_MONOTONIC_TIME_UNIT;
-    native_offset = time_sup.r.o.start - ERTS_MONOTONIC_TIME_UNIT;
+    erts_time_sup__.r.o.start = ((((ErtsMonotonicTime) 1) << 32)-1);
+    erts_time_sup__.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
+    erts_time_sup__.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
+    erts_time_sup__.r.o.start += ERTS_MONOTONIC_TIME_UNIT;
+    native_offset = erts_time_sup__.r.o.start - ERTS_MONOTONIC_BEGIN;
     abs_native_offset = native_offset;
 #else /* ARCH_64 */
     if (ERTS_MONOTONIC_TIME_UNIT <= 10*1000*1000) {
-	time_sup.r.o.start = 0;
-	native_offset = -ERTS_MONOTONIC_TIME_UNIT;
-	abs_native_offset = ERTS_MONOTONIC_TIME_UNIT;
+	erts_time_sup__.r.o.start = 0;
+	native_offset = -ERTS_MONOTONIC_BEGIN;
+	abs_native_offset = ERTS_MONOTONIC_BEGIN;
     }
     else {
-	time_sup.r.o.start = ((ErtsMonotonicTime) MIN_SMALL);
-	time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
-	time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
-	native_offset = time_sup.r.o.start - ERTS_MONOTONIC_TIME_UNIT;
+	erts_time_sup__.r.o.start = ((ErtsMonotonicTime) MIN_SMALL);
+	erts_time_sup__.r.o.start /= ERTS_MONOTONIC_TIME_UNIT;
+	erts_time_sup__.r.o.start *= ERTS_MONOTONIC_TIME_UNIT;
+	native_offset = erts_time_sup__.r.o.start - ERTS_MONOTONIC_BEGIN;
 	abs_native_offset = -1*native_offset;
     }
 #endif
 
-    time_sup.r.o.start_offset.native = (time_sup.r.o.start
-					- ERTS_MONOTONIC_TIME_UNIT);
-    time_sup.r.o.start_offset.nsec = (ErtsMonotonicTime)
+    erts_time_sup__.r.o.start_offset.native = native_offset;
+    erts_time_sup__.r.o.start_offset.nsec = (ErtsMonotonicTime)
 	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1000*1000*1000);
-    time_sup.r.o.start_offset.usec = (ErtsMonotonicTime)
+    erts_time_sup__.r.o.start_offset.usec = (ErtsMonotonicTime)
 	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1000*1000);
-    time_sup.r.o.start_offset.msec = (ErtsMonotonicTime)
+    erts_time_sup__.r.o.start_offset.msec = (ErtsMonotonicTime)
 	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1000);
-    time_sup.r.o.start_offset.sec = (ErtsMonotonicTime)
+    erts_time_sup__.r.o.start_offset.sec = (ErtsMonotonicTime)
 	erts_time_unit_conversion((Uint64) abs_native_offset,
 				  (Uint32) ERTS_MONOTONIC_TIME_UNIT,
 				  (Uint32) 1);
     if (native_offset < 0) {
-	time_sup.r.o.start_offset.nsec *= -1;
-	time_sup.r.o.start_offset.usec *= -1;
-	time_sup.r.o.start_offset.msec *= -1;
-	time_sup.r.o.start_offset.sec *= -1;
+	erts_time_sup__.r.o.start_offset.nsec *= -1;
+	erts_time_sup__.r.o.start_offset.usec *= -1;
+	erts_time_sup__.r.o.start_offset.msec *= -1;
+	erts_time_sup__.r.o.start_offset.sec *= -1;
     }
 
 #endif
@@ -1143,9 +1112,9 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	erts_os_times(&time_sup.inf.c.minit,
 		      &time_sup.inf.c.sinit);
 	time_sup.r.o.moffset = -1*time_sup.inf.c.minit;
-	time_sup.r.o.moffset += ERTS_MONOTONIC_TIME_UNIT;
+	time_sup.r.o.moffset += ERTS_MONOTONIC_BEGIN;
 	offset = time_sup.inf.c.sinit;
-	offset -= ERTS_MONOTONIC_TIME_UNIT;
+	offset -= ERTS_MONOTONIC_BEGIN;
 	init_time_offset(offset);
 
 	rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
@@ -1160,7 +1129,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit;
 	cdatap->curr.correction.drift = 0;
 	cdatap->curr.correction.error = 0;
-	cdatap->curr.erl_mtime = ERTS_MONOTONIC_TIME_UNIT;
+	cdatap->curr.erl_mtime = ERTS_MONOTONIC_BEGIN;
 	cdatap->curr.os_mtime = time_sup.inf.c.minit;
 	cdatap->last_check = time_sup.inf.c.minit;
 	cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER;
@@ -1179,7 +1148,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	ErtsMonotonicTime stime, offset;
 	time_sup.r.o.get_time = get_not_corrected_time;
 	stime = time_sup.inf.c.sinit = erts_os_system_time();
-	offset = stime - ERTS_MONOTONIC_TIME_UNIT;
+	offset = stime - ERTS_MONOTONIC_BEGIN;
 	time_sup.inf.c.not_corrected_moffset = offset;
 	init_time_offset(offset);
 	time_sup.f.c.last_not_corrected_time = 0;
@@ -1199,14 +1168,24 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 void
 erts_late_init_time_sup(void)
 {
-#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
-    /* Timer wheel must have beeen initialized */
-    if (time_sup.r.o.get_time != get_not_corrected_time)
-	late_init_time_correction();
-#endif
     erts_late_sys_init_time();
 }
 
+void
+erts_sched_init_time_sup(ErtsSchedulerData *esdp)
+{
+    esdp->timer_wheel = erts_create_timer_wheel(esdp);
+    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
+    esdp->timer_service = erts_create_timer_service();
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    if (esdp->no == 1) {
+	/* A timer wheel to use must have beeen initialized */
+	if (time_sup.r.o.get_time != get_not_corrected_time)
+	    late_init_time_correction(esdp);
+    }
+#endif	
+}
+
 ErtsTimeWarpMode erts_time_warp_mode(void)
 {
     return time_sup.r.o.warp_mode;
@@ -1349,6 +1328,7 @@ wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff)
     erts_smp_mtx_lock(&erts_timeofday_mtx);
 
     now = time_sup.r.o.get_time();
+    update_last_mtime(NULL, now);
 
     elapsed = ERTS_MONOTONIC_TO_MSEC(now);
     *ms_total = (UWord) elapsed;
@@ -1737,6 +1717,7 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec)
     
     mtime = time_sup.r.o.get_time();
     time_offset = get_time_offset();
+    update_last_mtime(NULL, mtime);
     now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset);
 
     erts_smp_mtx_lock(&erts_timeofday_mtx);
@@ -1761,9 +1742,11 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec)
 }
 
 ErtsMonotonicTime
-erts_get_monotonic_time(void)
+erts_get_monotonic_time(ErtsSchedulerData *esdp)
 {
-    return time_sup.r.o.get_time();
+    ErtsMonotonicTime mtime = time_sup.r.o.get_time();
+    update_last_mtime(esdp, mtime);
+    return mtime;    
 }
 
 void
@@ -1994,7 +1977,13 @@ make_time_val(Process *c_p, ErtsMonotonicTime time_val)
 Eterm
 erts_get_monotonic_start_time(struct process *c_p)
 {
-    return make_time_val(c_p, ERTS_MONOTONIC_TIME_START);
+    return make_time_val(c_p, ERTS_MONOTONIC_TIME_START_EXTERNAL);
+}
+
+Eterm
+erts_get_monotonic_end_time(struct process *c_p)
+{
+    return make_time_val(c_p, ERTS_MONOTONIC_TIME_END_EXTERNAL);
 }
 
 static Eterm
@@ -2186,13 +2175,16 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto
 BIF_RETTYPE monotonic_time_0(BIF_ALIST_0)
 {
     ErtsMonotonicTime mtime = time_sup.r.o.get_time();
+    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
     mtime += ERTS_MONOTONIC_OFFSET_NATIVE;
     BIF_RET(make_time_val(BIF_P, mtime));
 }
 
 BIF_RETTYPE monotonic_time_1(BIF_ALIST_1)
 {
-    BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, time_sup.r.o.get_time(), 1));
+    ErtsMonotonicTime mtime = time_sup.r.o.get_time();
+    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
+    BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime, 1));
 }
 
 BIF_RETTYPE system_time_0(BIF_ALIST_0)
@@ -2200,6 +2192,7 @@ BIF_RETTYPE system_time_0(BIF_ALIST_0)
     ErtsMonotonicTime mtime, offset;
     mtime = time_sup.r.o.get_time();
     offset = get_time_offset();
+    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
     BIF_RET(make_time_val(BIF_P, mtime + offset));
 }
 
@@ -2208,6 +2201,7 @@ BIF_RETTYPE system_time_1(BIF_ALIST_0)
     ErtsMonotonicTime mtime, offset;
     mtime = time_sup.r.o.get_time();
     offset = get_time_offset();
+    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
     BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime + offset, 0));
 }
 
@@ -2237,6 +2231,7 @@ BIF_RETTYPE timestamp_0(BIF_ALIST_0)
 
     mtime = time_sup.r.o.get_time();
     offset = get_time_offset();
+    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
     stime = ERTS_MONOTONIC_TO_USEC(mtime + offset);
     all_sec = stime / ERTS_MONOTONIC_TIME_MEGA;
     mega_sec = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 634fe533d0..44d4ef18dd 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1305,8 +1305,7 @@ erts_alloc_message_heap_state(Uint size,
     state = erts_smp_atomic32_read_acqb(&receiver->state);
     if (statep)
 	*statep = state;
-    if (state & (ERTS_PSFLG_OFF_HEAP_MSGS
-		 | ERTS_PSFLG_EXITING
+    if (state & (ERTS_PSFLG_EXITING
 		 | ERTS_PSFLG_PENDING_EXIT))
 	goto allocate_in_mbuf;
 #endif
@@ -1327,8 +1326,7 @@ erts_alloc_message_heap_state(Uint size,
 	state = erts_smp_atomic32_read_nob(&receiver->state);
 	if (statep)
 	    *statep = state;
-	if ((state & (ERTS_PSFLG_OFF_HEAP_MSGS
-		      | ERTS_PSFLG_EXITING
+	if ((state & (ERTS_PSFLG_EXITING
 		      | ERTS_PSFLG_PENDING_EXIT))
 	    || (receiver->flags & F_DISABLE_GC)
 	    || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 1db3a9fba7..3e5e97ab15 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -48,6 +48,7 @@
 #include "dtrace-wrapper.h"
 #include "erl_map.h"
 #include "erl_bif_unique.h"
+#include "erl_hl_timer.h"
 
 extern ErlDrvEntry fd_driver_entry;
 #ifndef __OSE__
@@ -379,11 +380,7 @@ static Port *create_port(char *name,
     prt->dist_entry = NULL;
     ERTS_PORT_INIT_CONNECTED(prt, pid);
     prt->common.u.alive.reg = NULL;
-#ifdef ERTS_SMP
-    prt->common.u.alive.ptimer = NULL;
-#else
-    sys_memset(&prt->common.u.alive.tm, 0, sizeof(ErlTimer));
-#endif
+    ERTS_PTMR_INIT(prt);
     erts_port_task_handle_init(&prt->timeout_task);
     prt->psd = NULL;
     prt->drv_data = (SWord) 0;
@@ -463,11 +460,7 @@ erts_port_free(Port *prt)
 			    | ERTS_PORT_SFLG_FREE));
     ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG);
 
-#ifdef ERTS_SMP
-    ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->common.refc) == 0);
-#else
-    ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->refc) == 0);
-#endif
+    ERTS_LC_ASSERT(erts_atomic_read_nob(&prt->common.refc.atmc) == 0);
 
     erts_port_task_fini_sched(&prt->sched);
 
@@ -736,11 +729,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	/*
 	 * Must clean up the port.
 	 */
-#ifdef ERTS_SMP
-	erts_cancel_smp_ptimer(port->common.u.alive.ptimer);
-#else
-	erts_cancel_timer(&(port->common.u.alive.tm));
-#endif
+	erts_cancel_port_timer(port);
 	stopq(port);
 	if (port->linebuf != NULL) {
 	    erts_free(ERTS_ALC_T_LINEBUF,
@@ -2806,7 +2795,8 @@ void erts_init_io(int port_tab_size,
 			 port_tab_size,
 			 common_element_size, /* Doesn't need to be excact */
 			 "port_table",
-			 legacy_port_tab);
+			 legacy_port_tab,
+			 1);
 
     erts_smp_atomic_init_nob(&erts_bytes_out, 0);
     erts_smp_atomic_init_nob(&erts_bytes_in, 0);
@@ -3073,7 +3063,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
 
     rp = (scheduler
 	  ? erts_proc_lookup(pid)
-	  : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC));
+	  : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC));
 
     if (rp) {
 	Eterm tuple;
@@ -3095,7 +3085,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
 	if (!scheduler)
-	    erts_smp_proc_dec_refc(rp);
+	    erts_proc_dec_refc(rp);
 
     }
 }
@@ -3139,7 +3129,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
 
     rp = (scheduler
 	  ? erts_proc_lookup(to)
-	  : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
+	  : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC));
 
     if (!rp)
 	return;
@@ -3194,7 +3184,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
-	erts_smp_proc_dec_refc(rp);
+	erts_proc_dec_refc(rp);
 }
 
 /* 
@@ -3280,7 +3270,7 @@ deliver_vec_message(Port* prt,			/* Port */
 
     rp = (scheduler
 	  ? erts_proc_lookup(to)
-	  : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
+	  : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC));
     if (!rp)
 	return;
 
@@ -3364,7 +3354,7 @@ deliver_vec_message(Port* prt,			/* Port */
 		       );
     erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
-	erts_smp_proc_dec_refc(rp);
+	erts_proc_dec_refc(rp);
 }
 
 
@@ -3454,11 +3444,8 @@ terminate_port(Port *prt)
 	send_closed_port_id = NIL;
     }
 
-#ifdef ERTS_SMP
-    erts_cancel_smp_ptimer(prt->common.u.alive.ptimer);
-#else
-    erts_cancel_timer(&prt->common.u.alive.tm);
-#endif
+    if (ERTS_PTMR_IS_SET(prt))
+	erts_cancel_port_timer(prt);
 
     drv = prt->drv_ptr;
     if ((drv != NULL) && (drv->stop != NULL)) {
@@ -5005,24 +4992,6 @@ erts_free_port_names(ErtsPortNames *pnp)
     erts_free(ERTS_ALC_T_PORT_NAMES, pnp);
 }
 
-static void schedule_port_timeout(Port *p)
-{
-    /*
-     * Scheduling of port timeouts can be done without port locking, but
-     * since the task handle is stored in the port structure and the ptimer
-     * structure is protected by the port lock we require the port to be
-     * locked for now...
-     *
-     * TODO: Implement scheduling of port timeouts without locking
-     *       the port.
-     * /Rickard
-     */
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
-    erts_port_task_schedule(p->common.id,
-			    &p->timeout_task,
-			    ERTS_PORT_TASK_TIMEOUT);
-}
-
 ErlDrvTermData driver_mk_term_nil(void)
 {
     return driver_term_nil;
@@ -5051,7 +5020,7 @@ void driver_report_exit(ErlDrvPort ix, int status)
 
    rp = (scheduler
 	 ? erts_proc_lookup(pid)
-	 : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC));
+	 : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC));
    if (!rp)
        return;
 
@@ -5069,7 +5038,7 @@ void driver_report_exit(ErlDrvPort ix, int status)
 
    erts_smp_proc_unlock(rp, rp_locks);
    if (!scheduler)
-       erts_smp_proc_dec_refc(rp);
+       erts_proc_dec_refc(rp);
 }
 
 #define ERTS_B2T_STATES_DEF_STATES_SZ 5
@@ -5385,7 +5354,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
     scheduler = erts_get_scheduler_id() != 0;
     rp = (scheduler
 	  ? erts_proc_lookup(to)
-	  : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC));
+	  : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC));
     if (!rp) {
 	res = 0;
 	goto done;
@@ -5682,14 +5651,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    HRelease(rp, hp_end, hp);
 	}
     }
-#ifdef ERTS_SMP
     if (rp) {
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
 	if (!scheduler)
-	    erts_smp_proc_dec_refc(rp);
+	    erts_proc_dec_refc(rp);
     }
-#endif
     cleanup_b2t_states(&b2t);
     DESTROY_ESTACK(stack);
     return res;
@@ -6635,18 +6602,6 @@ int driver_pushq(ErlDrvPort ix, char* buffer, ErlDrvSizeT len)
     return code;
 }
 
-static ERTS_INLINE void
-drv_cancel_timer(Port *prt)
-{
-#ifdef ERTS_SMP
-    erts_cancel_smp_ptimer(prt->common.u.alive.ptimer);
-#else
-    erts_cancel_timer(&prt->common.u.alive.tm);
-#endif
-    if (erts_port_task_is_scheduled(&prt->timeout_task))
-	erts_port_task_abort(&prt->timeout_task);
-}
-
 int driver_set_timer(ErlDrvPort ix, unsigned long t)
 {
     Port* prt = erts_drvport2port(ix);
@@ -6658,19 +6613,8 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t)
 
     if (prt->drv_ptr->timeout == NULL)
 	return -1;
-    drv_cancel_timer(prt);
-#ifdef ERTS_SMP
-    erts_create_smp_ptimer(&prt->common.u.alive.ptimer,
-			   prt->common.id,
-			   (ErlTimeoutProc) schedule_port_timeout,
-			   t);
-#else
-    erts_set_timer(&prt->common.u.alive.tm,
-		  (ErlTimeoutProc) schedule_port_timeout,
-		  NULL,
-		  prt,
-		  t);
-#endif
+
+    erts_set_port_timer(prt, (Sint64) t);
     return 0;
 }
 
@@ -6680,28 +6624,28 @@ int driver_cancel_timer(ErlDrvPort ix)
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
-    drv_cancel_timer(prt);
+    erts_cancel_port_timer(prt);
     return 0;
 }
 
-
 int
 driver_read_timer(ErlDrvPort ix, unsigned long* t)
 {
     Port* prt = erts_drvport2port(ix);
+    Sint64 left;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
-#ifdef ERTS_SMP
-    *t = (prt->common.u.alive.ptimer
-	  ? erts_time_left(&prt->common.u.alive.ptimer->timer.tm)
-	  : 0);
-#else
-    *t = erts_time_left(&prt->common.u.alive.tm);
-#endif
+
+    left = erts_read_port_timer(prt);
+    if (left < 0)
+	left = 0;
+
+    *t = (unsigned long) left;
+
     return 0;
 }
 
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index c626cb2780..4d557b3a17 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -269,7 +269,10 @@ erts_whereis_name_to_id(Process *c_p, Eterm name)
 #ifdef ERTS_SMP
     ErtsProcLocks c_p_locks = c_p ? ERTS_PROC_LOCK_MAIN : 0;
 
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    if (c_p) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
+#endif
+
     reg_safe_read_lock(c_p, &c_p_locks);
     if (c_p && !c_p_locks)
         erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
@@ -380,8 +383,6 @@ erts_whereis_name(Process *c_p,
 			erts_smp_proc_unlock(rp->p, need_locks);
 		    *proc = NULL;
 		}
-		if (*proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC))
-		    erts_smp_proc_inc_refc(rp->p);
 	    }
 #else
 	    if (rp->p
@@ -390,6 +391,8 @@ erts_whereis_name(Process *c_p,
 	    else
 		*proc = NULL;
 #endif
+	    if (*proc && (flags & ERTS_P2P_FLG_INC_REFC))
+		erts_proc_inc_refc(*proc);
 	}
     }
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index f7a21406f3..54059ee9a5 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -776,8 +776,6 @@ extern char *erts_sys_ddll_error(int code);
 /*
  * System interfaces for startup.
  */
-#include "erl_time.h"
-
 void erts_sys_schedule_interrupt(int set);
 #ifdef ERTS_SMP
 void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
@@ -819,7 +817,8 @@ int univ_to_local(
 int local_to_univ(Sint *year, Sint *month, Sint *day, 
 		  Sint *hour, Sint *minute, Sint *second, int isdst);
 void get_now(Uint*, Uint*, Uint*);
-ErtsMonotonicTime erts_get_monotonic_time(void);
+struct ErtsSchedulerData_;
+ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
 void get_sys_now(Uint*, Uint*, Uint*);
 void set_break_quit(void (*)(void), void (*)(void));
 
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 1fffb5f357..ea19d8b362 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -76,6 +76,11 @@
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
+
+#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24)
+#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY)
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #define ASSERT_NO_LOCKED_LOCKS		erts_lc_check_exact(NULL, 0)
@@ -83,20 +88,24 @@
 #define ASSERT_NO_LOCKED_LOCKS
 #endif
 
-#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24)
-#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY)
-
+#if 0
+#  define ERTS_TW_DEBUG
+#endif
+#if defined(DEBUG) && !defined(ERTS_TW_DEBUG)
+#  define ERTS_TW_DEBUG
+#endif
 
-/* BEGIN tiw_lock protected variables 
-**
-** The individual timer cells in tiw are also protected by the same mutex.
-*/
+#undef ERTS_TW_ASSERT
+#if defined(ERTS_TW_DEBUG)
+#  define ERTS_TW_ASSERT(E) ERTS_ASSERT(E)
+#else
+#  define ERTS_TW_ASSERT(E) ((void) 1)
+#endif
 
-/* timing wheel size NEED to be a power of 2 */
-#ifdef SMALL_MEMORY
-#define TIW_SIZE (1 << 13)
+#ifdef ERTS_TW_DEBUG
+#  define ERTS_TWHEEL_BUMP_YIELD_LIMIT 5
 #else
-#define TIW_SIZE (1 << 20)
+#  define ERTS_TWHEEL_BUMP_YIELD_LIMIT 100
 #endif
 
 /* Actual interval time chosen by sys_init_time() */
@@ -110,54 +119,22 @@ static int tiw_itime; /* Constant after init */
 #endif
 
 struct ErtsTimerWheel_ {
-    ErlTimer *w[TIW_SIZE];
+    ErtsTWheelTimer *w[ERTS_TIW_SIZE];
     ErtsMonotonicTime pos;
     Uint nto;
     struct {
-	ErlTimer *head;
-	ErlTimer *tail;
+	ErtsTWheelTimer *head;
+	ErtsTWheelTimer *tail;
 	Uint nto;
     } at_once;
+    int yield_slot;
+    int yield_slots_left;
+    int yield_start_pos;
+    ErtsTWheelTimer sentinel;
     int true_next_timeout_time;
     ErtsMonotonicTime next_timeout_time;
-    erts_atomic64_t next_timeout;
-    erts_smp_atomic32_t is_bumping;
-    erts_smp_mtx_t lock;
 };
 
-ErtsTimerWheel *erts_default_timer_wheel; /* managed by aux thread */
-
-static ERTS_INLINE ErtsTimerWheel *
-get_timer_wheel(ErlTimer *p)
-{
-    return (ErtsTimerWheel *) erts_smp_atomic_read_acqb(&p->wheel);
-}
-
-static ERTS_INLINE void
-set_timer_wheel(ErlTimer *p, ErtsTimerWheel *tiw)
-{
-    erts_smp_atomic_set_relb(&p->wheel, (erts_aint_t) tiw);
-}
-
-static ERTS_INLINE void
-init_next_timeout(ErtsTimerWheel *tiw,
-		  ErtsMonotonicTime time)
-{
-    erts_atomic64_init_nob(&tiw->next_timeout,
-			   (erts_aint64_t) time);
-}
-
-static ERTS_INLINE void
-set_next_timeout(ErtsTimerWheel *tiw,
-		 ErtsMonotonicTime time,
-		 int true_timeout)
-{
-    tiw->true_next_timeout_time = true_timeout;
-    tiw->next_timeout_time = time;
-    erts_atomic64_set_relb(&tiw->next_timeout,
-			   (erts_aint64_t) time);
-}
-
 /* get the time (in units of TIW_ITIME) to the next timeout,
    or -1 if there are no timeouts                     */
 
@@ -167,35 +144,23 @@ find_next_timeout(ErtsTimerWheel *tiw,
 		  ErtsMonotonicTime max_search_time)
 {
     int start_ix, tiw_pos_ix;
-    ErlTimer *p;
+    ErtsTWheelTimer *p;
     int true_min_timeout;
-    ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos, timeout_limit;
-
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw->lock));
+    ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos;
 
     if (tiw->true_next_timeout_time)
 	return tiw->next_timeout_time;
 
-    /* We never set next timeout beyond timeout_limit */
-    timeout_limit = curr_time + ERTS_MONOTONIC_DAY;
-
     if (tiw->nto == 0) { /* no timeouts in wheel */
-	true_min_timeout = tiw->true_next_timeout_time = 0;
-	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(timeout_limit);
+	true_min_timeout = 0;
+	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + ERTS_MONOTONIC_DAY);
 	goto found_next;
     }
 
-    /*
-     * Don't want others entering trying to bump
-     * timers while we are checking...
-     */
-    set_next_timeout(tiw, timeout_limit, 0);
-
-    true_min_timeout = 1;
     slot_timeout_pos = tiw->pos;
     min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
 
-    start_ix = tiw_pos_ix = (int) (tiw->pos & (TIW_SIZE-1));
+    start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1));
 
     do {
 	slot_timeout_pos++;
@@ -206,93 +171,125 @@ find_next_timeout(ErtsTimerWheel *tiw,
 
 	p = tiw->w[tiw_pos_ix];
 
-	while (p) {
-	    ErtsMonotonicTime timeout_pos;
-	    ASSERT(p != p->next);
-	    timeout_pos = p->timeout_pos;
-	    if (min_timeout_pos > timeout_pos) {
-		min_timeout_pos = timeout_pos;
-		if (min_timeout_pos <= slot_timeout_pos)
-		    goto found_next;
-	    }
-	    p = p->next;
+	if (p) {
+	    ErtsTWheelTimer *end = p;
+
+	    do  {
+		ErtsMonotonicTime timeout_pos;
+		timeout_pos = p->timeout_pos;
+		if (min_timeout_pos > timeout_pos) {
+		    true_min_timeout = 1;
+		    min_timeout_pos = timeout_pos;
+		    if (min_timeout_pos <= slot_timeout_pos)
+			goto found_next;
+		}
+		p = p->next;
+	    } while (p != end);
 	}
 
 	tiw_pos_ix++;
-	if (tiw_pos_ix == TIW_SIZE)
+	if (tiw_pos_ix == ERTS_TIW_SIZE)
 	    tiw_pos_ix = 0;
     } while (start_ix != tiw_pos_ix);
 
 found_next:
 
     min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos);
-    if (min_timeout != tiw->next_timeout_time)
-	set_next_timeout(tiw, min_timeout, true_min_timeout);
+    tiw->next_timeout_time = min_timeout;
+    tiw->true_next_timeout_time = true_min_timeout;
 
     return min_timeout;
 }
 
-static void
-remove_timer(ErtsTimerWheel *tiw, ErlTimer *p)
+static ERTS_INLINE void
+insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
 {
-    if (p->slot >= 0) {
-	/* Timer in wheel... */
-	ASSERT(p->slot < TIW_SIZE);
-	if (p->prev)
-	    p->prev->next = p->next;
-	else {
-	    ASSERT(tiw->w[p->slot] == p);
-	    tiw->w[p->slot] = p->next;
+    ERTS_TW_ASSERT(slot >= 0);
+    ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE);
+    p->slot = slot;
+    if (!tiw->w[slot]) {
+	tiw->w[slot] = p;
+	p->next = p;
+	p->prev = p;
+    }
+    else {
+	ErtsTWheelTimer *next, *prev;
+	next = tiw->w[slot];
+	prev = next->prev;
+	p->next = next;
+	p->prev = prev;
+	prev->next = p;
+	next->prev = p;
+    }
+}
+
+static ERTS_INLINE void
+remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
+{
+    int slot = p->slot;
+    ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE);
+
+    if (slot >= 0) {
+	/*
+	 * Timer in wheel or in circular
+	 * list of timers currently beeing
+	 * triggered (referred by sentinel).
+	 */
+	ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE);
+
+	if (p->next == p) {
+	    ERTS_TW_ASSERT(tiw->w[slot] == p);
+	    tiw->w[slot] = NULL;
 	}
-	if(p->next)
+	else {
+	    if (tiw->w[slot] == p)
+		tiw->w[slot] = p->next;
+	    p->prev->next = p->next;
 	    p->next->prev = p->prev;
+	}
     }
     else {
 	/* Timer in "at once" queue... */
-	ASSERT(p->slot == -1);
+	ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE);
 	if (p->prev)
 	    p->prev->next = p->next;
 	else {
-	    ASSERT(tiw->at_once.head == p);
+	    ERTS_TW_ASSERT(tiw->at_once.head == p);
 	    tiw->at_once.head = p->next;
 	}
 	if (p->next)
 	    p->next->prev = p->prev;
 	else {
-	    ASSERT(tiw->at_once.tail == p);
+	    ERTS_TW_ASSERT(tiw->at_once.tail == p);
 	    tiw->at_once.tail = p->prev;
 	}
-	ASSERT(tiw->at_once.nto > 0);
+	ERTS_TW_ASSERT(tiw->at_once.nto > 0);
 	tiw->at_once.nto--;
     }
 
-    p->slot = -2;
+    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
 
+#if 0
     p->next = NULL;
     p->prev = NULL;
+#endif
 
-    set_timer_wheel(p, NULL);
     tiw->nto--;
 }
 
 ErtsMonotonicTime
-erts_check_next_timeout_time(ErtsTimerWheel *tiw,
-			     ErtsMonotonicTime max_search_time)
+erts_check_next_timeout_time(ErtsSchedulerData *esdp)
 {
-    ErtsMonotonicTime next, curr;
-
-    curr = erts_get_monotonic_time();
-
-    erts_smp_mtx_lock(&tiw->lock);
-
-    next = find_next_timeout(tiw, curr, max_search_time);
-
-    erts_smp_mtx_unlock(&tiw->lock);
-
-    return next;
+    /*
+     * Called before a scheduler is about to wait. We wont
+     * check more than 10 minutes into the future.
+     */
+    return find_next_timeout(esdp->timer_wheel,
+			     erts_get_monotonic_time(esdp),
+			     ERTS_SEC_TO_MONOTONIC(10*60));
 }
 
-#ifndef DEBUG
+#ifndef ERTS_TW_DEBUG
 #define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0)
 #else
 #define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO))
@@ -300,198 +297,255 @@ static void
 debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos)
 {
     int slots, ix;
-    ErlTimer *tmr;
+    ErtsTWheelTimer *tmr;
     ErtsMonotonicTime tmp;
 
-    ix = (int) (tiw->pos & (TIW_SIZE-1));
+    ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1));
     tmp = skip_to_pos - tiw->pos;
-    ASSERT(tmp >= 0);
-    if (tmp < (ErtsMonotonicTime) TIW_SIZE)
+    ERTS_TW_ASSERT(tmp >= 0);
+    if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE)
 	slots = (int) tmp;
     else
-	slots = TIW_SIZE;
+	slots = ERTS_TIW_SIZE;
 
      while (slots > 0) {
-	tmr = tiw->w[ix];
-	while (tmr) {
-	    ASSERT(tmr->timeout_pos > skip_to_pos);
-	    tmr = tmr->next;
-	}
-	ix++;
-	if (ix == TIW_SIZE)
-	    ix = 0;
-	slots--;
+	 tmr = tiw->w[ix];
+	 if (tmr) {
+	     ErtsTWheelTimer *end = tmr;
+	     do {
+		 ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos);
+		 tmr = tmr->next;
+	     } while (tmr != end);
+	 }
+	 ix++;
+	 if (ix == ERTS_TIW_SIZE)
+	     ix = 0;
+	 slots--;
     }
 }
 #endif
 
+static ERTS_INLINE void
+timeout_timer(ErtsTWheelTimer *p)
+{
+    ErlTimeoutProc timeout;
+    void *arg;
+#if 0
+    p->next = NULL;
+    p->prev = NULL;
+#endif
+    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    timeout = p->u.func.timeout;
+    arg = p->u.func.arg;
+    (*timeout)(arg);
+}
+
 void
 erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
-    int tiw_pos_ix, slots;
-    ErlTimer *p, *timeout_head, **timeout_tail;
-    ErtsMonotonicTime bump_to, tmp_slots;
-
-    if (erts_smp_atomic32_cmpxchg_nob(&tiw->is_bumping, 1, 0) != 0)
-	return; /* Another thread is currently bumping... */
+    int tiw_pos_ix, slots, yielded_slot_restarted, yield_count;
+    ErtsMonotonicTime bump_to, tmp_slots, old_pos;
 
-    bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
+    yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT;
 
-    erts_smp_mtx_lock(&tiw->lock);
+    /*
+     * In order to be fair we always continue with work
+     * where we left off when restarting after a yield.
+     */
 
-    if (tiw->pos >= bump_to) {
-	timeout_head = NULL;
-	goto done;
+    if (tiw->yield_slot >= 0) {
+	yielded_slot_restarted = 1;
+	tiw_pos_ix = tiw->yield_slot;
+	slots = tiw->yield_slots_left;
+	bump_to = tiw->pos;
+	old_pos = tiw->yield_start_pos;
+	goto restart_yielded_slot;
     }
 
-    /* Don't want others here while we are bumping... */
-    set_next_timeout(tiw, curr_time + ERTS_MONOTONIC_DAY, 0);
+    do {
+
+	yielded_slot_restarted = 0;
+
+	bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 
-    if (!tiw->at_once.head) {
-	timeout_head = NULL;
-	timeout_tail = &timeout_head;
-    }
-    else {
-	ASSERT(tiw->nto >= tiw->at_once.nto);
-	p = timeout_head = tiw->at_once.head;
 	while (1) {
-	    set_timer_wheel(p, NULL);
-	    if (!p->next) {
-		timeout_tail = &p->next;
-		break;
+	    ErtsTWheelTimer *p;
+
+	    old_pos = tiw->pos;
+
+	    if (tiw->nto == 0) {
+	    empty_wheel:
+		ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to);
+		tiw->true_next_timeout_time = 0;
+		tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY;
+		tiw->pos = bump_to;
+		tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+		return;
 	    }
-	}
-	tiw->nto -= tiw->at_once.nto;
-	tiw->at_once.head = NULL;
-	tiw->at_once.tail = NULL;
-	tiw->at_once.nto = 0;
-    }
 
-    if (tiw->nto == 0) {
-	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to);
-	tiw->pos = bump_to;
-	goto done;
-    }
+	    p = tiw->at_once.head;
+	    while (p) {
+		if (--yield_count <= 0) {
+		    ERTS_TW_ASSERT(tiw->nto > 0);
+		    ERTS_TW_ASSERT(tiw->at_once.nto > 0);
+		    tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE;
+		    tiw->true_next_timeout_time = 1;
+		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos);
+		    return;
+		}
+
+		ERTS_TW_ASSERT(tiw->nto > 0);
+		ERTS_TW_ASSERT(tiw->at_once.nto > 0);
+		tiw->nto--;
+		tiw->at_once.nto--;
+		tiw->at_once.head = p->next;
+		if (p->next)
+		    p->next->prev = NULL;
+		else
+		    tiw->at_once.tail = NULL;
+
+		timeout_timer(p);
+
+		p = tiw->at_once.head;
+	    }
 
-    if (tiw->true_next_timeout_time) {
-	ErtsMonotonicTime skip_until_pos;
-	/*
-	 * No need inspecting slots where we know no timeouts
-	 * to trigger should reside.
-	 */
+	    if (tiw->pos >= bump_to)
+		break;
 
-	skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
-	if (skip_until_pos > bump_to)
-	    skip_until_pos = bump_to;
+	    if (tiw->nto == 0)
+		goto empty_wheel;
 
-	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos);
-	ASSERT(skip_until_pos > tiw->pos);
+	    if (tiw->true_next_timeout_time) {
+		ErtsMonotonicTime skip_until_pos;
+		/*
+		 * No need inspecting slots where we know no timeouts
+		 * to trigger should reside.
+		 */
 
-	tiw->pos = skip_until_pos - 1;
-    }
+		skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
+		if (skip_until_pos > bump_to)
+		    skip_until_pos = bump_to;
 
-    tiw_pos_ix = (int) ((tiw->pos+1) & (TIW_SIZE-1));
-    tmp_slots = (bump_to - tiw->pos);
-    if (tmp_slots < (ErtsMonotonicTime) TIW_SIZE)
-	slots = (int) tmp_slots;
-    else
-	slots = TIW_SIZE;
- 
-    while (slots > 0) {
-	p = tiw->w[tiw_pos_ix];
-	while (p) {
-	    ErlTimer *next = p->next;
-	    ASSERT(p != next);
-	    if (p->timeout_pos <= bump_to) { /* we have a timeout */
-		/* Remove from list */
-		remove_timer(tiw, p);
-		*timeout_tail = p;	/* Insert in timeout queue */
-		timeout_tail = &p->next;
+		skip_until_pos--;
+
+		if (skip_until_pos > tiw->pos) {
+		    ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos);
+
+		    tiw->pos = skip_until_pos;
+		}
+	    }
+
+	    tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1));
+	    tmp_slots = (bump_to - tiw->pos);
+	    if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE)
+		slots = (int) tmp_slots;
+	    else
+		slots = ERTS_TIW_SIZE;
+
+	    tiw->pos = bump_to;
+
+	    while (slots > 0) {
+
+		p = tiw->w[tiw_pos_ix];
+		if (p) {
+		    if (p->next == p) {
+			ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel);
+			ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+		    }
+		    else {
+			tiw->sentinel.next = p->next;
+			tiw->sentinel.prev = p->prev;
+			tiw->sentinel.next->prev = &tiw->sentinel;
+			tiw->sentinel.prev->next = &tiw->sentinel;
+		    }
+		    tiw->w[tiw_pos_ix] = NULL;
+
+		    while (1) {
+
+			if (p->timeout_pos > bump_to) {
+			    /* Very unusual case... */
+			    ++yield_count;
+			    insert_timer_into_slot(tiw, tiw_pos_ix, p);
+			}
+			else {
+			    /* Normal case... */
+			    timeout_timer(p);
+			    tiw->nto--;
+			}
+
+		    restart_yielded_slot:
+
+			p = tiw->sentinel.next;
+			if (p == &tiw->sentinel) {
+			    ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+			    break;
+			}
+
+			if (--yield_count <= 0) {
+			    tiw->true_next_timeout_time = 1;
+			    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos);
+			    tiw->yield_slot = tiw_pos_ix;
+			    tiw->yield_slots_left = slots;
+			    tiw->yield_start_pos = old_pos;
+			    return; /* Yield! */
+			}
+
+			tiw->sentinel.next = p->next;
+			p->next->prev = &tiw->sentinel;
+		    }
+		}
+		tiw_pos_ix++;
+		if (tiw_pos_ix == ERTS_TIW_SIZE)
+		    tiw_pos_ix = 0;
+		slots--;
 	    }
-	    p = next;
 	}
-	tiw_pos_ix++;
-	if (tiw_pos_ix == TIW_SIZE)
-	    tiw_pos_ix = 0;
-	slots--;
-    }
 
-    ASSERT(tmp_slots >= (ErtsMonotonicTime) TIW_SIZE
-	   || tiw_pos_ix == (int) ((bump_to+1) & (TIW_SIZE-1)));
+    } while (yielded_slot_restarted);
 
-    tiw->pos = bump_to;
+    tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    tiw->true_next_timeout_time = 0;
+    tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY;
 
     /* Search at most two seconds ahead... */
     (void) find_next_timeout(tiw, curr_time, ERTS_SEC_TO_MONOTONIC(2));
-
-done:
-
-    erts_smp_mtx_unlock(&tiw->lock);
-    
-    erts_smp_atomic32_set_nob(&tiw->is_bumping, 0);
-
-    /* Call timedout timers callbacks */
-    while (timeout_head) {
-	ErlTimeoutProc timeout;
-	void *arg;
-	p = timeout_head;
-	timeout_head = p->next;
-	/* Here comes hairy use of the timer fields!
-	 * They are reset without having the lock.
-	 * It is assumed that no code but this will
-	 * accesses any field until the ->timeout
-	 * callback is called.
-	 */
-	ASSERT(p->timeout_pos <= bump_to);
-	p->next = NULL;
-	p->prev = NULL;
-	p->slot = 0;
-	timeout = p->timeout;
-	arg = p->arg;
-	(*timeout)(arg);
-    }
 }
 
 Uint
 erts_timer_wheel_memory_size(void)
 {
-#ifdef ERTS_SMP
-    return sizeof(ErtsTimerWheel)*(1 + erts_no_schedulers);
-#else
-    return sizeof(ErtsTimerWheel);
-#endif
+    return sizeof(ErtsTimerWheel)*erts_no_schedulers;
 }
 
 ErtsTimerWheel *
-erts_create_timer_wheel(int no)
+erts_create_timer_wheel(ErtsSchedulerData *esdp)
 {
     ErtsMonotonicTime mtime;
     int i;
     ErtsTimerWheel *tiw;
-    tiw = (ErtsTimerWheel *) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
-					sizeof(ErtsTimerWheel));
-    for(i = 0; i < TIW_SIZE; i++)
+    tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL,
+					     sizeof(ErtsTimerWheel));
+    for(i = 0; i < ERTS_TIW_SIZE; i++)
 	tiw->w[i] = NULL;
 
-    erts_smp_atomic32_init_nob(&tiw->is_bumping, 0);
-    erts_smp_mtx_init_x(&tiw->lock, "timer_wheel", make_small(no));
-
-    mtime = erts_get_monotonic_time();
+    mtime = erts_get_monotonic_time(esdp);
     tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
     tiw->nto = 0;
     tiw->at_once.head = NULL;
     tiw->at_once.tail = NULL;
     tiw->at_once.nto = 0;
+    tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
-    init_next_timeout(tiw, mtime + ERTS_MONOTONIC_DAY);
+    tiw->sentinel.next = &tiw->sentinel;
+    tiw->sentinel.prev = &tiw->sentinel;
     return tiw;
 }
 
 ErtsNextTimeoutRef
 erts_get_next_timeout_reference(ErtsTimerWheel *tiw)
 {
-    return (ErtsNextTimeoutRef) &tiw->next_timeout;
+    return (ErtsNextTimeoutRef) &tiw->next_timeout_time;
 }
 
 
@@ -512,155 +566,83 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
 #else
     tiw_itime = itime;
 #endif
-
-    erts_default_timer_wheel = erts_create_timer_wheel(0);
 }
 
 void
-erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout,
-	       ErlCancelProc cancel, void *arg, Uint to)
+erts_twheel_set_timer(ErtsTimerWheel *tiw,
+		      ErtsTWheelTimer *p, ErlTimeoutProc timeout,
+		      ErlCancelProc cancel, void *arg,
+		      ErtsMonotonicTime timeout_pos)
 {
-    ErtsMonotonicTime timeout_time, timeout_pos;
-    ErtsMonotonicTime curr_time;
-    ErtsTimerWheel *tiw;
-    ErtsSchedulerData *esdp;
-
-    curr_time = erts_get_monotonic_time();
-    esdp = erts_get_scheduler_data();
-    if (esdp)
-	tiw = esdp->timer_wheel;
-    else
-	tiw = erts_default_timer_wheel;
+    ErtsMonotonicTime timeout_time;
 
-    erts_smp_mtx_lock(&tiw->lock);
+    p->u.func.timeout = timeout;
+    p->u.func.cancel = cancel;
+    p->u.func.arg = arg;
 
-    if (get_timer_wheel(p))
-	ERTS_INTERNAL_ERROR("Double set timer");
+    ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE);
 
-    p->timeout = timeout;
-    p->cancel = cancel;
-    p->arg = arg;
-
-    if (to == 0) {
-	timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
+    if (timeout_pos <= tiw->pos) {
 	tiw->nto++;
 	tiw->at_once.nto++;
 	p->next = NULL;
 	p->prev = tiw->at_once.tail;
-	tiw->at_once.tail = p;
-	if (!tiw->at_once.head)
+	if (tiw->at_once.tail) {
+	    ERTS_TW_ASSERT(tiw->at_once.head);
+	    tiw->at_once.tail->next = p;
+	}
+	else {	
+	    ERTS_TW_ASSERT(!tiw->at_once.head);
 	    tiw->at_once.head = p;
-	p->timeout_pos = timeout_pos;
-	p->slot = -1;
-	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+	}
+	tiw->at_once.tail = p;
+	p->timeout_pos = tiw->pos;
+	p->slot = ERTS_TWHEEL_SLOT_AT_ONCE;
+	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos);
     }
     else {
-	int tm;
-	ErtsMonotonicTime ticks;
-
-	ticks = ERTS_MSEC_TO_CLKTCKS(to);
-	timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time - 1) + 1 + ticks;
+	int slot;
 
 	/* calculate slot */
-	tm = (int) (timeout_pos & (TIW_SIZE-1));
-	p->slot = tm;
-  
-	/* insert at head of list at slot */
-	p->next = tiw->w[tm];
-	p->prev = NULL;
-	if (p->next != NULL)
-	    p->next->prev = p;
-	tiw->w[tm] = p;
+	slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1));
+
+	insert_timer_into_slot(tiw, slot, p);
 
 	tiw->nto++;
 
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
 	p->timeout_pos = timeout_pos;
-
-	ASSERT(ERTS_MSEC_TO_MONOTONIC(to) <= timeout_time - curr_time);
-	ASSERT(timeout_time - curr_time
-	       < ERTS_MSEC_TO_MONOTONIC(to) + ERTS_CLKTCKS_TO_MONOTONIC(1));
     }
 
-    if (timeout_time < tiw->next_timeout_time)
-	set_next_timeout(tiw, timeout_time, 1);
-
-    set_timer_wheel(p, tiw);
-
-    erts_smp_mtx_unlock(&tiw->lock);
-
-#if defined(ERTS_SMP)
-    if (tiw == erts_default_timer_wheel)
-	erts_interupt_aux_thread_timed(timeout_time);
-#endif
-
+    if (timeout_time < tiw->next_timeout_time) {
+	tiw->true_next_timeout_time = 1;
+	tiw->next_timeout_time = timeout_time;
+    }
 }
 
 void
-erts_cancel_timer(ErlTimer *p)
+erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 {
-    ErtsTimerWheel *tiw;
-    ErlCancelProc cancel;
-    void *arg = NULL; /* Shut up faulty warning... */
-
-    tiw = get_timer_wheel(p);
-    if (!tiw)
-	return;
-    
-    erts_smp_mtx_lock(&tiw->lock);
-    if (tiw != get_timer_wheel(p))
-	cancel = NULL;
-    else {
+    if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) {
+	ErlCancelProc cancel;
+	void *arg;
 	remove_timer(tiw, p);
-
-	cancel = p->cancel;
-	arg = p->arg;
+	cancel = p->u.func.cancel;
+	arg = p->u.func.arg;
+	if (cancel)
+	    (*cancel)(arg);
     }
-    erts_smp_mtx_unlock(&tiw->lock);
-
-    if (cancel)
-	(*cancel)(arg);
 }
 
-/*
-  Returns the amount of time left in ms until the timer 'p' is triggered.
-  0 is returned if 'p' isn't active.
-  0 is returned also if the timer is overdue (i.e., would have triggered
-  immediately if it hadn't been cancelled).
-*/
-Uint
-erts_time_left(ErlTimer *p)
-{
-    ErtsTimerWheel *tiw;
-    ErtsMonotonicTime current_time, timeout_time;
-
-    tiw = get_timer_wheel(p);
-    if (!tiw)
-	return 0;
-
-    erts_smp_mtx_lock(&tiw->lock);
-    if (tiw != get_timer_wheel(p))
-	timeout_time = ERTS_MONOTONIC_TIME_MIN;
-    else
-	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos);
-    erts_smp_mtx_unlock(&tiw->lock);
-
-    current_time = erts_get_monotonic_time();
-    if (timeout_time <= current_time)
-	return 0;
-    return (Uint) ERTS_MONOTONIC_TO_MSEC(timeout_time - current_time);
-}
-
-#ifdef DEBUG
+#ifdef ERTS_TW_DEBUG
 void erts_p_slpq(void)
 {
-    ErtsTimerWheel *tiw = erts_default_timer_wheel;
-    ErtsMonotonicTime current_time = erts_get_monotonic_time();
+    erts_printf("Not yet implemented...\n");
+#if 0
+    ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL);
     int i;
-    ErlTimer* p;
+    ErtsTWheelTimer* p;
   
-    erts_smp_mtx_lock(&tiw->lock);
-
     /* print the whole wheel, starting at the current position */
     erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n",
 		current_time, tiw->pos, tiw->nto);
@@ -673,7 +655,7 @@ void erts_p_slpq(void)
 			p->slot);
 	}
     }
-    for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw->pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) {
+    for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) {
 	if (tiw->w[i] != NULL) {
 	    erts_printf("%d:\n", i);
 	    for(p = tiw->w[i]; p != NULL; p = p->next) {
@@ -682,7 +664,6 @@ void erts_p_slpq(void)
 	    }
 	}
     }
-
-    erts_smp_mtx_unlock(&tiw->lock);
+#endif
 }
-#endif /* DEBUG */
+#endif /* ERTS_TW_DEBUG */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index cb4ef2b376..4cbc4b0caf 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -50,6 +50,8 @@
 #include "erl_ptab.h"
 #include "erl_check_io.h"
 #include "erl_bif_unique.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
 #ifdef HIPE
 #  include "hipe_mode_switch.h"
 #endif
@@ -4397,145 +4399,6 @@ is_string(Eterm list)
     return 0;
 }
 
-#ifdef ERTS_SMP
-
-/*
- * Process and Port timers in smp case
- */
-
-ERTS_SCHED_PREF_PRE_ALLOC_IMPL(ptimer_pre, ErtsSmpPTimer, 1000)
-
-#define ERTS_PTMR_FLGS_ALLCD_SIZE \
-  2
-#define ERTS_PTMR_FLGS_ALLCD_MASK \
-  ((((Uint32) 1) << ERTS_PTMR_FLGS_ALLCD_SIZE) - 1)
-
-#define ERTS_PTMR_FLGS_PREALLCD	((Uint32) 1)
-#define ERTS_PTMR_FLGS_SLALLCD	((Uint32) 2)
-#define ERTS_PTMR_FLGS_LLALLCD	((Uint32) 3)
-#define ERTS_PTMR_FLG_CANCELLED	(((Uint32) 1) << (ERTS_PTMR_FLGS_ALLCD_SIZE+0))
-
-static void
-init_ptimers(void)
-{
-    init_ptimer_pre_alloc();
-}
-
-static ERTS_INLINE void
-free_ptimer(ErtsSmpPTimer *ptimer)
-{
-    switch (ptimer->timer.flags & ERTS_PTMR_FLGS_ALLCD_MASK) {
-    case ERTS_PTMR_FLGS_PREALLCD:
-	(void) ptimer_pre_free(ptimer);
-	break;
-    case ERTS_PTMR_FLGS_SLALLCD:
-	erts_free(ERTS_ALC_T_SL_PTIMER, (void *) ptimer);
-	break;
-    case ERTS_PTMR_FLGS_LLALLCD:
-	erts_free(ERTS_ALC_T_LL_PTIMER, (void *) ptimer);
-	break;
-    default:
-	erl_exit(ERTS_ABORT_EXIT,
-		 "Internal error: Bad ptimer alloc type\n");
-	break;
-    }
-}
-
-/* Callback for process timeout cancelled */
-static void
-ptimer_cancelled(ErtsSmpPTimer *ptimer)
-{
-    free_ptimer(ptimer);
-}
-
-/* Callback for process timeout */
-static void
-ptimer_timeout(ErtsSmpPTimer *ptimer)
-{
-    if (is_internal_pid(ptimer->timer.id)) {
-	Process *p;
-	p = erts_pid2proc_opt(NULL,
-			      0,
-			      ptimer->timer.id,
-			      ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
-			      ERTS_P2P_FLG_ALLOW_OTHER_X);
-	if (p) {
-	    if (!ERTS_PROC_IS_EXITING(p)
-		&& !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) {
-		ASSERT(*ptimer->timer.timer_ref == ptimer);
-		*ptimer->timer.timer_ref = NULL;
-		(*ptimer->timer.timeout_func)(p);
-	    }
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-	}
-    }
-    else {
-	Port *p;
-	ASSERT(is_internal_port(ptimer->timer.id));
-	p = erts_id2port_sflgs(ptimer->timer.id,
-			       NULL,
-			       0,
-			       ERTS_PORT_SFLGS_DEAD);
-	if (p) {
-	    if (!(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) {
-		ASSERT(*ptimer->timer.timer_ref == ptimer);
-		*ptimer->timer.timer_ref = NULL;
-		(*ptimer->timer.timeout_func)(p);
-	    }
-	    erts_port_release(p);
-	}
-    }
-    free_ptimer(ptimer);
-}
-
-void
-erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
-		       Eterm id,
-		       ErlTimeoutProc timeout_func,
-		       Uint timeout)
-{
-    ErtsSmpPTimer *res = ptimer_pre_alloc();
-    if (res)
-	res->timer.flags = ERTS_PTMR_FLGS_PREALLCD;
-    else {
-	if (timeout < ERTS_ALC_MIN_LONG_LIVED_TIME) {
-	    res = erts_alloc(ERTS_ALC_T_SL_PTIMER, sizeof(ErtsSmpPTimer));
-	    res->timer.flags = ERTS_PTMR_FLGS_SLALLCD;
-	}
-	else {
-	    res = erts_alloc(ERTS_ALC_T_LL_PTIMER, sizeof(ErtsSmpPTimer));
-	    res->timer.flags = ERTS_PTMR_FLGS_LLALLCD;
-	}
-    }
-    res->timer.timeout_func = timeout_func;
-    res->timer.timer_ref = timer_ref;
-    res->timer.id = id;
-    erts_init_timer(&res->timer.tm);
-
-    ASSERT(!*timer_ref);
-
-    *timer_ref = res;
-
-    erts_set_timer(&res->timer.tm,
-		  (ErlTimeoutProc) ptimer_timeout,
-		  (ErlCancelProc) ptimer_cancelled,
-		  (void*) res,
-		  timeout);
-}
-
-void
-erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer)
-{
-    if (ptimer) {
-	ASSERT(*ptimer->timer.timer_ref == ptimer);
-	*ptimer->timer.timer_ref = NULL;
-	ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED;
-	erts_cancel_timer(&ptimer->timer.tm);
-    }
-}
-
-#endif
-
 static int trim_threshold;
 static int top_pad;
 static int mmap_threshold;
@@ -4545,9 +4408,7 @@ Uint tot_bin_allocated;
 
 void erts_init_utils(void)
 {
-#ifdef ERTS_SMP
-    init_ptimers();
-#endif
+
 }
 
 void erts_init_utils_mem(void) 
-- 
cgit v1.2.3


From e7335bb1197294ef6f5fb3ad73bb22b9343603be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 8 May 2015 09:28:46 +0200
Subject: erts: Fix erts_debug:size/1 for large Maps

---
 erts/emulator/beam/erl_map.c | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index bb2a2bcdf9..256d6a8e81 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2586,12 +2586,12 @@ BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
     DECL_AM(hashmap);
     DECL_AM(hashmap_node);
     DECL_AM(flatmap);
-    if (is_flatmap(BIF_ARG_1)) {
-	BIF_RET(AM_flatmap);
-    } else if (is_hashmap(BIF_ARG_1)) {
+    if (is_map(BIF_ARG_1)) {
         Eterm hdr = *(boxed_val(BIF_ARG_1));
         ASSERT(is_header(hdr));
         switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_HEAD_FLATMAP:
+                BIF_RET(AM_flatmap);
             case HAMT_SUBTAG_HEAD_ARRAY:
             case HAMT_SUBTAG_HEAD_BITMAP:
                 BIF_RET(AM_hashmap);
@@ -2612,23 +2612,22 @@ BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
  */
 
 BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) {
-    if (is_hashmap(BIF_ARG_1)) {
+    if (is_map(BIF_ARG_1)) {
         Eterm node = BIF_ARG_1;
         Eterm *ptr, hdr, *hp, res = NIL;
         Uint  sz = 0;
         ptr = boxed_val(node);
         hdr = *ptr;
-
         ASSERT(is_header(hdr));
 
         switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-            case HAMT_SUBTAG_NODE_BITMAP:
-                sz   = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-                ptr += 1;
-                break;
+            case HAMT_SUBTAG_HEAD_FLATMAP:
+                BIF_ERROR(BIF_P, BADARG);
             case HAMT_SUBTAG_HEAD_BITMAP:
-                sz   = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-                ptr += 2;
+                ptr++;
+            case HAMT_SUBTAG_NODE_BITMAP:
+                ptr++;
+                sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
                 break;
             case HAMT_SUBTAG_HEAD_ARRAY:
                 sz   = 16;
@@ -2642,12 +2641,9 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) {
         hp = HAlloc(BIF_P, 2*sz);
         while(sz--) { res = CONS(hp, *ptr++, res); hp += 2; }
         BIF_RET(res);
-    } else if (is_flatmap(BIF_ARG_1)) {
-	BIF_ERROR(BIF_P, BADARG);
-    } else {
-	BIF_P->fvalue = BIF_ARG_1;
-	BIF_ERROR(BIF_P, BADMAP);
     }
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, BADMAP);
 }
 
 
-- 
cgit v1.2.3


From 2dc1ca3bc664177a7d985c3e190464285e9b75fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@plataformatec.com.br>
Date: Fri, 9 May 2014 17:08:43 +0200
Subject: Add erts_send_error_term_to_logger

This function allows us to send format and args to the
logger which can then be formatted and customized from
Erlang land.
---
 erts/emulator/beam/sys.h   |   1 +
 erts/emulator/beam/utils.c | 190 +++++++++++++++++++++++++++++++--------------
 2 files changed, 132 insertions(+), 59 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 54059ee9a5..cd53069872 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -657,6 +657,7 @@ erts_dsprintf_buf_t *erts_create_logger_dsbuf(void);
 int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *);
 int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *);
 int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *);
+int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm);
 int erts_send_info_to_logger_str(Eterm, char *); 
 int erts_send_warning_to_logger_str(Eterm, char *);
 int erts_send_error_to_logger_str(Eterm, char *);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index e21e916de5..e3034c1e24 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2222,97 +2222,157 @@ tail_recur:
 #undef MAKE_HASH_CDR_POST_OP
 }
 
-static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
+static Eterm
+do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
+			   ErlHeapFragment **bp, Process **p, Uint sz)
 {
-    /* error_logger ! 
-       {notify,{info_msg,gleader,{emulator,"~s~n",[<message as list>]}}} |
-       {notify,{error,gleader,{emulator,"~s~n",[<message as list>]}}} |
-       {notify,{warning_msg,gleader,{emulator,"~s~n",[<message as list>}]}} */
-    Eterm* hp;
-    Uint sz;
     Uint gl_sz;
-    Eterm gl;
-    Eterm list,plist,format,tuple1,tuple2,tuple3;
-    ErlOffHeap *ohp;
-    ErlHeapFragment *bp = NULL;
-#if !defined(ERTS_SMP)
-    Process *p;
-#endif
-
-    ASSERT(is_atom(tag));
-
-    if (len <= 0) {
-	return -1;
-    }
+    gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader);
+    sz = sz + gl_sz;
 
 #ifndef ERTS_SMP
 #ifdef USE_THREADS
-    p = NULL;
     if (erts_get_scheduler_data()) /* Must be scheduler thread */
 #endif
     {
-	p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
-	if (p) {
-	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+	*p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
+	if (*p) {
+	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&(*p)->state);
 	    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))
-		p = NULL;
+		*p = NULL;
 	}
     }
 
-    if (!p) {
-	/* buf *always* points to a null terminated string */
-	erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n",
-		     tag, buf);
-	return 0;
+    if (!*p) {
+	return NIL;
     }
-    /* So we have an error logger, lets build the message */
-#endif
-    gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader);
-    sz = len * 2 /* message list */+ 2 /* cons surrounding message list */
-	+ gl_sz + 
-	3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */ +
-	8 /* "~s~n" */;
 
-#ifndef ERTS_SMP
-    if (sz <= HeapWordsLeft(p)) {
-	ohp = &MSO(p);
-	hp = HEAP_TOP(p);
-	HEAP_TOP(p) += sz;
+    /* So we have an error logger, lets build the message */
+    if (sz <= HeapWordsLeft(*p)) {
+	*ohp = &MSO(*p);
+	*hp = HEAP_TOP(*p);
+	HEAP_TOP(*p) += sz;
     } else {
 #endif
-	bp = new_message_buffer(sz);
-	ohp = &bp->off_heap;
-	hp = bp->mem;
+	*bp = new_message_buffer(sz);
+	*ohp = &(*bp)->off_heap;
+	*hp = (*bp)->mem;
 #ifndef ERTS_SMP
     }
 #endif
-    gl = (is_nil(gleader)
+
+    return (is_nil(gleader)
 	  ? am_noproc
 	  : (IS_CONST(gleader)
 	     ? gleader
-	     : copy_struct(gleader,gl_sz,&hp,ohp)));
-    list = buf_to_intlist(&hp, buf, len, NIL);
-    plist = CONS(hp,list,NIL);
-    hp += 2;
-    format = buf_to_intlist(&hp, "~s~n", 4, NIL);
-    tuple1 = TUPLE3(hp, am_emulator, format, plist);
-    hp += 4;
-    tuple2 = TUPLE3(hp, tag, gl, tuple1);
-    hp += 4;
-    tuple3 = TUPLE2(hp, am_notify, tuple2);
+	     : copy_struct(gleader,gl_sz,hp,*ohp)));
+}
+
+static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *bp,
+				   Process *p, Eterm message)
+{
 #ifdef HARDDEBUG
-    erts_fprintf(stderr, "%T\n", tuple3);
+    erts_fprintf(stderr, "%T\n", message);
 #endif
 #ifdef ERTS_SMP
     {
 	Eterm from = erts_get_current_pid();
 	if (is_not_internal_pid(from))
 	    from = NIL;
-	erts_queue_error_logger_message(from, tuple3, bp);
+	erts_queue_error_logger_message(from, message, bp);
     }
 #else
-    erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL);
+    erts_queue_message(p, NULL /* only used for smp build */, bp, message, NIL);
 #endif
+}
+
+/* error_logger !
+   {notify,{info_msg,gleader,{emulator,format,[args]}}} |
+   {notify,{error,gleader,{emulator,format,[args]}}} |
+   {notify,{warning_msg,gleader,{emulator,format,[args}]}} */
+static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
+{
+    Uint sz;
+    Eterm gl;
+    Eterm list,args,format,tuple1,tuple2,tuple3;
+
+    Eterm *hp = NULL;
+    ErlOffHeap *ohp = NULL;
+    ErlHeapFragment *bp = NULL;
+    Process *p = NULL;
+
+    ASSERT(is_atom(tag));
+
+    if (len <= 0) {
+	return -1;
+    }
+
+    sz = len * 2 /* message list */ + 2 /* cons surrounding message list */
+	+ 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */
+	+ 8 /* "~s~n" */;
+
+    /* gleader size is accounted and allocated next */
+    gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz);
+
+    if(is_nil(gl)) {
+       /* buf *always* points to a null terminated string */
+       erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n",
+                    tag, buf);
+       return 0;
+    }
+
+    list = buf_to_intlist(&hp, buf, len, NIL);
+    args = CONS(hp,list,NIL);
+    hp += 2;
+    format = buf_to_intlist(&hp, "~s~n", 4, NIL);
+    tuple1 = TUPLE3(hp, am_emulator, format, args);
+    hp += 4;
+    tuple2 = TUPLE3(hp, tag, gl, tuple1);
+    hp += 4;
+    tuple3 = TUPLE2(hp, am_notify, tuple2);
+
+    do_send_logger_message(hp, ohp, bp, p, tuple3);
+    return 0;
+}
+
+static int do_send_term_to_logger(Eterm tag, Eterm gleader,
+				  char *buf, int len, Eterm args)
+{
+    Uint sz;
+    Eterm gl;
+    Uint args_sz;
+    Eterm format,tuple1,tuple2,tuple3;
+
+    Eterm *hp = NULL;
+    ErlOffHeap *ohp = NULL;
+    ErlHeapFragment *bp = NULL;
+    Process *p = NULL;
+
+    ASSERT(is_atom(tag));
+
+    args_sz = size_object(args);
+    sz = len * 2 /* format */ + args_sz
+	+ 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */;
+
+    /* gleader size is accounted and allocated next */
+    gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz);
+
+    if(is_nil(gl)) {
+       /* buf *always* points to a null terminated string */
+       erts_fprintf(stderr, "(no error logger present) %T: \"%s\" %T\n",
+                    tag, buf, args);
+       return 0;
+    }
+
+    format = buf_to_intlist(&hp, buf, len, NIL);
+    args = copy_struct(args, args_sz, &hp, ohp);
+    tuple1 = TUPLE3(hp, am_emulator, format, args);
+    hp += 4;
+    tuple2 = TUPLE3(hp, tag, gl, tuple1);
+    hp += 4;
+    tuple3 = TUPLE2(hp, am_notify, tuple2);
+
+    do_send_logger_message(hp, ohp, bp, p, tuple3);
     return 0;
 }
 
@@ -2340,6 +2400,12 @@ send_error_to_logger(Eterm gleader, char *buf, int len)
     return do_send_to_logger(am_error, gleader, buf, len);
 }
 
+static ERTS_INLINE int
+send_error_term_to_logger(Eterm gleader, char *buf, int len, Eterm args)
+{
+    return do_send_term_to_logger(am_error, gleader, buf, len, args);
+}
+
 #define LOGGER_DSBUF_INC_SZ 256
 
 static erts_dsprintf_buf_t *
@@ -2414,6 +2480,12 @@ erts_send_error_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp)
     return res;
 }
 
+int
+erts_send_error_term_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp, Eterm args)
+{
+    return send_error_term_to_logger(gleader, dsbufp->str, dsbufp->str_len, args);
+}
+
 int
 erts_send_info_to_logger_str(Eterm gleader, char *str)
 {
-- 
cgit v1.2.3


From 42d25222fa7395ad9fb58de2c964afa4caefb5b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@plataformatec.com.br>
Date: Fri, 9 May 2014 18:07:07 +0200
Subject: Send format and args on process exit to error_logger

Previously, the emulator would generate a whole string
with values and call the error_logger passing "~s~n".
This commit changes it to a format string containing ~p
with the respective values as arguments.
---
 erts/emulator/beam/beam_emu.c | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9fe02c3724..a21622f424 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5577,18 +5577,35 @@ next_catch(Process* c_p, Eterm *reg) {
 static void
 terminate_proc(Process* c_p, Eterm Value)
 {
+    Eterm *hp;
+    Eterm Args = NIL;
+
     /* Add a stacktrace if this is an error. */
     if (GET_EXC_CLASS(c_p->freason) == EXTAG_ERROR) {
         Value = add_stacktrace(c_p, Value, c_p->ftrace);
     }
     /* EXF_LOG is a primary exception flag */
     if (c_p->freason & EXF_LOG) {
+	int alive = erts_is_alive;
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	erts_dsprintf(dsbufp, "Error in process %T ", c_p->common.id);
-	if (erts_is_alive)
-	    erts_dsprintf(dsbufp, "on node %T ", erts_this_node->sysname);
-	erts_dsprintf(dsbufp,"with exit value: %0.*T\n", display_items, Value);
-	erts_send_error_to_logger(c_p->group_leader, dsbufp);
+
+        /* Build the format message */
+	erts_dsprintf(dsbufp, "Error in process ~p ");
+	if (alive)
+	    erts_dsprintf(dsbufp, "on node ~p ");
+	erts_dsprintf(dsbufp, "with exit value:~n~p~n");
+
+        /* Build the args in reverse order */
+	hp = HAlloc(c_p, 2);
+	Args = CONS(hp, Value, Args);
+	if (alive) {
+	    hp = HAlloc(c_p, 2);
+	    Args = CONS(hp, erts_this_node->sysname, Args);
+	}
+	hp = HAlloc(c_p, 2);
+	Args = CONS(hp, c_p->common.id, Args);
+
+	erts_send_error_term_to_logger(c_p->group_leader, dsbufp, Args);
     }
     /*
      * If we use a shared heap, the process will be garbage-collected.
-- 
cgit v1.2.3


From 5917fd5468a952fce7ebda8b57a6218fbb7bb878 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 12 May 2015 11:55:34 +0200
Subject: erts: Fix erts_send_error_term_to_logger memory leak

---
 erts/emulator/beam/utils.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index e3034c1e24..965de748c9 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2483,7 +2483,10 @@ erts_send_error_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp)
 int
 erts_send_error_term_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp, Eterm args)
 {
-    return send_error_term_to_logger(gleader, dsbufp->str, dsbufp->str_len, args);
+    int res;
+    res = send_error_term_to_logger(gleader, dsbufp->str, dsbufp->str_len, args);
+    destroy_logger_dsbuf(dsbufp);
+    return res;
 }
 
 int
-- 
cgit v1.2.3


From 849a2ed2db2bd54422ec9284468f80cc86978974 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 11 May 2015 17:56:03 +0200
Subject: Timer fixes, documentation, and test cases

---
 erts/emulator/beam/erl_alloc.c    | 16 ++++++++++
 erts/emulator/beam/erl_hl_timer.c | 66 ++++++++++++++++++++++-----------------
 erts/emulator/beam/time.c         | 60 +++++++++++++++--------------------
 3 files changed, 79 insertions(+), 63 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 71902c2f9f..7d099997d8 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2331,6 +2331,22 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_MSG_REF);
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_LL_PTIMER);
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_HL_PTIMER);
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_BIF_TIMER);
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_ABIF_TIMER);
     }
 
     if (want.atom || want.atom_used) {
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 2245799819..f1806d25be 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -850,8 +850,6 @@ hl_timer_destroy(ErtsHLTimer *tmr)
     if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR))
 	erts_free(ERTS_ALC_T_HL_PTIMER, tmr);
     else {
-	if (tmr->btm.bp)
-	    free_message_buffer(tmr->btm.bp);
 	if (roflgs & ERTS_TMR_ROFLG_PRE_ALC)
 	    bif_timer_pre_free(tmr);
 	else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
@@ -898,10 +896,6 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
 	 * Message buffer can be dropped at
 	 * once...
 	 */
-	if (tmr->btm.bp) {
-	    free_message_buffer(tmr->btm.bp);
-	    tmr->btm.bp = NULL;
-	}
 	size = sizeof(ErtsHLTimer);
     }
 
@@ -1121,6 +1115,7 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
 {
     ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND;
     Process *proc;
+    int queued_message = 0;
     int dec_refc = 0;
     Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
@@ -1159,6 +1154,7 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
 #endif
 		);
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND);
+	    queued_message = 1;
 	    proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND;
 	    tmr->btm.bp = NULL;
 	    if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
@@ -1172,6 +1168,8 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
 	if (dec_refc)
 	    hl_timer_pre_dec_refc(tmr);
     }
+    if (!queued_message && tmr->btm.bp)
+	free_message_buffer(tmr->btm.bp);
 }
 
 static ERTS_INLINE void
@@ -1689,6 +1687,8 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
 					  rcvr, ERTS_PROC_LOCK_BTM,
 					  ERTS_P2P_FLG_INC_REFC);
 	if (!proc) {
+	    if (tmr->btm.bp)
+		free_message_buffer(tmr->btm.bp);
 	    hlt_delete_timer(esdp, tmr);
 	    hl_timer_destroy(tmr);
 	}
@@ -1721,6 +1721,9 @@ cancel_bif_timer(ErtsHLTimer *tmr)
     if (state != ERTS_TMR_STATE_ACTIVE)
 	return 0;
 
+    if (tmr->btm.bp)
+	free_message_buffer(tmr->btm.bp);
+
     res = -1;
 
     roflgs = tmr->head.roflgs;
@@ -1834,18 +1837,25 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 	if (!async)
 	    hsz += REF_THING_SIZE;
 	else {
-	    if (is_non_value(tref))
+	    if (is_non_value(tref) || proc != c_p)
 		hsz += REF_THING_SIZE;
 	    hsz += 1; /* upgrade to 3-tuple */
 	}
 	if (time_left > (Sint64) MAX_SMALL)
 	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
 
-	hp = erts_alloc_message_heap(hsz,
-				     &bp,
-				     &ohp,
-				     proc,
-				     &proc_locks);
+	if (proc == c_p) {
+	    bp = NULL;
+	    ohp = NULL;
+	    hp = HAlloc(c_p, hsz);
+	}
+	else {
+	    hp = erts_alloc_message_heap(hsz,
+					 &bp,
+					 &ohp,
+					 proc,
+					 &proc_locks);
+	}
 
 #ifdef ERTS_HLT_DEBUG
 	hp_end = hp + hsz;
@@ -1871,7 +1881,7 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 	}
 	else {
 	    Eterm tag = cancel ? am_cancel_timer : am_read_timer;
-	    if (is_value(tref))
+	    if (is_value(tref) && proc == c_p)
 		ref = tref;
 	    else {
 		write_ref_thing(hp,
@@ -1987,8 +1997,6 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
     }
     else {
 	Eterm tag, res, msg;
-	ErlOffHeap *ohp;
-	ErlHeapFragment* bp;
 	Uint hsz;
 	Eterm *hp;
 	ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
@@ -1997,11 +2005,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 	if (time_left > (Sint64) MAX_SMALL)
 	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
 
-	hp = erts_alloc_message_heap(hsz,
-				     &bp,
-				     &ohp,
-				     c_p,
-				     &proc_locks);
+	hp = HAlloc(c_p, hsz);
 	if (cancel)
 	    tag = am_cancel_timer;
 	else
@@ -2016,7 +2020,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 
 	msg = TUPLE3(hp, tag, tref, res);
 
-	erts_queue_message(c_p, &proc_locks, bp,
+	erts_queue_message(c_p, &proc_locks, NULL,
 			   msg, NIL
 #ifdef USE_VM_PROBES
 			   , NIL
@@ -2166,7 +2170,7 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info,
     if (async)
 	*async = 0;
     if (info)
-	*info = 0;
+	*info = 1;
     if (abs)
 	*abs = 0;
     if (accessor)
@@ -2235,13 +2239,19 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp)
     tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 
     if (sid == (Uint32) esdp->no) {
-	if (state == ERTS_TMR_STATE_ACTIVE)
+	if (state == ERTS_TMR_STATE_ACTIVE) {
+	    if (tmr->btm.bp)
+		free_message_buffer(tmr->btm.bp);
 	    hlt_delete_timer(esdp, tmr);
+	}
 	hl_timer_dec_refc(tmr, roflgs);
     }
     else {
-	if (state == ERTS_TMR_STATE_ACTIVE)
+	if (state == ERTS_TMR_STATE_ACTIVE) {
+	    if (tmr->btm.bp)
+		free_message_buffer(tmr->btm.bp);
 	    queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
+	}
 	else
 	    hl_timer_dec_refc(tmr, roflgs);
     }
@@ -2415,7 +2425,7 @@ BIF_RETTYPE send_after_3(BIF_ALIST_3)
     tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
 			     0, &timeout_pos, &short_time);
     if (tres != 0)
-	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+	BIF_ERROR(BIF_P, BADARG);
 
     return setup_bif_timer(BIF_P, timeout_pos, short_time,
 			   BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0);
@@ -2433,7 +2443,7 @@ BIF_RETTYPE send_after_4(BIF_ALIST_4)
     tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
 			     abs, &timeout_pos, &short_time);
     if (tres != 0)
-	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+	BIF_ERROR(BIF_P, BADARG);
 
     return setup_bif_timer(BIF_P, timeout_pos, short_time,
 			   BIF_ARG_2, accessor, BIF_ARG_3, 0);
@@ -2447,7 +2457,7 @@ BIF_RETTYPE start_timer_3(BIF_ALIST_3)
     tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
 			     0, &timeout_pos, &short_time);
     if (tres != 0)
-	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+	BIF_ERROR(BIF_P, BADARG);
 
     return setup_bif_timer(BIF_P, timeout_pos, short_time,
 			   BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0);
@@ -2465,7 +2475,7 @@ BIF_RETTYPE start_timer_4(BIF_ALIST_4)
     tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
 			     abs, &timeout_pos, &short_time);
     if (tres != 0)
-	BIF_ERROR(BIF_P, tres < 0 ? BADARG : SYSTEM_LIMIT);
+	BIF_ERROR(BIF_P, BADARG);
 
     return setup_bif_timer(BIF_P, timeout_pos, short_time,
 			   BIF_ARG_2, accessor, BIF_ARG_3, !0);
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index ea19d8b362..8bffdedb2b 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -135,39 +135,40 @@ struct ErtsTimerWheel_ {
     ErtsMonotonicTime next_timeout_time;
 };
 
-/* get the time (in units of TIW_ITIME) to the next timeout,
-   or -1 if there are no timeouts                     */
-
 static ERTS_INLINE ErtsMonotonicTime
-find_next_timeout(ErtsTimerWheel *tiw,
-		  ErtsMonotonicTime curr_time,
-		  ErtsMonotonicTime max_search_time)
+find_next_timeout(ErtsSchedulerData *esdp,
+		  ErtsTimerWheel *tiw,
+		  int search_all,
+		  ErtsMonotonicTime curr_time,       /* When !search_all */
+		  ErtsMonotonicTime max_search_time) /* When !search_all */
 {
     int start_ix, tiw_pos_ix;
     ErtsTWheelTimer *p;
-    int true_min_timeout;
+    int true_min_timeout = 0;
     ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos;
 
-    if (tiw->true_next_timeout_time)
-	return tiw->next_timeout_time;
-
     if (tiw->nto == 0) { /* no timeouts in wheel */
-	true_min_timeout = 0;
-	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + ERTS_MONOTONIC_DAY);
+	if (!search_all)
+	    min_timeout_pos = tiw->pos;
+	else {
+	    curr_time = erts_get_monotonic_time(esdp);
+	    tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
+	}
+	min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY);
 	goto found_next;
     }
 
-    slot_timeout_pos = tiw->pos;
-    min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
+    slot_timeout_pos = min_timeout_pos = tiw->pos;
+    if (search_all)
+	min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY);
+    else
+	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
 
     start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1));
 
     do {
-	slot_timeout_pos++;
-	if (slot_timeout_pos >= min_timeout_pos) {
-	    true_min_timeout = 0;
+	if (++slot_timeout_pos >= min_timeout_pos)
 	    break;
-	}
 
 	p = tiw->w[tiw_pos_ix];
 
@@ -269,24 +270,16 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 
     p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
 
-#if 0
-    p->next = NULL;
-    p->prev = NULL;
-#endif
-
     tiw->nto--;
 }
 
 ErtsMonotonicTime
 erts_check_next_timeout_time(ErtsSchedulerData *esdp)
 {
-    /*
-     * Called before a scheduler is about to wait. We wont
-     * check more than 10 minutes into the future.
-     */
-    return find_next_timeout(esdp->timer_wheel,
-			     erts_get_monotonic_time(esdp),
-			     ERTS_SEC_TO_MONOTONIC(10*60));
+    ErtsTimerWheel *tiw = esdp->timer_wheel;
+    if (tiw->true_next_timeout_time)
+	return tiw->next_timeout_time;
+    return find_next_timeout(esdp, tiw, 1, 0, 0);
 }
 
 #ifndef ERTS_TW_DEBUG
@@ -330,14 +323,11 @@ timeout_timer(ErtsTWheelTimer *p)
 {
     ErlTimeoutProc timeout;
     void *arg;
-#if 0
-    p->next = NULL;
-    p->prev = NULL;
-#endif
     p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
     timeout = p->u.func.timeout;
     arg = p->u.func.arg;
     (*timeout)(arg);
+    ASSERT_NO_LOCKED_LOCKS;
 }
 
 void
@@ -508,7 +498,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
     tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY;
 
     /* Search at most two seconds ahead... */
-    (void) find_next_timeout(tiw, curr_time, ERTS_SEC_TO_MONOTONIC(2));
+    (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2));
 }
 
 Uint
-- 
cgit v1.2.3


From cce681109c05ffa8cb88e83eb70ccf73c13d0c75 Mon Sep 17 00:00:00 2001
From: Jesper Louis Andersen <jesper.louis.andersen@gmail.com>
Date: Tue, 12 May 2015 12:44:14 +0200
Subject: Correct usage of sizeof() for pointer types

Given some pointer *x, calling sizeof(x) will give us
the size of the pointer (4/8 bytes) not the size fo the
underlying dereferenced structure. Use coccinelle to
search for these occurrences in the source code, and
correct them one by one. In the case of
erl_node_tables.c, the erts_snprintf() calls used a
much too small buffer.

- run_erl.c: Use the size of the signal type, not
  its pointer.
- erl_node_tables.c: Use the size of the _BUFFER in
  erts_snprintf() to make sure we can use the full space.
---
 erts/emulator/beam/erl_node_tables.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index c6d136f951..bc775cd663 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -705,7 +705,7 @@ erts_set_this_node(Eterm sysname, Uint creation)
     erts_this_node->sysname = sysname;
     erts_this_node->creation = creation;
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
-    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname),
+    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
                   "%T", sysname);
     (void) hash_put(&erts_node_table, (void *) erts_this_node);
 
@@ -794,7 +794,7 @@ void erts_init_node_tables(void)
     erts_this_node->creation			= 0;
     erts_this_node->dist_entry			= erts_this_dist_entry;
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
-    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname),
+    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
                   "%T", erts_this_node->sysname);
 
     (void) hash_put(&erts_node_table, (void *) erts_this_node);
-- 
cgit v1.2.3


From b7c9657a141dd002a08256ba272cb125fec2b16d Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Sun, 17 May 2015 17:37:13 -0400
Subject: Avoid timer wheel work in dirty schedulers

Dirty schedulers are limited in the type of work they can
perform. Make sure they do not participate in any work or make any
function calls related to timer wheels.
---
 erts/emulator/beam/erl_process.c | 149 ++++++++++++++++++++++++---------------
 1 file changed, 92 insertions(+), 57 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index af8db519d4..c5bfcd3fb4 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2887,22 +2887,29 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    }
 
 	    if (aux_work) {
-		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
-		current_time = erts_get_monotonic_time(esdp);
-		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			sched_wall_time_change(esdp, 1);
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+		    current_time = erts_get_monotonic_time(esdp);
+		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
+			}
+			erts_bump_timers(esdp->timer_wheel, current_time);
 		    }
-		    erts_bump_timers(esdp->timer_wheel, current_time);
 		}
 	    }
 	    else {
 		ErtsMonotonicTime timeout_time;
-		timeout_time = erts_check_next_timeout_time(esdp);
-		current_time = erts_get_monotonic_time(esdp);
-		if (current_time >= timeout_time) {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+		int do_timeout = 0;
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		    timeout_time = erts_check_next_timeout_time(esdp);
+		    current_time = erts_get_monotonic_time(esdp);
+		    do_timeout = (current_time >= timeout_time);
+		} else
+		    timeout_time = ERTS_MONOTONIC_TIME_MAX;
+		if (do_timeout) {
+		    if (!thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			sched_wall_time_change(esdp, 1);
 		    }
@@ -2926,23 +2933,28 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 			    int res;
 			    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 			    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			    current_time = erts_get_monotonic_time(esdp);
+			    current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
+				erts_get_monotonic_time(esdp);
 			    do {
 				Sint64 timeout;
 				if (current_time >= timeout_time)
 				    break;
-				timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
-								 - current_time
-								 - 1) + 1;
+				if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+				    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+								     - current_time
+								     - 1) + 1;
+				} else
+				    timeout = -1;
 				res = erts_tse_twait(ssi->event, timeout);
-				current_time = erts_get_monotonic_time(esdp);
+				current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
+				    erts_get_monotonic_time(esdp);
 			    } while (res == EINTR);
 			}
 		    }
 		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
 			erts_thr_progress_finalize_wait(esdp);
 		}
-		if (current_time >= timeout_time)
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time)
 		    erts_bump_timers(esdp->timer_wheel, current_time);
 	    }
 
@@ -3010,9 +3022,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
 	    erl_sys_schedule(1); /* Might give us something to do */
 
-	    current_time = erts_get_monotonic_time(esdp);
-	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
-		erts_bump_timers(esdp->timer_wheel, current_time);
+	    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		current_time = erts_get_monotonic_time(esdp);
+		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		    erts_bump_timers(esdp->timer_wheel, current_time);
+	    }
 
 	sys_aux_work:
 #ifndef ERTS_SMP
@@ -3021,15 +3035,18 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
-		if (!working)
-		    sched_wall_time_change(esdp, working = 1);
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		    if (!working)
+			sched_wall_time_change(esdp, working = 1);
 #ifdef ERTS_SMP
-		if (!thr_prgr_active)
-		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+		    if (!thr_prgr_active)
+			erts_thr_progress_active(esdp, thr_prgr_active = 1);
 #endif
+		}
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
 #ifdef ERTS_SMP
-		if (aux_work && erts_thr_progress_update(esdp))
+		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work &&
+		    erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
 #endif
 	    }
@@ -3127,7 +3144,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	erl_sys_schedule(0);
 
-	{
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
@@ -6790,7 +6807,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	}
     }
 
-    (void) erts_get_monotonic_time(esdp);
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+	(void) erts_get_monotonic_time(esdp);
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
@@ -6906,7 +6924,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			 & ERTS_RUNQ_FLGS_QMASK);
 		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 		if (aux_work|qmask) {
-		    if (!thr_prgr_active) {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			sched_wall_time_change(esdp, 1);
 		    }
@@ -6914,7 +6932,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			aux_work = handle_aux_work(&esdp->aux_work_data,
 						   aux_work,
 						   1);
-		    if (aux_work && erts_thr_progress_update(esdp))
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work &&
+			erts_thr_progress_update(esdp))
 			erts_thr_progress_leader_update(esdp);
 		    if (qmask) {
 			erts_smp_runq_lock(esdp->run_queue);
@@ -6924,32 +6943,40 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		}
 
 		if (aux_work) {
-		    current_time = erts_get_monotonic_time(esdp);
-		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-			if (!thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			    sched_wall_time_change(esdp, 1);
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			current_time = erts_get_monotonic_time(esdp);
+			if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+			    if (!thr_prgr_active) {
+				erts_thr_progress_active(esdp, thr_prgr_active = 1);
+				sched_wall_time_change(esdp, 1);
+			    }
+			    erts_bump_timers(esdp->timer_wheel, current_time);
 			}
-			erts_bump_timers(esdp->timer_wheel, current_time);
 		    }
 		}
 		else {
 		    ErtsMonotonicTime timeout_time;
-		    timeout_time = erts_check_next_timeout_time(esdp);
-		    current_time = erts_get_monotonic_time(esdp);
-
-		    if (current_time >= timeout_time) {
+		    int do_timeout = 0;
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			timeout_time = erts_check_next_timeout_time(esdp);
+			current_time = erts_get_monotonic_time(esdp);
+			do_timeout = (current_time >= timeout_time);
+		    } else
+			timeout_time = ERTS_MONOTONIC_TIME_MAX;
+		    if (do_timeout) {
 			if (!thr_prgr_active) {
 			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			    sched_wall_time_change(esdp, 1);
 			}
 		    }
-		    else {		    
-			if (thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			    sched_wall_time_change(esdp, 0);
+		    else {
+			if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			    if (thr_prgr_active) {
+				erts_thr_progress_active(esdp, thr_prgr_active = 0);
+				sched_wall_time_change(esdp, 0);
+			    }
+			    erts_thr_progress_prepare_wait(esdp);
 			}
-			erts_thr_progress_prepare_wait(esdp);
 			flgs = sched_spin_suspended(ssi,
 						    ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
 			if (flgs == (ERTS_SSI_FLG_SLEEPING
@@ -6962,23 +6989,29 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 					 | ERTS_SSI_FLG_SUSPENDED)) {
 				int res;
 
-				current_time = erts_get_monotonic_time(esdp);
+				current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
+				    erts_get_monotonic_time(esdp);
 				do {
 				    Sint64 timeout;
 				    if (current_time >= timeout_time)
 					break;
-				    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
-								     - current_time
-								     - 1) + 1;
+				    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+					timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+									 - current_time
+									 - 1) + 1;
+				    } else
+					timeout = -1;
 				    res = erts_tse_twait(ssi->event, timeout);
-				    current_time = erts_get_monotonic_time(esdp);
+				    current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
+					erts_get_monotonic_time(esdp);
 				} while (res == EINTR);
 			    }
 			}
-			erts_thr_progress_finalize_wait(esdp);
+			if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+			    erts_thr_progress_finalize_wait(esdp);
 		    }
 
-		    if (current_time >= timeout_time)
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time)
 			erts_bump_timers(esdp->timer_wheel, current_time);
 		}
 
@@ -9196,13 +9229,15 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-	if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
-	    (void) erts_get_monotonic_time(esdp);
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	    if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
+		(void) erts_get_monotonic_time(esdp);
 
-	if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-	    erts_smp_runq_unlock(rq);
-	    erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time);
-	    erts_smp_runq_lock(rq);
+	    if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+		erts_smp_runq_unlock(rq);
+		erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time);
+		erts_smp_runq_lock(rq);
+	    }
 	}
 	BM_STOP_TIMER(system);
 
-- 
cgit v1.2.3


From e5899f39d043409e2a48f34c845ad28cad8a28e6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 May 2015 19:25:09 +0200
Subject: erts: Fix warning about const pointer to make_boxed and make_list

---
 erts/emulator/beam/erl_term.c | 4 ++--
 erts/emulator/beam/erl_term.h | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index bc04d7b78e..565528193e 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -128,10 +128,10 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \
     return _unchecked_##FUN(x); \
 }
 
-ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer);
+ET_DEFINE_CHECKED(Eterm,make_boxed,const Eterm*,_is_taggable_pointer);
 ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header);
 ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond);
-ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer);
+ET_DEFINE_CHECKED(Eterm,make_list,const Eterm*,_is_taggable_pointer);
 ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header);
 ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond);
 ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 602aab46dc..7b15b34da1 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -198,7 +198,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #endif
 #define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
 #define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED)
-_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*)
+_ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*)
 #define make_boxed(x)		_ET_APPLY(make_boxed,(x))
 #if 1
 #define _is_not_boxed(x)	((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED))
@@ -214,7 +214,7 @@ _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
 
 /* cons cell ("list") access methods */
 #define _unchecked_make_list(x)	((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST)
-_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*)
+_ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*)
 #define make_list(x)		_ET_APPLY(make_list,(x))
 #if 1
 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST))
-- 
cgit v1.2.3


From 56c9154f2a0efb8a6d1347b19b797ac17b6608d4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 May 2015 19:34:45 +0200
Subject: erts: Fix calculation of reclaimed data during full gc

The old code did not take take the old-heap into acount.
---
 erts/emulator/beam/erl_gc.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 1785fc27be..14e7dde778 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1223,7 +1223,8 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     Uint new_sz;
     Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
 
-    size_before = fragments + (HEAP_TOP(p) - HEAP_START(p));
+    size_before = fragments + (HEAP_TOP(p) - HEAP_START(p))
+                            + (OLD_HTOP(p) - OLD_HEAP(p));
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
-- 
cgit v1.2.3


From 441842ce023bf8ef5dc84f2d5061b0b7c79c8130 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Fri, 17 Apr 2015 22:04:31 +0200
Subject: Map error logger warnings to warning messages by default

Also fix and document the broken +We option.
---
 erts/emulator/beam/erl_init.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 86d3416423..a905cbf81e 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -622,7 +622,7 @@ void erts_usage(void)
 
     erts_fprintf(stderr, "-v             turn on chatty mode (GCs will be reported etc)\n");
 
-    erts_fprintf(stderr, "-W<i|w>        set error logger warnings mapping,\n");
+    erts_fprintf(stderr, "-W<i|w|e>      set error logger warnings mapping,\n");
     erts_fprintf(stderr, "               see error_logger documentation for details\n");
     erts_fprintf(stderr, "-zdbbl size    set the distribution buffer busy limit in kilobytes\n");
     erts_fprintf(stderr, "               valid range is [1-%d]\n", INT_MAX/1024);
@@ -1250,7 +1250,7 @@ erl_start(int argc, char **argv)
     verbose = DEBUG_DEFAULT;
 #endif
 
-    erts_error_logger_warnings = am_error;
+    erts_error_logger_warnings = am_warning;
 
     while (i < argc) {
 	if (argv[i][0] != '-') {
@@ -1993,11 +1993,12 @@ erl_start(int argc, char **argv)
 	    case 'i':
 		erts_error_logger_warnings = am_info;
 		break;
+	    case 'e':
+		erts_error_logger_warnings = am_error;
+		break;
 	    case 'w':
 		erts_error_logger_warnings = am_warning;
 		break;
-	    case 'e': /* The default */
-		erts_error_logger_warnings = am_error;
 	    default:
 		erts_fprintf(stderr, "unrecognized warning_map option %s\n", arg);
 		erts_usage();
-- 
cgit v1.2.3


From 933df3caf2b62303e8b5f36af2eafc6a45a24330 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 22 May 2015 21:59:48 +0200
Subject: Fix statistics reported about fix alloc types

---
 erts/emulator/beam/erl_alloc_util.c | 61 +++++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2f277690e4..b92533f228 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -718,7 +718,7 @@ static void make_name_atoms(Allctr_t *allctr);
 static Block_t *create_carrier(Allctr_t *, Uint, UWord);
 static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **);
 static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp);
-static void dealloc_block(Allctr_t *, void *, int);
+static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int);
 
 /* internal data... */
 
@@ -1067,17 +1067,21 @@ typedef struct {
 } ErtsAllctrFixDDBlock_t;
 #endif
 
+#define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS)
+
 static ERTS_INLINE void
 dealloc_fix_block(Allctr_t *allctr,
 		  ErtsAlcType_t type,
 		  void *ptr,
+		  ErtsAlcFixList_t *fix,
 		  int dec_cc_on_redirect)
 {
 #ifdef ERTS_SMP
     /* May be redirected... */
-    ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type;
+    ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0);
+    ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE;
 #endif
-    dealloc_block(allctr, ptr, dec_cc_on_redirect);
+    dealloc_block(allctr, ptr, fix, dec_cc_on_redirect);
 }
 
 static ERTS_INLINE void
@@ -1123,8 +1127,7 @@ fix_cpool_check_shrink(Allctr_t *allctr,
 	    if (fix->u.cpool.min_list_size > fix->list_size)
 		fix->u.cpool.min_list_size = fix->list_size;
 
-	    fix->u.cpool.allocated--;
-	    dealloc_fix_block(allctr, type, p, 0);
+	    dealloc_fix_block(allctr, type, p, fix, 0);
 	}
     }
 }
@@ -1170,7 +1173,8 @@ static ERTS_INLINE void
 fix_cpool_free(Allctr_t *allctr,
 	       ErtsAlcType_t type,
 	       void *p,
-	       Carrier_t **busy_pcrr_pp)
+	       Carrier_t **busy_pcrr_pp,
+	       int unuse)
 {
     ErtsAlcFixList_t *fix;
 
@@ -1178,8 +1182,9 @@ fix_cpool_free(Allctr_t *allctr,
 	   && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
 
     fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
-    
-    fix->u.cpool.used--;
+
+    if (unuse)
+	fix->u.cpool.used--;
 
     if ((!busy_pcrr_pp || !*busy_pcrr_pp)
 	&& !fix->u.cpool.shrink_list
@@ -1237,8 +1242,7 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
 	    fix->list = *((void **) ptr);
 	    fix->list_size--;
 	    fix->u.cpool.shrink_list--;
-	    fix->u.cpool.allocated--;
-	    dealloc_fix_block(allctr, type, ptr, 0);
+	    dealloc_fix_block(allctr, type, ptr, fix, 0);
 	}
 	if (fix->u.cpool.min_list_size > fix->list_size)
 	    fix->u.cpool.min_list_size = fix->list_size;
@@ -1399,7 +1403,7 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
 	    ptr = fix->list;
 	    fix->list = *((void **) ptr);
 	    fix->list_size--;
-	    dealloc_block(allctr, ptr, 0);
+	    dealloc_block(allctr, ptr, NULL, 0);
 	    fix->u.nocpool.allocated--;
 	}
 	if (fix->list_size != 0) {
@@ -1746,11 +1750,13 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr)
 
     type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type;
 
-    ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
-	   && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+    ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE
+	   <= (type & ~ERTS_ALC_FIX_NO_UNUSE));
+    ASSERT((type & ~ERTS_ALC_FIX_NO_UNUSE)
+	   <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
 
     if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
-	fix_nocpool_free(allctr, type, ptr);
+	fix_nocpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), ptr);
     else {
 	Block_t *blk = UMEM2BLK(ptr);
 	Carrier_t *busy_pcrr_p;
@@ -1765,7 +1771,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr)
 				      NULL, &busy_pcrr_p);
 	if (used_allctr == allctr) {
 	doit:
-	    fix_cpool_free(allctr, type, ptr, &busy_pcrr_p);
+	    fix_cpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE),
+			   ptr, &busy_pcrr_p,
+			   !(type & ERTS_ALC_FIX_NO_UNUSE));
 	    clear_busy_pool_carrier(allctr, busy_pcrr_p);
 	}
 	else {
@@ -1885,7 +1893,7 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	    if (fix)
 		handle_delayed_fix_dealloc(allctr, ptr);
 	    else
-		dealloc_block(allctr, ptr, 1);
+		dealloc_block(allctr, ptr, NULL, 1);
 	}
     }
 
@@ -1991,15 +1999,24 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
 			   ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL)
 
 static void
-dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect)
+dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_redirect)
 {
     Block_t *blk = UMEM2BLK(ptr);
 
     ERTS_SMP_LC_ASSERT(!allctr->thread_safe
 		       || erts_lc_mtx_is_locked(&allctr->mutex));
 
-    if (IS_SBC_BLK(blk))
+    if (IS_SBC_BLK(blk)) {
 	destroy_carrier(allctr, blk, NULL);
+#ifdef ERTS_SMP
+	if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+	    ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type;
+	    if (!(type & ERTS_ALC_FIX_NO_UNUSE))
+		fix->u.cpool.used--;
+	    fix->u.cpool.allocated--;
+	}
+#endif
+    }
 #ifndef ERTS_SMP
     else
 	mbc_free(allctr, ptr, NULL);
@@ -2012,6 +2029,12 @@ dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect)
 	used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr,
 				      NULL, &busy_pcrr_p);
 	if (used_allctr == allctr) {
+	    if (fix) {
+		ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type;
+		if (!(type & ERTS_ALC_FIX_NO_UNUSE))
+		    fix->u.cpool.used--;
+		fix->u.cpool.allocated--;
+	    }
 	    mbc_free(allctr, ptr, &busy_pcrr_p);
 	    clear_busy_pool_carrier(allctr, busy_pcrr_p);
 	}
@@ -5215,7 +5238,7 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
 
 	if (allctr->fix) {
 	    if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
-		fix_cpool_free(allctr, type, p, busy_pcrr_pp);
+		fix_cpool_free(allctr, type, p, busy_pcrr_pp, 1);
 	    else
 		fix_nocpool_free(allctr, type, p);
 	}
-- 
cgit v1.2.3


From 79729c7d95cfc2b163c55071589e83019247c5a1 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 13 May 2015 16:28:59 +0200
Subject: Fix bug causing timeout to overwrite exit instruction

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index af8db519d4..0a8897320d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9649,7 +9649,7 @@ Process *schedule(Process *p, int calls)
 
 	ASSERT(erts_proc_read_refc(p) > 0);
 
-	if (ERTS_PTMR_IS_TIMED_OUT(p)) {
+	if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) {
 	    BeamInstr** pi;
 #ifdef ERTS_SMP
 	    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
-- 
cgit v1.2.3


From 528954b7ec9642f9ec987707352d558f9fd41446 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Sat, 23 May 2015 00:56:21 +0200
Subject: erts: Fix garbage collect literals in code purge

During code purging and check_process_code, the checking of the binary reference
embedded in the match binary state was omitted for the tracing tests. This would cause
the binary match state to reference deallocated memory.
---
 erts/emulator/beam/beam_bif_load.c | 11 ++++++++++-
 erts/emulator/beam/erl_gc.c        | 29 ++++++++++++++++++++++++-----
 2 files changed, 34 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index df1983a83d..ef42bb20d3 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -33,6 +33,7 @@
 #include "beam_catches.h"
 #include "erl_binary.h"
 #include "erl_nif.h"
+#include "erl_bits.h"
 #include "erl_thr_progress.h"
 
 static void set_default_trace_pattern(Eterm module);
@@ -940,7 +941,15 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	    break;
 	case TAG_PRIMARY_HEADER:
 	    if (!header_is_transparent(val)) {
-		Eterm* new_p = p + thing_arityval(val);
+		Eterm* new_p;
+                if (header_is_bin_matchstate(val)) {
+                    ErlBinMatchState *ms = (ErlBinMatchState*) p;
+                    ErlBinMatchBuffer *mb = &(ms->mb);
+                    if (in_area(EXPAND_POINTER(mb->orig), mod_start, mod_size)) {
+                        return 1;
+                    }
+                }
+		new_p = p + thing_arityval(val);
 		ASSERT(start <= new_p && new_p < end);
 		p = new_p;
 	    }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 0db42d4325..3856fc0a6a 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -677,7 +677,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     Uint area_size;
     Eterm* old_htop;
     Uint n;
-    struct erl_off_heap_header** prev;
+    struct erl_off_heap_header** prev = NULL;
 
     if (p->flags & F_DISABLE_GC)
 	return;
@@ -786,10 +786,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
      */
 
     if (oh) {
-	prev = &MSO(p).first;
-	while (*prev) {
-	    prev = &(*prev)->next;
-	}
+        prev = &MSO(p).first;
+        while (*prev) {
+            prev = &(*prev)->next;
+        }
     }
 
     /*
@@ -818,6 +818,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 	oh = oh->next;
     }
 
+    if (prev) {
+        *prev = NULL;
+    }
+
     /*
      * We no longer need this temporary area.
      */
@@ -1869,6 +1873,21 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	    if (!header_is_thing(gval)) {
 		heap_ptr++;
 	    } else {
+		if (header_is_bin_matchstate(gval)) {
+		    ErlBinMatchState *ms = (ErlBinMatchState*) heap_ptr;
+		    ErlBinMatchBuffer *mb = &(ms->mb);
+		    Eterm* origptr;
+		    origptr = &(mb->orig);
+		    ptr = boxed_val(*origptr);
+		    val = *ptr;
+		    if (IS_MOVED_BOXED(val)) {
+			*origptr = val;
+			mb->base = binary_bytes(*origptr);
+		    } else if (in_area(ptr, src, src_size)) {
+			MOVE_BOXED(ptr,val,htop,origptr);
+			mb->base = binary_bytes(*origptr);
+		    }
+		}
 		heap_ptr += (thing_arityval(gval)+1);
 	    }
 	    break;
-- 
cgit v1.2.3


From 2f38cce79ae1bf318cdc8884bd74d1407778d91d Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Thu, 30 Apr 2015 15:46:52 -0400
Subject: Fix for enif_schedule_nif and exceptions

Fix a place where part of the implementation of enif_schedule_nif was
not using the ErlNifEnv exception_thrown field when it should have
been.

Also make the result of enif_schedule_nif return false when passed to
enif_is_exception, and add an assertion for this to the nif_SUITE.c
tests.
---
 erts/emulator/beam/erl_nif.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 426a00304e..7876785a76 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -445,7 +445,7 @@ int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term)
 
 int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term)
 {
-    return term == THE_NON_VALUE;
+    return env->exception_thrown && term == THE_NON_VALUE;
 }
 
 int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term)
@@ -1843,6 +1843,7 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     NifExport* ep;
     ERL_NIF_TERM result;
 
+    ASSERT(!env->exception_thrown);
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
     ep->fp = NULL;
@@ -1855,7 +1856,7 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
      * which case we need to restore the original NIF MFA.
      */
     if (ep->fp == NULL)
-	restore_nif_mfa(proc, ep, is_non_value(result) && proc->freason != TRAP);
+	restore_nif_mfa(proc, ep, env->exception_thrown);
     return result;
 }
 
-- 
cgit v1.2.3


From 884afd8efb8672df2889df98b5b94f3dbb53952c Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Mon, 27 Apr 2015 23:39:37 -0400
Subject: Enhance enif_has_pending_exception

Sverker Eriksson came up with the following idea: to handle a future
ability for NIFs to raise more than just badarg exceptions, modify the
recently-added enif_has_pending_exception function to take a second
argument: a pointer to ERL_NIF_TERM. If this argument is a null
pointer, ignore it. Otherwise, if the first argument, an ErlNifEnv*,
has an associated exception, set the pointed-to ERL_NIF_TERM of the
second argument to the value of the exception term. Add new tests and
documentation for this modification.
---
 erts/emulator/beam/erl_nif.c           | 5 ++++-
 erts/emulator/beam/erl_nif_api_funcs.h | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 7876785a76..36da6519f7 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -741,8 +741,11 @@ Eterm enif_make_badarg(ErlNifEnv* env)
     BIF_ERROR(env->proc, BADARG);
 }
 
-int enif_has_pending_exception(ErlNifEnv* env)
+int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason)
 {
+    if (env->exception_thrown && reason != NULL) {
+	*reason = am_badarg;
+    }
     return env->exception_thrown;
 }
 
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index bdcbb32c46..fe25c803ec 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -156,7 +156,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIte
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter));
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
-ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env));
+ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
-- 
cgit v1.2.3


From 17735c9f3879145f43a3e4be0369b7117b1b7b84 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Wed, 6 May 2015 23:15:04 -0400
Subject: Add enif_raise_exception

Add enif_raise_exception function to allow NIFs to raise error
exceptions holding any Erlang terms. This does not replace or
deprecate the enif_make_badarg function, though, because raising
badarg errors is so idiomatic in NIFs. Reimplement enif_make_badarg on
top of enif_raise_exception. Add new tests for enif_raise_exception
for both normal and dirty NIFs. Add documentation for
enif_raise_exception.
---
 erts/emulator/beam/erl_nif.c           | 63 ++++++++++++++++++++++------------
 erts/emulator/beam/erl_nif_api_funcs.h |  2 ++
 2 files changed, 43 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 36da6519f7..8c222a6db7 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -736,15 +736,21 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
 
 
 Eterm enif_make_badarg(ErlNifEnv* env)
+{
+    return enif_raise_exception(env, am_badarg);
+}
+
+Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason)
 {
     env->exception_thrown = 1;
-    BIF_ERROR(env->proc, BADARG);
+    env->proc->fvalue = reason;
+    BIF_ERROR(env->proc, EXC_ERROR);
 }
 
 int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason)
 {
     if (env->exception_thrown && reason != NULL) {
-	*reason = am_badarg;
+	*reason = env->proc->fvalue;
     }
     return env->exception_thrown;
 }
@@ -1541,12 +1547,13 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent)
  * NIF exports need a few more items than the Export struct provides,
  * including the erl_module_nif* and a NIF function pointer, so the
  * NifExport below adds those. The Export member must be first in the
- * struct. The saved_mfa, saved_argc, nif_level, alloced_argv_sz and argv
- * members are used to track the MFA and arguments of the top NIF in case a
- * chain of one or more enif_schedule_nif() calls results in an exception,
- * since in that case the original MFA and registers have to be restored
- * before returning to Erlang to ensure stacktrace information associated
- * with the exception is correct.
+ * struct. The saved_mfa, exception_thrown, saved_argc, rootset_extra, and
+ * rootset members are used to track the MFA, any pending exception, and
+ * arguments of the top NIF in case a chain of one or more
+ * enif_schedule_nif() calls results in an exception, since in that case
+ * the original MFA and registers have to be restored before returning to
+ * Erlang to ensure stacktrace information associated with the exception is
+ * correct.
  */
 typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]);
 
@@ -1555,25 +1562,28 @@ typedef struct {
     struct erl_module_nif* m;
     NativeFunPtr fp;
     Eterm saved_mfa[3];
+    int exception_thrown;
     int saved_argc;
-    int alloced_argv_sz;
-    Eterm argv[1];
+    int rootset_extra;
+    Eterm rootset[1];
 } NifExport;
 
 /*
  * If a process has saved arguments, they need to be part of the GC
  * rootset. The function below is called from setup_rootset() in
- * erl_gc.c. This function is declared in erl_process.h.
+ * erl_gc.c. This function is declared in erl_process.h. Any exception term
+ * saved in the NifExport is also made part of the GC rootset here; it
+ * always resides in rootset[0].
  */
 int
 erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj)
 {
     NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    int gc = (ep && ep->saved_argc > 0);
+    int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL);
 
     if (gc) {
-	*objv = ep->argv;
-	*nobj = ep->saved_argc;
+	*objv = ep->rootset;
+	*nobj = 1 + ep->saved_argc;
     }
     return gc;
 }
@@ -1585,14 +1595,14 @@ static NifExport*
 allocate_nif_sched_data(Process* proc, int argc)
 {
     NifExport* ep;
-    size_t argv_extra, total;
+    size_t total;
     int i;
 
-    argv_extra = argc > 1 ? sizeof(Eterm)*(argc-1) : 0;
-    total = sizeof(NifExport) + argv_extra;
+    total = sizeof(NifExport) + argc*sizeof(Eterm);
     ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total);
     sys_memset((void*) ep, 0, total);
-    ep->alloced_argv_sz = argc;
+    ep->rootset_extra = argc;
+    ep->rootset[0] = NIL;
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
 	ep->exp.addressv[i] = &ep->exp.code[3];
     }
@@ -1633,15 +1643,22 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     if (!ep)
 	ep = allocate_nif_sched_data(proc, argc);
-    else if (need_save && ep->alloced_argv_sz < argc) {
+    else if (need_save && ep->rootset_extra < argc) {
 	NifExport* new_ep = allocate_nif_sched_data(proc, argc);
 	destroy_nif_export(ep);
 	ep = new_ep;
     }
+    if (env->exception_thrown) {
+	ep->exception_thrown = 1;
+	ep->rootset[0] = env->proc->fvalue;
+    } else {
+	ep->exception_thrown = 0;
+	ep->rootset[0] = NIL;
+    }
     ERTS_VBUMP_ALL_REDS(proc);
     for (i = 0; i < argc; i++) {
 	if (need_save)
-	    ep->argv[i] = reg[i];
+	    ep->rootset[i+1] = reg[i];
 	reg[i] = (Eterm) argv[i];
     }
     if (need_save) {
@@ -1677,7 +1694,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception)
     proc->current[2] = ep->saved_mfa[2];
     if (exception)
 	for (i = 0; i < ep->saved_argc; i++)
-	    reg[i] = ep->argv[i];
+	    reg[i] = ep->rootset[i+1];
     ep->saved_argc = 0;
     ep->saved_mfa[0] = THE_NON_VALUE;
 }
@@ -1702,6 +1719,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
+    ASSERT(!ep->exception_thrown);
     if (ep->fp)
 	restore_nif_mfa(proc, ep, 0);
     return argv[0];
@@ -1719,9 +1737,10 @@ dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
+    ASSERT(ep->exception_thrown);
     if (ep->fp)
 	restore_nif_mfa(proc, ep, 1);
-    return enif_make_badarg(env);
+    return enif_raise_exception(env, ep->rootset[0]);
 }
 
 /*
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index fe25c803ec..e38a016958 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -157,6 +157,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIte
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
 ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -307,6 +308,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair)
 #  define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif)
 #  define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception)
+#  define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From 951b8c9076cc9a5283483633052587984274b8aa Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 27 May 2015 17:53:59 +0200
Subject: erts: Rename ErlNifMapIteratorEntry enums

To differentiate between first/last map entry
and head/tail which is before/after first/last map entry.
---
 erts/emulator/beam/erl_nif.c | 8 ++++----
 erts/emulator/beam/erl_nif.h | 8 ++++++--
 2 files changed, 10 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 426a00304e..4df34e4a06 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2024,8 +2024,8 @@ int enif_map_iterator_create(ErlNifEnv *env,
 	size_t offset;
 
 	switch (entry) {
-	    case ERL_NIF_MAP_ITERATOR_HEAD: offset = 0; break;
-	    case ERL_NIF_MAP_ITERATOR_TAIL: offset = flatmap_get_size(mp) - 1; break;
+	    case ERL_NIF_MAP_ITERATOR_FIRST: offset = 0; break;
+	    case ERL_NIF_MAP_ITERATOR_LAST: offset = flatmap_get_size(mp) - 1; break;
 	    default: goto error;
 	}
 
@@ -2048,12 +2048,12 @@ int enif_map_iterator_create(ErlNifEnv *env,
         WSTACK_INIT(iter->u.hash.wstack, ERTS_ALC_T_NIF);
 
         switch (entry) {
-	    case ERL_NIF_MAP_ITERATOR_HEAD:
+	    case ERL_NIF_MAP_ITERATOR_FIRST:
                 iter->idx = 1;
                 hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 0);
                 iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws);
                 break;
-	    case ERL_NIF_MAP_ITERATOR_TAIL:
+	    case ERL_NIF_MAP_ITERATOR_LAST:
                 iter->idx = hashmap_size(map);
                 hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 1);
                 iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws);
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index c4fdfd4187..af806736fd 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -218,8 +218,12 @@ typedef struct /* All fields all internal and may change */
 } ErlNifMapIterator;
 
 typedef enum {
-    ERL_NIF_MAP_ITERATOR_HEAD = 1,
-    ERL_NIF_MAP_ITERATOR_TAIL = 2
+    ERL_NIF_MAP_ITERATOR_FIRST = 1,
+    ERL_NIF_MAP_ITERATOR_LAST = 2,
+
+    /* deprecated synonyms (undocumented in 17 and 18-rc) */
+    ERL_NIF_MAP_ITERATOR_HEAD = ERL_NIF_MAP_ITERATOR_FIRST,
+    ERL_NIF_MAP_ITERATOR_TAIL = ERL_NIF_MAP_ITERATOR_LAST
 } ErlNifMapIteratorEntry;
 
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
-- 
cgit v1.2.3


From 957f619382923be72835500f56e75d8bbe553892 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 26 May 2015 20:12:08 +0200
Subject: erts: Fix magic binary alignment on 32-bit

Caused bus error on 32-bit sparc from unaligned 64-bit word in
binary_to_term trap context.

Also add _UNALIGNED_ magic macros to avoid double alignment padding
in NIF resources.
---
 erts/emulator/beam/erl_binary.h | 18 ++++++++++++++---
 erts/emulator/beam/erl_bits.c   |  8 ++++++++
 erts/emulator/beam/erl_nif.c    | 26 +++++++++++++++---------
 erts/emulator/beam/global.h     | 45 +++++++++++++++++++++++++++++++++--------
 4 files changed, 77 insertions(+), 20 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 8d264d166e..6b96787d40 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -194,6 +194,9 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size);
 ERTS_GLB_INLINE void erts_bin_free(Binary *bp);
+ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
+                                                  void (*destructor)(Binary *),
+                                                  int unaligned);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
 						 void (*destructor)(Binary *));
 
@@ -332,21 +335,30 @@ erts_bin_free(Binary *bp)
 }
 
 ERTS_GLB_INLINE Binary *
-erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
+erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *),
+                           int unaligned)
 {
-    Uint bsize = ERTS_MAGIC_BIN_SIZE(size);
+    Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size)
+                           : ERTS_MAGIC_BIN_SIZE(size);
     Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize);
     ASSERT(bsize > size);
     if (!bptr)
 	erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize);
     ERTS_CHK_BIN_ALIGNMENT(bptr);
     bptr->flags = BIN_FLAG_MAGIC;
-    bptr->orig_size = ERTS_MAGIC_BIN_ORIG_SIZE(size);
+    bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size)
+                                : ERTS_MAGIC_BIN_ORIG_SIZE(size);
     erts_refc_init(&bptr->refc, 0);
     ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor;
     return bptr;
 }
 
+ERTS_GLB_INLINE Binary *
+erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
+{
+    return erts_create_magic_binary_x(size, destructor, 0);
+}
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* !__ERL_BINARY_H */
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index b8ae93fa58..2e29bf8895 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -107,6 +107,14 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0)
 void
 erts_init_bits(void)
 {
+    ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0);
+    ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0);
+    ERTS_CT_ASSERT(ERTS_MAGIC_BIN_BYTES_TO_ALIGN ==
+                   (offsetof(ErtsMagicBinary,u.aligned.data)
+                    - offsetof(ErtsMagicBinary,u.unaligned.data)));
+    ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes)
+                == offsetof(Binary,orig_bytes));
+
     erts_smp_atomic_init_nob(&bits_bufs_size, 0);
 #if defined(ERTS_SMP)
     /* erl_process.c calls erts_bits_init_state() on all state instances */
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 426a00304e..f42ccf23c2 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1199,7 +1199,11 @@ typedef struct enif_resource_t
     struct enif_resource_type_t* type;
 #ifdef DEBUG
     erts_refc_t nif_refc;
+# ifdef ARCH_32
+    byte align__[4];
+# endif
 #endif
+
     char data[1];
 }ErlNifResource;
 
@@ -1375,7 +1379,7 @@ static void rollback_opened_resource_types(void)
 
 static void nif_resource_dtor(Binary* bin)
 {
-    ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin);
+    ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
     ErlNifResourceType* type = resource->type;
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
 
@@ -1396,8 +1400,10 @@ static void nif_resource_dtor(Binary* bin)
 
 void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 {
-    Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor);
-    ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin);
+    Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size),
+                                             &nif_resource_dtor,
+                                             1); /* unaligned */
+    ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
 
     ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */
     resource->type = type;
@@ -1412,7 +1418,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 void enif_release_resource(void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
-    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
 #ifdef DEBUG
@@ -1426,7 +1432,7 @@ void enif_release_resource(void* obj)
 void enif_keep_resource(void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
-    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
 #ifdef DEBUG
@@ -1438,7 +1444,7 @@ void enif_keep_resource(void* obj)
 ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
-    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
     Eterm* hp = alloc_heap(env,PROC_BIN_SIZE);
     return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
 }
@@ -1467,7 +1473,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
 	return 0; / * Or should we allow "resource binaries" as handles? * /
     }*/
     mbin = pb->val;
-    resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin);
+    resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
 	|| resource->type != type) {	
 	return 0;
@@ -1479,8 +1485,8 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
 size_t enif_sizeof_resource(void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
-    Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary;
-    return ERTS_MAGIC_BIN_DATA_SIZE(bin) - offsetof(ErlNifResource,data);
+    Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary;
+    return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErlNifResource,data);
 }
 
 
@@ -2689,6 +2695,8 @@ erts_unload_nif(struct erl_module_nif* lib)
 
 void erl_nif_init()
 {
+    ERTS_CT_ASSERT((offsetof(ErlNifResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN);
+
     resource_type_list.next = &resource_type_list;
     resource_type_list.prev = &resource_type_list;
     resource_type_list.dtor = NULL;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 340c7033ab..ee1f70b748 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -230,9 +230,23 @@ typedef struct {
     ERTS_INTERNAL_BINARY_FIELDS
     SWord orig_size;
     void (*destructor)(Binary *);
-    char magic_bin_data[1];
+    union {
+        struct {
+            ERTS_BINARY_STRUCT_ALIGNMENT
+            char data[1];
+        } aligned;
+        struct {
+            char data[1];
+        } unaligned;
+    } u;
 } ErtsMagicBinary;
 
+#ifdef ARCH_32
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4
+#else
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0
+#endif
+
 typedef union {
     Binary binary;
     ErtsMagicBinary magic_binary;
@@ -252,15 +266,30 @@ typedef union {
 #define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \
   ((ErtsBinary *) (BP))->magic_binary.destructor
 #define ERTS_MAGIC_BIN_DATA(BP) \
-  ((void *) ((ErtsBinary *) (BP))->magic_binary.magic_bin_data)
-#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \
-  ((BP)->orig_size - sizeof(void (*)(Binary *)))
+  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data)
+#define ERTS_MAGIC_DATA_OFFSET \
+  (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes))
 #define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \
-  (sizeof(void (*)(Binary *)) + (Sz))
+  (ERTS_MAGIC_DATA_OFFSET + (Sz))
 #define ERTS_MAGIC_BIN_SIZE(Sz) \
-  (offsetof(ErtsMagicBinary,magic_bin_data) + (Sz))
-#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \
-  ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,magic_bin_data)))
+  (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz))
+
+/* On 32-bit arch these macro variants will save memory
+   by not forcing 8-byte alignment for the magic payload.
+*/
+#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \
+  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data)
+#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \
+  (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes))
+#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \
+  ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET)
+#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \
+  (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz))
+#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \
+  (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz))
+#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \
+  ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data)))
+
 
 #define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary)
 #define ErlDrvBinary2Binary(D) ((Binary *) \
-- 
cgit v1.2.3


From 512c321bdaf25b685124405e287a3839be467149 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 2 Jun 2015 19:23:25 +0200
Subject: erts: Add save/restore for PSTACK

---
 erts/emulator/beam/global.h | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ee1f70b748..07806c823c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -777,6 +777,15 @@ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */                    \
                  ERTS_ALC_T_ESTACK   /* alloc_type */                      \
 }
 
+#define PSTACK_CHANGE_ALLOCATOR(s,t)					\
+do {									\
+    if (s.pstart != (byte*)PSTK_DEF_STACK(s)) {				\
+	erl_exit(1, "Internal error - trying to change allocator "	\
+		 "type of active pstack\n");				\
+    }									\
+    s.alloc_type = (t);							\
+ } while (0)
+
 #define PSTACK_DESTROY(s)				\
 do {							\
     if (s.pstart != (byte*)PSTK_DEF_STACK(s)) {		\
@@ -786,6 +795,8 @@ do {							\
 
 #define PSTACK_IS_EMPTY(s) (s.psp < s.pstart)
 
+#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart)
+
 #define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp))
 
 #define PSTACK_PUSH(s) 		                                           \
@@ -796,6 +807,37 @@ do {							\
 
 #define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE)))
 
+/*
+ * Do not free the stack after this, it may have pointers into what
+ * was saved in 'dst'.
+ */
+#define PSTACK_SAVE(s,dst)\
+do {\
+    if (s.pstart == (byte*)PSTK_DEF_STACK(s)) {\
+	UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\
+	(dst)->pstart = erts_alloc(s.alloc_type,\
+				   sizeof(PSTK_DEF_STACK(s)));\
+	memcpy((dst)->pstart, s.pstart, _pbytes);\
+	(dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\
+	(dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\
+	(dst)->alloc_type = s.alloc_type;\
+    } else\
+        *(dst) = s;\
+ } while (0)
+
+/*
+ * Use on empty stack, only the allocator can be changed before this.
+ * The src stack is reset to NULL.
+ */
+#define PSTACK_RESTORE(s, src)			        \
+do {						        \
+    ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s));	\
+    s = *(src);  /* struct copy */		        \
+    (src)->pstart = NULL;			        \
+    ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE)));  \
+    ASSERT(s.psp < s.pend);			        \
+} while (0)
+
 
 /* binary.c */
 
-- 
cgit v1.2.3


From 9b683f4bf98500f76a33f098879ee733790baa1f Mon Sep 17 00:00:00 2001
From: Dan Gudmundsson <dgud@erlang.org>
Date: Thu, 28 May 2015 10:57:55 +0200
Subject: Reorder scheduler information in crashdumps

To make it easier to parse stack trace information from tools
---
 erts/emulator/beam/erl_process.c | 66 +++++++++++++++++++++-------------------
 1 file changed, 34 insertions(+), 32 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b64a7f8902..4940ffc4a0 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12509,38 +12509,6 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
         erts_print(to, to_arg, "%T", esdp->current_port->common.id);
     erts_print(to, to_arg, "\n");
 
-    p = esdp->current_process;
-    erts_print(to, to_arg, "Current Process: ");
-    if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) {
-      flg = erts_smp_atomic32_read_dirty(&p->state);
-      erts_print(to, to_arg, "%T\n", p->common.id);
-      
-      erts_print(to, to_arg, "Current Process State: ");
-      erts_dump_process_state(to, to_arg, flg);
-      
-      erts_print(to, to_arg, "Current Process Internal State: ");
-      erts_dump_extended_process_state(to, to_arg, flg);
-      
-      erts_print(to, to_arg, "Current Process Program counter: %p (", p->i);
-      print_function_from_pc(to, to_arg, p->i);
-      erts_print(to, to_arg, ")\n");
-      erts_print(to, to_arg, "Current Process CP: %p (", p->cp);
-      print_function_from_pc(to, to_arg, p->cp);
-      erts_print(to, to_arg, ")\n");
-      
-      /* Getting this stacktrace can segfault if we are very very
-	 unlucky if called while a process is being garbage collected.
-	 Therefore we only call this on other schedulers if we either
-	 have protection against segfaults, or we know that the process
-	 is not garbage collecting. It *should* always be safe to call
-	 on a process owned by us, even if it is currently being garbage
-	 collected.
-      */
-      erts_print(to, to_arg, "Current Process Limited Stack Trace:\n");
-      erts_limited_stack_trace(to, to_arg, p);
-    } else
-      erts_print(to, to_arg, "\n");
-
     for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) {
         erts_print(to, to_arg, "Run Queue ");
         switch (i) {
@@ -12627,6 +12595,40 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
         }
     }
     erts_print(to, to_arg, "\n");
+
+    /* This *MUST* to be the last information in scheduler block */
+    p = esdp->current_process;
+    erts_print(to, to_arg, "Current Process: ");
+    if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) {
+	flg = erts_smp_atomic32_read_dirty(&p->state);
+	erts_print(to, to_arg, "%T\n", p->common.id);
+
+	erts_print(to, to_arg, "Current Process State: ");
+	erts_dump_process_state(to, to_arg, flg);
+
+	erts_print(to, to_arg, "Current Process Internal State: ");
+	erts_dump_extended_process_state(to, to_arg, flg);
+
+	erts_print(to, to_arg, "Current Process Program counter: %p (", p->i);
+	print_function_from_pc(to, to_arg, p->i);
+	erts_print(to, to_arg, ")\n");
+	erts_print(to, to_arg, "Current Process CP: %p (", p->cp);
+	print_function_from_pc(to, to_arg, p->cp);
+	erts_print(to, to_arg, ")\n");
+
+	/* Getting this stacktrace can segfault if we are very very
+	   unlucky if called while a process is being garbage collected.
+	   Therefore we only call this on other schedulers if we either
+	   have protection against segfaults, or we know that the process
+	   is not garbage collecting. It *should* always be safe to call
+	   on a process owned by us, even if it is currently being garbage
+	   collected.
+	*/
+	erts_print(to, to_arg, "Current Process Limited Stack Trace:\n");
+	erts_limited_stack_trace(to, to_arg, p);
+    } else
+	erts_print(to, to_arg, "\n");
+
 }
 
 /*
-- 
cgit v1.2.3


From d79f77985100d2f90a0021f7ebae20ec7c99e93f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 5 Jun 2015 16:00:30 +0200
Subject: erts: Fix faulty list optimization in make_internal_hash

Reported-by: Rory Byrne
---
 erts/emulator/beam/utils.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 965de748c9..cecd88197e 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1140,7 +1140,7 @@ make_hash2(Eterm term)
 
     ERTS_UNDEF(hash_xor_pairs, 0);
 
-/* (HCONST * {2, ..., 16}) mod 2^32 */
+/* (HCONST * {2, ..., 22}) mod 2^32 */
 #define HCONST_2 0x3c6ef372UL
 #define HCONST_3 0xdaa66d2bUL
 #define HCONST_4 0x78dde6e4UL
@@ -1161,6 +1161,7 @@ make_hash2(Eterm term)
 #define HCONST_19 0xbe1e08bbUL
 #define HCONST_20 0x5c558274UL
 #define HCONST_21 0xfa8cfc2dUL
+#define HCONST_22 0x98c475e6UL
 
 #define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF))
 #define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF))
@@ -1645,8 +1646,9 @@ make_internal_hash(Eterm term)
 		    break;
 		ptr = list_val(term);
 	    }
-	    if (c > 0)
-		UINT32_HASH(sh, HCONST_4);
+            if (c > 0)
+                UINT32_HASH_2(sh, (Uint32)c, HCONST_22);
+
 	    if (is_list(term)) {
 		tmp = CDR(ptr);
                 CONST_HASH(HCONST_17);  /* Hash CAR in cons cell */
-- 
cgit v1.2.3


From eea59350ddc1f8c2d86e10f5d38f0cda2f9081f3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 8 Jun 2015 14:52:41 +0200
Subject: erts: Refactor arg swapping for maps:merge

---
 erts/emulator/beam/erl_map.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index a1bd39dbc8..5802ec76ba 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -80,7 +80,7 @@ typedef struct {
 
 static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
 static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -947,7 +947,7 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
 	BIF_P->fvalue = BIF_ARG_2;
     } else if (is_hashmap(BIF_ARG_1)) {
 	if (is_hashmap(BIF_ARG_2)) {
-	    BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
+	    BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
 	} else if (is_flatmap(BIF_ARG_2)) {
 	    /* Will always become a tree */
 	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1));
@@ -1113,12 +1113,12 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
 
-    return swap_args ? hashmap_merge(p, tree, res) : hashmap_merge(p, res, tree);
+    return hashmap_merge(p, res, tree, swap_args);
 }
 
 #define HALLOC_EXTRA 200
 
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
+static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args) {
 #define PSTACK_TYPE struct HashmapMergePStackType
     struct HashmapMergePStackType {
 	Eterm *srcA, *srcB;
@@ -1133,7 +1133,7 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
     Eterm hdrA, hdrB;
     Uint32 ahx, bhx;
     Uint size;  /* total key-value counter */
-    int keepA = 0;
+    int keepA = swap_args;
     unsigned int lvl = 0;
     DeclareTmpHeap(th,2,p);
     Eterm res = THE_NON_VALUE;
-- 
cgit v1.2.3


From a6038dac4b387a0f18be7d18467bbf8f83e8c765 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 8 Jun 2015 16:40:39 +0200
Subject: Fix lost aux work flags when setting multiple flags

All flags was not always set when setting multiple aux work
flags at once. This scenario is fortunately quite uncommon
and only caused further delay in memory deallocations.
---
 erts/emulator/beam/erl_process.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ea63d20dfa..f4228e0026 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1193,11 +1193,11 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi,
     ERTS_DBG_CHK_SSI_AUX_WORK(ssi);
 
     old_flgs = erts_atomic32_read_nob(&ssi->aux_work);
-    if ((old_flgs & flgs) == 0) {
+    if ((old_flgs & flgs) != flgs) {
 
 	old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs);
 
-	if ((old_flgs & flgs) == 0) {
+	if ((old_flgs & flgs) != flgs) {
 #ifdef ERTS_SMP
 	    erts_sched_poke(ssi);
 #else
@@ -1217,7 +1217,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
 
     old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs);
 
-    if ((old_flgs & flgs) == 0) {
+    if ((old_flgs & flgs) != flgs) {
 #ifdef ERTS_SMP
 	erts_sched_poke(ssi);
 #else
-- 
cgit v1.2.3


From f71ce89e2ca069b8526f27aee333deea9209a8b6 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 2 Jun 2015 19:50:47 +0200
Subject: Fix test cases

---
 erts/emulator/beam/erl_bif_info.c |   9 ++-
 erts/emulator/beam/erl_process.c  | 156 +++++++++++++++++++++++++++-----------
 erts/emulator/beam/erl_process.h  |  21 ++++-
 3 files changed, 136 insertions(+), 50 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f74aea80a7..ae8a5c266a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4050,7 +4050,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	}
 	else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) {
 	    if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) {
-		if (erts_debug_wait_deallocations(BIF_P)) {
+		int flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS;
+		if (erts_debug_wait_completed(BIF_P, flag)) {
+		    ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
+		}
+	    }
+	    if (ERTS_IS_ATOM_STR("timer_cancellations", BIF_ARG_2)) {
+		int flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS;
+		if (erts_debug_wait_completed(BIF_P, flag)) {
 		    ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
 		}
 	    }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 0a8897320d..42aad63f5c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -508,6 +508,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS
     valid |= ERTS_SSI_AUX_WORK_REAP_PORTS;
 #endif
+    valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
 
     if (~valid & value)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -1193,11 +1194,11 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi,
     ERTS_DBG_CHK_SSI_AUX_WORK(ssi);
 
     old_flgs = erts_atomic32_read_nob(&ssi->aux_work);
-    if ((old_flgs & flgs) == 0) {
+    if ((old_flgs & flgs) != flgs) {
 
 	old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs);
 
-	if ((old_flgs & flgs) == 0) {
+	if ((old_flgs & flgs) != flgs) {
 #ifdef ERTS_SMP
 	    erts_sched_poke(ssi);
 #else
@@ -1217,7 +1218,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
 
     old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs);
 
-    if ((old_flgs & flgs) == 0) {
+    if ((old_flgs & flgs) != flgs) {
 #ifdef ERTS_SMP
 	erts_sched_poke(ssi);
 #else
@@ -1715,11 +1716,6 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
 	awdp->dd.thr_prgr = wakeup;
 	haw_thr_prgr_soft_wakeup(awdp, wakeup);
     }
-    else if (awdp->dd.completed_callback) {
-	awdp->dd.completed_callback(awdp->dd.completed_arg);
-	awdp->dd.completed_callback = NULL;
-	awdp->dd.completed_arg = NULL;
-    }
     return aux_work & ~ERTS_SSI_AUX_WORK_DD;
 }
 
@@ -1761,11 +1757,6 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     }
     else {
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
-	if (awdp->dd.completed_callback) {
-	    awdp->dd.completed_callback(awdp->dd.completed_arg);
-	    awdp->dd.completed_callback = NULL;
-	    awdp->dd.completed_arg = NULL;
-	}
     }
 
     return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
@@ -1955,78 +1946,142 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *),
 #endif
 }
 
-#ifdef ERTS_SMP
+static ERTS_INLINE erts_aint32_t
+handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    erts_aint32_t saved_aux_work, flags;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+
+    flags = awdp->debug.wait_completed.flags;
+
+    if (aux_work & flags)
+	return aux_work;
 
-static erts_atomic32_t completed_dealloc_count;
+    saved_aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+
+    if (saved_aux_work & flags)
+	return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+
+    awdp->debug.wait_completed.callback(awdp->debug.wait_completed.arg);
+
+    awdp->debug.wait_completed.flags = 0;
+    awdp->debug.wait_completed.callback = NULL;
+    awdp->debug.wait_completed.arg = NULL;
+
+    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED);
+
+    return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+}
+
+static erts_atomic32_t debug_wait_completed_count;
+static int debug_wait_completed_flags;
 
 static void
-completed_dealloc(void *vproc)
+thr_debug_wait_completed(void *vproc)
 {
-    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) {
+    if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 0) {
 	erts_resume((Process *) vproc, (ErtsProcLocks) 0);
 	erts_proc_dec_refc((Process *) vproc);
     }
 }
 
 static void
-setup_completed_dealloc(void *vproc)
+setup_thr_debug_wait_completed(void *vproc)
 {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    ErtsAuxWorkData *awdp = (esdp
-			     ? &esdp->aux_work_data
-			     : aux_thread_aux_work_data);
-    erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
-    set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD);
-    awdp->dd.completed_callback = completed_dealloc;
-    awdp->dd.completed_arg = vproc;
+    ErtsAuxWorkData *awdp;
+    erts_aint32_t wait_flags, aux_work_flags;
+#ifdef ERTS_SMP
+    awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data;
+#else
+    awdp = &esdp->aux_work_data;
+#endif
+
+    wait_flags = 0;
+    aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+
+    if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) {
+	erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
+	wait_flags |= (ERTS_SSI_AUX_WORK_DD
+		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR
+		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+#ifdef ERTS_SMP
+	aux_work_flags |= ERTS_SSI_AUX_WORK_DD;
+#endif
+    }
+
+    if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) {
+	wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS
+		       | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR
+		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+#ifdef ERTS_SMP
+	if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp))
+	    aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
+#endif
+    }
+
+    set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags);
+
+    awdp->debug.wait_completed.flags = wait_flags;
+    awdp->debug.wait_completed.callback = thr_debug_wait_completed;
+    awdp->debug.wait_completed.arg = vproc;
 }
 
 static void
-prep_setup_completed_dealloc(void *vproc)
+prep_setup_thr_debug_wait_completed(void *vproc)
 {
-    erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1);
-    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) {
+    erts_aint32_t count = (erts_aint32_t) erts_no_schedulers;
+#ifdef ERTS_SMP
+    count += 1; /* aux thread */
+#endif
+    if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) {
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
-					  setup_completed_dealloc,
+					  setup_thr_debug_wait_completed,
 					  vproc);
+#ifdef ERTS_SMP
 	/* aux_thread */
 	erts_schedule_misc_aux_work(0,
-				    setup_completed_dealloc,
+				    setup_thr_debug_wait_completed,
 				    vproc);
+#endif
     }
 }
 
-#endif /* ERTS_SMP */
 
 int
-erts_debug_wait_deallocations(Process *c_p)
+erts_debug_wait_completed(Process *c_p, int flags)
 {
-#ifndef ERTS_SMP
-    erts_alloc_fix_alloc_shrink(1, 0);
-    return 1;
-#else
     /* Only one process at a time can do this */
-    erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1));
-    if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count,
+    erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers);
+#ifdef ERTS_SMP
+    count += 2; /* aux thread */
+#endif
+    if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count,
 				      count,
 				      0)) {
+	debug_wait_completed_flags = flags;
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	erts_proc_inc_refc(c_p);
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
-					  prep_setup_completed_dealloc,
+					  prep_setup_thr_debug_wait_completed,
 					  (void *) c_p);
+#ifdef ERTS_SMP
 	/* aux_thread */
 	erts_schedule_misc_aux_work(0,
-				    prep_setup_completed_dealloc,
+				    prep_setup_thr_debug_wait_completed,
 				    (void *) c_p);
+#endif
 	return 1;
     }
     return 0;
-#endif
 }
 
 
@@ -2227,6 +2282,14 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS,
 		    handle_reap_ports);
 
+    /*
+     * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be
+     * the last flag checked!
+     */
+
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED,
+		    handle_debug_wait_completed);
+
     ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
 
 #ifdef ERTS_SMP
@@ -5337,8 +5400,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST;
     awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
-    awdp->dd.completed_callback = NULL;
-    awdp->dd.completed_arg = NULL;
     awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST;
     awdp->later_op.size = 0;
@@ -5369,6 +5430,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 	    awdp->delayed_wakeup.sched2jix[i] = -1;
     }
 #endif
+    awdp->debug.wait_completed.flags = 0;
+    awdp->debug.wait_completed.callback = NULL;
+    awdp->debug.wait_completed.arg = NULL;
 }
 
 static void
@@ -5677,11 +5741,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     init_swtreq_alloc();
 #endif
 
+    erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
+    debug_wait_completed_flags = 0;
 
 #ifdef ERTS_SMP
 
-    erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */
-
     aux_thread_aux_work_data =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 					   sizeof(ErtsAuxWorkData));
@@ -12459,6 +12523,8 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
                 erts_print(to, to_arg, "MSEG_CACHE_CHECK"); break;
             case ERTS_SSI_AUX_WORK_REAP_PORTS:
                 erts_print(to, to_arg, "REAP_PORTS"); break;
+            case ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED:
+                erts_print(to, to_arg, "DEBUG_WAIT_COMPLETED"); break;
             default:
                 erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
             }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b1c30e7652..6578d33a11 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -270,6 +270,9 @@ typedef enum {
 /*
  * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
  * eachother. Most frequent - lowest bit number.
+ *
+ * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be highest bit
+ * and last flag checked...
  */
 
 #define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP	(((erts_aint32_t) 1) << 0)
@@ -288,8 +291,9 @@ typedef enum {
 #define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 13)
 #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 14)
 #define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 15)
+#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED	(((erts_aint32_t) 1) << 16)
 
-#define ERTS_SSI_AUX_WORK_MAX                                           16
+#define ERTS_SSI_AUX_WORK_MAX                                           17
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
@@ -515,8 +519,6 @@ typedef struct {
 #ifdef ERTS_SMP
     struct {
 	ErtsThrPrgrVal thr_prgr;
-	void (*completed_callback)(void *);
-	void (*completed_arg)(void *);
     } dd;
     struct {
 	ErtsThrPrgrVal thr_prgr;
@@ -545,6 +547,13 @@ typedef struct {
 	ErtsDelayedAuxWorkWakeupJob *job;
     } delayed_wakeup;
 #endif
+    struct {
+	struct {
+	    erts_aint32_t flags;
+	    void (*callback)(void *);
+	    void *arg;
+	} wait_completed;
+    } debug;
 } ErtsAuxWorkData;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -1692,7 +1701,11 @@ Eterm erts_get_reader_groups_map(Process *c_p);
 Eterm erts_debug_reader_groups_map(Process *c_p, int groups);
 
 Uint erts_debug_nbalance(void);
-int erts_debug_wait_deallocations(Process *c_p);
+
+#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
+#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)
+
+int erts_debug_wait_completed(Process *c_p, int flags);
 
 Uint erts_process_memory(Process *c_p);
 
-- 
cgit v1.2.3


From c7f308d1bfaf1f2f6f597ebda6679da03e3e732e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 2 Jun 2015 21:48:19 +0200
Subject: Aux work flag descriptions

---
 erts/emulator/beam/erl_process.c | 82 +++++++++++++++++++------------------
 erts/emulator/beam/erl_process.h | 87 +++++++++++++++++++++++++++++-----------
 2 files changed, 107 insertions(+), 62 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 42aad63f5c..3cca13ae25 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -165,6 +165,9 @@ Uint erts_no_dirty_cpu_schedulers;
 Uint erts_no_dirty_io_schedulers;
 #endif
 
+static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
+int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS;
+
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY		(4*1024*1024)
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY			(512*1024)
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM			(64*1024)
@@ -567,6 +570,41 @@ erts_pre_init_process(void)
     erts_tsd_key_create(&sched_data_key, "erts_sched_data_key");
 #endif
 
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX]
+	= "DELAYED_AW_WAKEUP";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_IX]
+	= "DD";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX]
+	= "DD_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX]
+	= "FIX_ALLOC_DEALLOC";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX]
+	= "FIX_ALLOC_LOWER_LIM";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX]
+	= "THR_PRGR_LATER_OP";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX]
+	= "CNCLD_TMRS";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX]
+	= "CNCLD_TMRS_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_IX]
+	= "ASYNC_READY";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX]
+	= "ASYNC_READY_CLEAN";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX]
+	= "MISC_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX]
+	= "MISC";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX]
+	= "CHECK_CHILDREN";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX]
+	= "SET_TMO";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX]
+	= "MSEG_CACHE_CHECK";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX]
+	= "REAP_PORTS";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX]
+	= "DEBUG_WAIT_COMPLETED";
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
  {
      int ix;
@@ -12487,47 +12525,13 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
 
     flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work);
     erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: ");
-    for (i = 0; i < ERTS_SSI_AUX_WORK_MAX && flg; i++) {
+    for (i = 0; i < ERTS_SSI_AUX_WORK_NO_FLAGS && flg; i++) {
         erts_aint32_t chk = (1 << i);
         if (flg & chk) {
-            switch (chk) {
-            case ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP:
-                erts_print(to, to_arg, "DELAYED_AW_WAKEUP"); break;
-            case ERTS_SSI_AUX_WORK_DD:
-                erts_print(to, to_arg, "DELAYED_DEALLOC"); break;
-            case ERTS_SSI_AUX_WORK_DD_THR_PRGR:
-                erts_print(to, to_arg, "DELAYED_DEALLOC_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC:
-                erts_print(to, to_arg, "FIX_ALLOC_DEALLOC"); break;
-            case ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM:
-                erts_print(to, to_arg, "FIX_ALLOC_LOWER_LIM"); break;
-            case ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP:
-                erts_print(to, to_arg, "THR_PRGR_LATER_OP"); break;
-            case ERTS_SSI_AUX_WORK_CNCLD_TMRS:
-                erts_print(to, to_arg, "CANCELED_TIMERS"); break;
-            case ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR:
-                erts_print(to, to_arg, "CANCELED_TIMERS_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_ASYNC_READY:
-                erts_print(to, to_arg, "ASYNC_READY"); break;
-            case ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN:
-                erts_print(to, to_arg, "ASYNC_READY_CLEAN"); break;
-            case ERTS_SSI_AUX_WORK_MISC_THR_PRGR:
-                erts_print(to, to_arg, "MISC_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_MISC:
-                erts_print(to, to_arg, "MISC"); break;
-            case ERTS_SSI_AUX_WORK_CHECK_CHILDREN:
-                erts_print(to, to_arg, "CHECK_CHILDREN"); break;
-            case ERTS_SSI_AUX_WORK_SET_TMO:
-                erts_print(to, to_arg, "SET_TMO"); break;
-            case ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK:
-                erts_print(to, to_arg, "MSEG_CACHE_CHECK"); break;
-            case ERTS_SSI_AUX_WORK_REAP_PORTS:
-                erts_print(to, to_arg, "REAP_PORTS"); break;
-            case ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED:
-                erts_print(to, to_arg, "DEBUG_WAIT_COMPLETED"); break;
-            default:
-                erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
-            }
+	    if (erts_aux_work_flag_descr[i])
+                erts_print(to, to_arg, "%s", erts_aux_work_flag_descr[i]);
+	    else
+                erts_print(to, to_arg, "1<<%d", i);
             if (flg > chk)
                 erts_print(to, to_arg, " | ");
             flg -= chk;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 6578d33a11..c06ac7affa 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -268,32 +268,73 @@ typedef enum {
   | ERTS_SSI_FLG_SUSPENDED)
 
 /*
- * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
- * eachother. Most frequent - lowest bit number.
+ * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
+ * order relative eachother. Most frequent at lowest at lowest
+ * index.
  *
- * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be highest bit
- * and last flag checked...
+ * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
+ * highest index...
+ *
+ * Remember to update description in erts_pre_init_process()
+ * when adding new flags...
  */
 
-#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP	(((erts_aint32_t) 1) << 0)
-#define ERTS_SSI_AUX_WORK_DD			(((erts_aint32_t) 1) << 1)
-#define ERTS_SSI_AUX_WORK_DD_THR_PRGR		(((erts_aint32_t) 1) << 2)
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC	(((erts_aint32_t) 1) << 3)
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM	(((erts_aint32_t) 1) << 4)
-#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP	(((erts_aint32_t) 1) << 5)
-#define ERTS_SSI_AUX_WORK_CNCLD_TMRS		(((erts_aint32_t) 1) << 6)
-#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR	(((erts_aint32_t) 1) << 7)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 8)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 9)
-#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 10)
-#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 11)
-#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 12)
-#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 13)
-#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 14)
-#define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 15)
-#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED	(((erts_aint32_t) 1) << 16)
-
-#define ERTS_SSI_AUX_WORK_MAX                                           17
+typedef enum {
+    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
+    ERTS_SSI_AUX_WORK_DD_IX,
+    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
+    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
+    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
+    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
+    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
+    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
+    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
+    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
+    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
+    ERTS_SSI_AUX_WORK_MISC_IX,
+    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
+    ERTS_SSI_AUX_WORK_SET_TMO_IX,
+    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
+    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
+    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */
+
+    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
+} ErtsSsiAuxWorkFlagIndex;
+
+#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
+#define ERTS_SSI_AUX_WORK_DD \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
+#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
+#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
+#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
+#define ERTS_SSI_AUX_WORK_MISC \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
+#define ERTS_SSI_AUX_WORK_SET_TMO \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
+#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
+#define ERTS_SSI_AUX_WORK_REAP_PORTS \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
+#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
-- 
cgit v1.2.3


From 46efe0d85bc2f55dd432f403f20d4841b07023ed Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 4 Jun 2015 16:45:41 +0200
Subject: Disable accessor timer option

---
 erts/emulator/beam/erl_alloc.c     |  4 +++
 erts/emulator/beam/erl_alloc.types |  2 +-
 erts/emulator/beam/erl_hl_timer.c  | 57 ++++++++++++++++++++++++++++++++------
 erts/emulator/beam/erl_process.c   |  8 ++++++
 erts/emulator/beam/erl_process.h   |  2 ++
 5 files changed, 64 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index dcae5509ec..d11f24220a 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -581,8 +581,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	= erts_timer_type_size(ERTS_ALC_T_HL_PTIMER);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_BIF_TIMER);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER);
+#endif
 
 #ifdef HARD_DEBUG
     hdbg_init();
@@ -2343,10 +2345,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_BIF_TIMER);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_ABIF_TIMER);
+#endif
     }
 
     if (want.atom || want.atom_used) {
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 57c506458c..2721e13250 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -167,7 +167,7 @@ type	TIMER_SERVICE	LONG_LIVED	SYSTEM		timer_service
 type    LL_PTIMER	FIXED_SIZE	PROCESSES	ll_ptimer
 type    HL_PTIMER	FIXED_SIZE	PROCESSES	hl_ptimer
 type    BIF_TIMER	FIXED_SIZE	PROCESSES	bif_timer
-type    ABIF_TIMER	FIXED_SIZE	PROCESSES	accessor_bif_timer
+# type    ABIF_TIMER	FIXED_SIZE	PROCESSES	accessor_bif_timer
 type    TIMER_REQUEST	SHORT_LIVED	PROCESSES	timer_request
 type    BTM_YIELD_STATE	SHORT_LIVED	PROCESSES	btm_yield_state
 type	REG_TABLE	STANDARD	SYSTEM		reg_tab
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 51cd843935..0cb75a8b5f 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -93,11 +93,13 @@ static void hdbg_chk_srv(ErtsHLTimerService *srv);
 /* Bit 0 to 9 contains scheduler id (see mask below) */
 #define ERTS_TMR_ROFLG_HLT		(((Uint32) 1) << 10)
 #define ERTS_TMR_ROFLG_BIF_TMR		(((Uint32) 1) << 11)
-#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 12)
-#define ERTS_TMR_ROFLG_PRE_ALC		(((Uint32) 1) << 13)
-#define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 14)
-#define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 15)
-#define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 16)
+#define ERTS_TMR_ROFLG_PRE_ALC		(((Uint32) 1) << 12)
+#define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 13)
+#define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 14)
+#define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 15)
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 16)
+#endif
 
 #define ERTS_TMR_ROFLG_SID_MASK	\
     (ERTS_TMR_ROFLG_HLT - (Uint32) 1)
@@ -172,15 +174,21 @@ struct ErtsHLTimer_ {
 	Eterm message;
 	ErlHeapFragment *bp;
     } btm;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     struct {
 	Eterm accessor;
 	ErtsHLTimerTree tree;
     } abtm;
+#endif
 };
 
 #define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm)
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 #define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm)
 #define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer)
+#else
+#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer)
+#endif
 
 typedef struct {
     ErtsTmrHead head; /* NEED to be first! */
@@ -584,6 +592,8 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 
 #include "erl_rbtree.h"
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+
 #define ERTS_RBT_PREFIX abtm
 #define ERTS_RBT_T ErtsHLTimer
 #define ERTS_RBT_KEY_T Uint32 *
@@ -634,6 +644,8 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 
 #include "erl_rbtree.h"
 
+#endif /* ERTS_BTM_ACCESSOR_SUPPORT */
+
 #ifdef ERTS_SMP
 static void init_canceled_queue(ErtsHLTCncldTmrQ *cq);
 #endif
@@ -673,7 +685,9 @@ erts_timer_type_size(ErtsAlcType_t type)
     case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer);
     case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE;
     case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE;
+#endif
     default: ERTS_INTERNAL_ERROR("Unknown type");
     }
     return 0;
@@ -852,8 +866,10 @@ hl_timer_destroy(ErtsHLTimer *tmr)
     else {
 	if (roflgs & ERTS_TMR_ROFLG_PRE_ALC)
 	    bif_timer_pre_free(tmr);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
 	    erts_free(ERTS_ALC_T_ABIF_TIMER, tmr);
+#endif
 	else
 	    erts_free(ERTS_ALC_T_BIF_TIMER, tmr);
     }
@@ -948,6 +964,8 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
 #endif
 }
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+
 static void
 hlt_delete_abtm(ErtsHLTimer *tmr)
 {
@@ -971,6 +989,8 @@ hlt_delete_abtm(ErtsHLTimer *tmr)
     }
 }
 
+#endif
+
 static ErtsHLTimer *
 create_hl_timer(ErtsSchedulerData *esdp,
 		ErtsMonotonicTime timeout_pos,
@@ -982,7 +1002,9 @@ create_hl_timer(ErtsSchedulerData *esdp,
     ErtsHLTimer *tmr, *st_tmr;
     erts_aint32_t refc;
     Uint32 roflgs;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     int is_abif_tmr = is_bif_tmr && is_value(acsr) && acsr != rcvr;
+#endif
 
     check_canceled_queue(esdp, srv);
 
@@ -1001,10 +1023,12 @@ create_hl_timer(ErtsSchedulerData *esdp,
     }
     else {
     alloc_bif_timer:
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	if (is_abif_tmr)
 	    tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER,
 			     ERTS_ABIF_TIMER_SIZE);
 	else
+#endif
 	    tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
 			     ERTS_BIF_TIMER_SIZE);
     }
@@ -1057,6 +1081,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	tmr->btm.refn[2] = refn[2];
 
 	tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	if (is_abif_tmr) {
 	    Process *aproc;
 	    roflgs |= ERTS_TMR_ROFLG_ABIF_TMR;
@@ -1071,6 +1096,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
 		erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM);
 	    }
 	}
+#endif
     }
 
     tmr->head.roflgs = roflgs;
@@ -1120,8 +1146,10 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
     Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
 	hlt_delete_abtm(tmr);
+#endif
 
     if (is_reg_name) {
 	Eterm pid;
@@ -1300,8 +1328,10 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
 	    tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 	}
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
 	    hlt_delete_abtm(tmr);
+#endif
     }
 
     if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) {
@@ -1943,8 +1973,10 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
      */
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM);
     tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     if (!tmr)
 	tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn);
+#endif
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM);
     if (!tmr)
 	return 0;
@@ -2184,11 +2216,13 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info,
 	    if (!abs || !bool_arg(tp[2], abs))
 		return 0;
 	    break;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	case am_accessor:
 	    if (!accessor || is_not_internal_pid(tp[2]))
 		return 0;
 	    *accessor = tp[2];
 	    break;
+#endif
 	default:
 	    return 0;
 	}
@@ -2250,7 +2284,9 @@ typedef struct {
     ErtsBifTimers *bif_timers;
     union {
 	proc_btm_rbt_yield_state_t proc_btm_yield_state;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	abtm_rbt_yield_state_t abtm_yield_state;
+#endif
     } u;
 } ErtsBifTimerYieldState;
 
@@ -2291,6 +2327,8 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
     return res;
 }
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+
 static void
 detach_bif_timer(ErtsHLTimer *tmr, void *vesdp)
 {
@@ -2335,6 +2373,8 @@ int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp
     return res;
 }
 
+#endif /* ERTS_BTM_ACCESSOR_SUPPORT */
+
 static ERTS_INLINE int
 parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
 		  ErtsMonotonicTime *conv_arg, int abs,
@@ -2525,7 +2565,8 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo,
 	else
 	    tmr = (void *) create_hl_timer(esdp, timeout_pos,
 					   short_time, 0, (void *) c_p,
-					   c_p->common.id, NIL, NIL, NULL);
+					   c_p->common.id, THE_NON_VALUE,
+					   NIL, NULL);
 	erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr);
     }
 }
@@ -2613,8 +2654,8 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
     else
 	tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, 0,
 				       (void *) c_prt,
-				       c_prt->common.id, NIL, NIL,
-				       NULL);
+				       c_prt->common.id, THE_NON_VALUE,
+				       NIL, NULL);
     erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
 }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 3cca13ae25..410f0214b9 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10922,7 +10922,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg_inq.len = 0;
 #endif
     p->bif_timers = NULL;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     p->accessor_bif_timers = NULL;
+#endif
     p->mbuf = NULL;
     p->mbuf_sz = 0;
     p->psd = NULL;
@@ -11102,7 +11104,9 @@ void erts_init_empty_process(Process *p)
     p->msg.save = &p->msg.first;
     p->msg.len = 0;
     p->bif_timers = NULL;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     p->accessor_bif_timers = NULL;
+#endif
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
     p->seq_trace_lastcnt = 0;
@@ -11197,7 +11201,9 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->msg.first == NULL);
     ASSERT(p->msg.len == 0);
     ASSERT(p->bif_timers == NULL);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     ASSERT(p->accessor_bif_timers == NULL);
+#endif
     ASSERT(p->dictionary == NULL);
     ASSERT(p->catches == 0);
     ASSERT(p->cp == NULL);
@@ -12153,6 +12159,7 @@ erts_continue_exit_process(Process *p)
 	p->bif_timers = NULL;
     }
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     if (p->accessor_bif_timers) {
 	if (erts_detach_accessor_bif_timers(p,
 					    p->accessor_bif_timers,
@@ -12163,6 +12170,7 @@ erts_continue_exit_process(Process *p)
 	ASSERT(erts_proc_read_refc(p) > 0);
 	p->accessor_bif_timers = NULL;
     }
+#endif
 
 #ifdef ERTS_SMP
     if (p->flags & F_HAVE_BLCKD_MSCHED) {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c06ac7affa..1da1239609 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -975,7 +975,9 @@ struct process {
     ErlMessageQueue msg;	/* Message queue */
 
     ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
+#endif
 
     ProcDict  *dictionary;       /* Process dictionary, may be NULL */
 
-- 
cgit v1.2.3


From ea84ab6c03994f8d6d9f07d8740f0547f8a3cb51 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 4 Jun 2015 22:08:33 +0200
Subject: Callback timer

---
 erts/emulator/beam/erl_hl_timer.c | 372 +++++++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_hl_timer.h |  11 +-
 erts/emulator/beam/erl_time.h     |   8 +
 erts/emulator/beam/time.c         |  38 ++++
 4 files changed, 362 insertions(+), 67 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 0cb75a8b5f..8eacb921fe 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -81,6 +81,13 @@ static void hdbg_chk_srv(ErtsHLTimerService *srv);
 #error "ERTS_REF_NUMBERS changed. Update me..."
 #endif
 
+typedef enum {
+    ERTS_TMR_BIF,
+    ERTS_TMR_PROC,
+    ERTS_TMR_PORT,
+    ERTS_TMR_CALLBACK
+} ErtsTmrType;
+
 #define ERTS_BIF_TIMER_SHORT_TIME 5000
 
 #ifdef ERTS_SMP
@@ -97,8 +104,9 @@ static void hdbg_chk_srv(ErtsHLTimerService *srv);
 #define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 13)
 #define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 14)
 #define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 15)
+#define ERTS_TMR_ROFLG_CALLBACK		(((Uint32) 1) << 16)
 #ifdef ERTS_BTM_ACCESSOR_SUPPORT
-#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 16)
+#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 17)
 #endif
 
 #define ERTS_TMR_ROFLG_SID_MASK	\
@@ -143,6 +151,7 @@ typedef struct {
     Uint32 roflgs;
     erts_smp_atomic32_t refc;
     union {
+	void *arg;
 	erts_atomic_t next;
     } u;
 } ErtsTmrHead;
@@ -158,6 +167,7 @@ struct ErtsHLTimer_ {
 	Process *proc;
 	Port *port;
 	Eterm name;
+	void (*callback)(void *);
     } receiver;
 
 #ifdef ERTS_HLT_HARD_DEBUG
@@ -192,7 +202,10 @@ struct ErtsHLTimer_ {
 
 typedef struct {
     ErtsTmrHead head; /* NEED to be first! */
-    void *p;
+    union {
+	void *p;
+	void (*callback)(void *);
+    } u;
     ErtsTWheelTimer tw_tmr;
 } ErtsTWTimer;
 
@@ -348,8 +361,8 @@ refn_is_lt(Uint32 *x, Uint32 *y)
 #define ERTS_RBT_WANT_SMALLEST
 #define ERTS_RBT_WANT_LOOKUP_INSERT
 #define ERTS_RBT_WANT_REPLACE
+#define ERTS_RBT_WANT_FOREACH
 #ifdef ERTS_HLT_HARD_DEBUG
-#  define ERTS_RBT_WANT_FOREACH
 #  define ERTS_RBT_WANT_LOOKUP
 #endif
 #define ERTS_RBT_UNDEF
@@ -460,8 +473,6 @@ same_time_list_foreach_destroy_yielding(ErtsHLTimer **root,
     }    
 }
 
-#ifdef ERTS_HLT_HARD_DEBUG
-
 static ERTS_INLINE void
 same_time_list_foreach(ErtsHLTimer *root,
 		       void (*op)(ErtsHLTimer *, void *),
@@ -476,6 +487,8 @@ same_time_list_foreach(ErtsHLTimer *root,
     }
 }
 
+#ifdef ERTS_HLT_HARD_DEBUG
+
 static ERTS_INLINE ErtsHLTimer *
 same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 {
@@ -761,9 +774,9 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr)
      * dropped at once...
      */
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC)
-	erts_proc_dec_refc((Process *) tmr->p);
-    else
-	erts_port_dec_refc((Port *) tmr->p);
+	erts_proc_dec_refc((Process *) tmr->u.p);
+    else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT)
+	erts_port_dec_refc((Port *) tmr->u.p);
 
     erts_schedule_thr_prgr_later_cleanup_op(
 	scheduled_tw_timer_destroy,
@@ -785,7 +798,7 @@ static void
 tw_proc_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Process *proc = (Process *) twtp->p;
+    Process *proc = (Process *) twtp->u.p;
     if (proc_timeout_common(proc, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -795,7 +808,7 @@ static void
 tw_port_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Port *port = (Port *) twtp->p;
+    Port *port = (Port *) twtp->u.p;
     if (port_timeout_common(port, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -815,13 +828,26 @@ cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr)
     erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr);
 }
 
+static void
+tw_callback_timeout(void *vtwtp)
+{
+    ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
+    void (*callback)(void *) = twtp->u.callback;
+    void *arg = twtp->head.u.arg;
+    tw_timer_dec_refc(twtp);
+    (*callback)(arg);
+}
+
 static ErtsTWTimer *
 create_tw_timer(ErtsSchedulerData *esdp, 
-		void *p, int is_proc,
+		ErtsTmrType type, void *p,
+		void (*callback)(void *), void *arg,
 		ErtsMonotonicTime timeout_pos)
 {
     ErtsTWTimer *tmr;
     void (*timeout_func)(void *);
+    void (*cancel_func)(void *);
+    erts_aint32_t refc;
 
     tmr = tw_timer_alloc();
     erts_twheel_init_timer(&tmr->tw_tmr);
@@ -829,24 +855,48 @@ create_tw_timer(ErtsSchedulerData *esdp,
     tmr->head.roflgs = (Uint32) esdp->no;
     ERTS_HLT_ASSERT((tmr->head.roflgs
 		     & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
-    tmr->p = p;
-    if (is_proc) {
+
+    switch (type) {
+
+    case ERTS_TMR_PROC:
+	tmr->u.p = p;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
 	timeout_func = tw_proc_timeout;
+	cancel_func = tw_ptimer_cancel;
 	erts_proc_inc_refc((Process *) p);
-    }
-    else {
+	refc = 2;
+	break;
+
+    case ERTS_TMR_PORT:
+	tmr->u.p = p;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT;
 	timeout_func = tw_port_timeout;
+	cancel_func = tw_ptimer_cancel;
 	erts_port_inc_refc((Port *) p);
+	refc = 2;
+	break;
+
+    case ERTS_TMR_CALLBACK:
+	tmr->head.u.arg = arg;
+	tmr->u.callback = callback;
+
+	tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK;
+	timeout_func = tw_callback_timeout;
+	cancel_func = NULL;
+	refc = 1;
+	break;
+
+    default:
+	ERTS_INTERNAL_ERROR("Unsupported timer type");
+	return NULL;
     }
 
-    erts_smp_atomic32_init_nob(&tmr->head.refc, 2);
+    erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
 
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &tmr->tw_tmr,
 			  timeout_func,
-			  tw_ptimer_cancel,
+			  cancel_func,
 			  tmr,
 			  timeout_pos);
 
@@ -994,17 +1044,15 @@ hlt_delete_abtm(ErtsHLTimer *tmr)
 static ErtsHLTimer *
 create_hl_timer(ErtsSchedulerData *esdp,
 		ErtsMonotonicTime timeout_pos,
-		int short_time, int is_bif_tmr,
+		int short_time, ErtsTmrType type,
 		void *rcvrp, Eterm rcvr, Eterm acsr,
-		Eterm msg, Uint32 *refn)
+		Eterm msg, Uint32 *refn,
+		void (*callback)(void *), void *arg)
 {
     ErtsHLTimerService *srv = esdp->timer_service;
     ErtsHLTimer *tmr, *st_tmr;
     erts_aint32_t refc;
     Uint32 roflgs;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    int is_abif_tmr = is_bif_tmr && is_value(acsr) && acsr != rcvr;
-#endif
 
     check_canceled_queue(esdp, srv);
 
@@ -1012,45 +1060,69 @@ create_hl_timer(ErtsSchedulerData *esdp,
 
     roflgs = ((Uint32) esdp->no) | ERTS_TMR_ROFLG_HLT;
 
-    if (!is_bif_tmr)
+    if (type != ERTS_TMR_BIF) {
+
 	tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER,
 			 ERTS_HL_PTIMER_SIZE);
-    else if (short_time) {
-	tmr = bif_timer_pre_alloc();
-	if (!tmr)
-	    goto alloc_bif_timer;
-	roflgs |= ERTS_TMR_ROFLG_PRE_ALC;
-    }
-    else {
-    alloc_bif_timer:
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	if (is_abif_tmr)
-	    tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER,
-			     ERTS_ABIF_TIMER_SIZE);
-	else
-#endif
-	    tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
-			     ERTS_BIF_TIMER_SIZE);
-    }
+	tmr->timeout = timeout_pos;
 
-    tmr->timeout = timeout_pos;
+	switch (type) {
+
+	case ERTS_TMR_PROC:
+	    ERTS_HLT_ASSERT(is_internal_pid(rcvr));
 
-    if (!is_bif_tmr) {
-	if (is_internal_pid(rcvr)) {
 	    erts_proc_inc_refc((Process *) rcvrp);
 	    tmr->receiver.proc = (Process *) rcvrp;
 	    roflgs |= ERTS_TMR_ROFLG_PROC;
-	}
-	else {
-	    erts_port_inc_refc((Port *) rcvrp);
+	    refc = 2;
+	    break;
+
+	case ERTS_TMR_PORT:
 	    ERTS_HLT_ASSERT(is_internal_port(rcvr));
+	    erts_port_inc_refc((Port *) rcvrp);
 	    tmr->receiver.port = (Port *) rcvrp;
 	    roflgs |= ERTS_TMR_ROFLG_PORT;
+	    refc = 2;
+	    break;
+
+	case ERTS_TMR_CALLBACK:
+	    roflgs |= ERTS_TMR_ROFLG_CALLBACK;
+	    tmr->receiver.callback = callback;
+	    tmr->head.u.arg = arg;
+	    refc = 1;
+	    break;
+
+	default:
+	    ERTS_INTERNAL_ERROR("Unsupported timer type");
+	    return NULL;
 	}
-	refc = 2;
+
     }
-    else {
+    else { /* ERTS_TMR_BIF */
 	Uint hsz;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+	int is_abif_tmr = is_value(acsr) && acsr != rcvr;
+#endif
+
+	if (short_time) {
+	    tmr = bif_timer_pre_alloc();
+	    if (!tmr)
+		goto alloc_bif_timer;
+	    roflgs |= ERTS_TMR_ROFLG_PRE_ALC;
+	}
+	else {
+	alloc_bif_timer:
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+	    if (is_abif_tmr)
+		tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER,
+				 ERTS_ABIF_TIMER_SIZE);
+	    else
+#endif
+		tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
+				 ERTS_BIF_TIMER_SIZE);
+	}
+
+	tmr->timeout = timeout_pos;
 
 	roflgs |= ERTS_TMR_ROFLG_BIF_TMR;
 	if (is_internal_pid(rcvr)) {
@@ -1081,6 +1153,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	tmr->btm.refn[2] = refn[2];
 
 	tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+
 #ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	if (is_abif_tmr) {
 	    Process *aproc;
@@ -1097,6 +1170,8 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	    }
 	}
 #endif
+
+	btm_rbt_insert(&srv->btm_tree, tmr);
     }
 
     tmr->head.roflgs = roflgs;
@@ -1124,9 +1199,6 @@ create_hl_timer(ErtsSchedulerData *esdp,
     if (st_tmr)
 	same_time_list_insert(&st_tmr->time.tree.same_time, tmr);
 
-    if (is_bif_tmr)
-	btm_rbt_insert(&srv->btm_tree, tmr);
-
 #ifdef ERTS_HLT_HARD_DEBUG
     tmr->pending_timeout = 0;
 #endif
@@ -1231,10 +1303,13 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv)
 	    hlt_bif_timer_timeout(tmr, roflgs);
 	else if (roflgs & ERTS_TMR_ROFLG_PROC)
 	    hlt_proc_timeout(tmr);
-	else {
-	    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PORT);
+	else if (roflgs & ERTS_TMR_ROFLG_PORT)
 	    hlt_port_timeout(tmr);
+	else {
+	    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK);
+	    (*tmr->receiver.callback)(tmr->head.u.arg);
 	}
+
     }
 
     tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
@@ -1700,8 +1775,9 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
 
     tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg;
 
-    tmr = create_hl_timer(esdp, timeout_pos, short_time, 1, NULL, 
-			  rcvr, acsr, tmo_msg, internal_ref_numbers(ref));
+    tmr = create_hl_timer(esdp, timeout_pos, short_time,
+			  ERTS_TMR_BIF, NULL,  rcvr, acsr, tmo_msg,
+			  internal_ref_numbers(ref), NULL, NULL);
 
     UnUseTmpHeap(4, c_p);
 
@@ -2538,6 +2614,80 @@ BIF_RETTYPE read_timer_2(BIF_ALIST_2)
     return ret;
 }
 
+static void
+start_callback_timer(ErtsSchedulerData *esdp,
+		     int twt,
+		     ErtsMonotonicTime timeout_pos,
+		     void (*callback)(void *),
+		     void *arg)
+		     
+{
+    if (twt)
+	create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL,
+			callback, arg, timeout_pos);
+    else
+	create_hl_timer(esdp, timeout_pos, 0,
+			ERTS_TMR_CALLBACK, NULL,
+			NIL, THE_NON_VALUE, NIL,
+			NULL, callback, arg);
+}
+
+typedef struct {
+    int twt;
+    ErtsMonotonicTime timeout_pos;
+    void (*callback)(void *);
+    void *arg;
+} ErtsStartCallbackTimerRequest;
+
+static void
+scheduled_start_callback_timer(void *vsctr)
+{
+    ErtsStartCallbackTimerRequest *sctr
+	= (ErtsStartCallbackTimerRequest *) vsctr;
+
+    start_callback_timer(erts_get_scheduler_data(),
+			 sctr->twt,
+			 sctr->timeout_pos,
+			 sctr->callback,	
+			 sctr->arg);
+
+    erts_free(ERTS_ALC_T_TIMER_REQUEST, vsctr);
+}
+
+void
+erts_start_timer_callback(ErtsMonotonicTime tmo,
+			  void (*callback)(void *),
+			  void *arg)
+{
+    ErtsSchedulerData *esdp;
+    ErtsMonotonicTime timeout_pos;
+    int twt;
+
+    esdp = erts_get_scheduler_data();
+    timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp),
+				  tmo);
+    twt = tmo < ERTS_TIMER_WHEEL_MSEC;
+
+    if (esdp)
+	start_callback_timer(esdp,
+			     twt,
+			     timeout_pos,
+			     callback,
+			     arg);
+    else {
+	ErtsStartCallbackTimerRequest *sctr;
+	sctr = erts_alloc(ERTS_ALC_T_TIMER_REQUEST,
+			  sizeof(ErtsStartCallbackTimerRequest));
+	sctr->twt = twt;
+	sctr->timeout_pos = timeout_pos;
+	sctr->callback = callback;
+	sctr->arg = arg;
+	erts_schedule_misc_aux_work(1,
+				    scheduled_start_callback_timer,
+				    (void *) sctr);
+    }
+}
+
 /*
  * Process and Port timer functionality.
  *
@@ -2561,12 +2711,13 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo,
 	c_p->flags &= ~F_TIMO;
 
 	if (tmo < ERTS_TIMER_WHEEL_MSEC)
-	    tmr = (void *) create_tw_timer(esdp, (void *) c_p, 1, timeout_pos);
+	    tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p,
+					   NULL, NULL, timeout_pos);
 	else
-	    tmr = (void *) create_hl_timer(esdp, timeout_pos,
-					   short_time, 0, (void *) c_p,
+	    tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time,
+					   ERTS_TMR_PROC, (void *) c_p,
 					   c_p->common.id, THE_NON_VALUE,
-					   NIL, NULL);
+					   NIL, NULL, NULL, NULL);
 	erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr);
     }
 }
@@ -2649,13 +2800,12 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
     timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo);
 
     if (tmo < ERTS_TIMER_WHEEL_MSEC)
-	tmr = (void *) create_tw_timer(esdp, (void *) c_prt, 0,
-				       timeout_pos);
+	tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt,
+				       NULL, NULL, timeout_pos);
     else
-	tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, 0,
-				       (void *) c_prt,
-				       c_prt->common.id, THE_NON_VALUE,
-				       NIL, NULL);
+	tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT,
+				       (void *) c_prt, c_prt->common.id,
+				       THE_NON_VALUE, NIL, NULL, NULL, NULL);
     erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
 }
 
@@ -2800,6 +2950,98 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm,
     }
 }
 
+typedef struct {
+    void (*tclbk)(void *);
+    void (*func)(void *,
+		 ErtsMonotonicTime,
+		 void *);
+    void *arg;
+} ErtsDebugForeachCallbackTimer;
+
+static void
+debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct)
+{
+    ErtsDebugForeachCallbackTimer *dfct
+	= (ErtsDebugForeachCallbackTimer *) vdfct;
+
+    if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
+	&& (tmr->receiver.callback && dfct->tclbk))
+	(*dfct->func)(dfct->arg,
+		      tmr->timeout,
+		      tmr->head.u.arg);
+}
+
+static void
+debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct)
+{
+    ErtsDebugForeachCallbackTimer *dfct
+	= (ErtsDebugForeachCallbackTimer *) vdfct;
+
+    if (tmr->time.tree.same_time)
+	same_time_list_foreach(tmr->time.tree.same_time,
+			       debug_callback_timer_foreach_list,
+			       vdfct);
+
+    if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
+	&& (tmr->receiver.callback && dfct->tclbk))
+	(*dfct->func)(dfct->arg,
+		      tmr->timeout,
+		      tmr->head.u.arg);
+}
+
+static void
+debug_tw_callback_timer(void *vdfct,
+			ErtsMonotonicTime timeout_pos,
+			void *vtwtp)
+{
+    ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
+    ErtsDebugForeachCallbackTimer *dfct
+	= (ErtsDebugForeachCallbackTimer *) vdfct;
+
+    if (twtp->u.callback == dfct->tclbk)
+	(*dfct->func)(dfct->arg,
+		      timeout_pos,
+		      twtp->head.u.arg);
+}
+
+void
+erts_debug_callback_timer_foreach(void (*tclbk)(void *),
+				  void (*func)(void *,
+					       ErtsMonotonicTime,
+					       void *),
+				  void *arg)
+{
+    int six;
+    ErtsDebugForeachCallbackTimer dfct;
+
+    dfct.tclbk = tclbk;
+    dfct.func = func;
+    dfct.arg = arg;
+
+    if (!erts_smp_thr_progress_is_blocking())
+	ERTS_INTERNAL_ERROR("Not blocking thread progress");
+
+    for (six = 0; six < erts_no_schedulers; six++) {
+	ErtsHLTimerService *srv =
+	    erts_aligned_scheduler_data[six].esd.timer_service;
+	ErtsTimerWheel *twheel =
+	    erts_aligned_scheduler_data[six].esd.timer_wheel;
+
+	erts_twheel_debug_foreach(twheel,
+				  tw_callback_timeout,
+				  debug_tw_callback_timer,
+				  (void *) &dfct);
+
+	if (srv->yield.root)
+	    debug_callback_timer_foreach(srv->yield.root,
+					 (void *) &dfct);
+
+	time_rbt_foreach(srv->btm_tree,
+			 debug_callback_timer_foreach,
+			 (void *) &dfct);
+    }
+}
+
 #ifdef ERTS_HLT_HARD_DEBUG
 
 typedef struct {
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 30889a71da..24c57fc873 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -59,7 +59,9 @@ int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **);
 int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **);
 ErtsHLTimerService *erts_create_timer_service(void);
 void erts_hl_timer_init(void);
-
+void erts_start_timer_callback(ErtsMonotonicTime,
+			       void (*)(void *),
+			       void *);
 #ifdef ERTS_SMP
 void
 erts_handle_canceled_timers(void *vesdp,
@@ -76,5 +78,10 @@ void erts_debug_bif_timer_foreach(void (*func)(Eterm,
 					       ErlHeapFragment *,
 					       void *),
 				  void *arg);
-
+void
+erts_debug_callback_timer_foreach(void (*tclbk)(void *),
+				  void (*func)(void *,
+					       ErtsMonotonicTime,
+					       void *),
+				  void *arg);
 #endif /* ERL_HL_TIMER_H__ */
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 4560cd23af..e64b86496a 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -453,4 +453,12 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+void
+erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
+			  void (*tclbk)(void *),
+			  void (*func)(void *,
+				       ErtsMonotonicTime,
+				       void *),
+			  void *arg);
+
 #endif /* timer wheel api */
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 8bffdedb2b..fe7c5826f5 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -529,6 +529,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
     tiw->sentinel.next = &tiw->sentinel;
     tiw->sentinel.prev = &tiw->sentinel;
+    tiw->sentinel.u.func.timeout = NULL;
+    tiw->sentinel.u.func.cancel = NULL;
+    tiw->sentinel.u.func.arg = NULL;
     return tiw;
 }
 
@@ -624,6 +627,41 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
     }
 }
 
+void
+erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
+			  void (*tclbk)(void *),
+			  void (*func)(void *,
+				       ErtsMonotonicTime,
+				       void *),
+			  void *arg)
+{
+    ErtsTWheelTimer *tmr;
+    int ix;
+
+    tmr = tiw->sentinel.next;
+    while (tmr != &tiw->sentinel) {
+	if (tmr->u.func.timeout == tclbk)
+	    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+	tmr = tmr->next;
+    }
+
+    for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) {
+	if (tmr->u.func.timeout == tclbk)
+	    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+    }
+
+    for (ix = 0; ix < ERTS_TIW_SIZE; ix++) {
+	tmr = tiw->w[ix];
+	if (tmr) {
+	    do {
+		if (tmr->u.func.timeout == tclbk)
+		    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+		tmr = tmr->next;
+	    } while (tmr != tiw->w[ix]);
+	}
+    }
+}
+
 #ifdef ERTS_TW_DEBUG
 void erts_p_slpq(void)
 {
-- 
cgit v1.2.3


From 014865c474c4e8229cae556d9dff22ba75d6a5bb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 28 May 2015 10:06:34 +0200
Subject: Only read lock node table when reading info

---
 erts/emulator/beam/erl_node_tables.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index bcf6311079..900fdb3ab5 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -556,14 +556,14 @@ erts_node_table_size(void)
 #endif
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
-	erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
+	erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
 #ifdef DEBUG
     hash_get_info(&hi, &erts_node_table);
     ASSERT(node_entries == hi.objs);
 #endif
     res = hash_table_sz(&erts_node_table) + node_entries*sizeof(ErlNode);
     if (lock)
-	erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+	erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
     return res;
 }
 
@@ -572,10 +572,10 @@ erts_node_table_info(int to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
-	erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
+	erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
     hash_info(to, to_arg, &erts_node_table);
     if (lock)
-	erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+	erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
 }
 
 
@@ -674,13 +674,13 @@ void erts_print_node_info(int to,
     pnd.no_total = 0;
 
     if (lock)
-	erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
+	erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
     hash_foreach(&erts_node_table, print_node, (void *) &pnd);
     if (pnd.no_sysname != 0) {
 	erts_print(to, to_arg, "\n");
     }
     if (lock)
-	erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+	erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
 
     if(no_sysname)
 	*no_sysname = pnd.no_sysname;
-- 
cgit v1.2.3


From 3219822bb549a9d0d40ff29463f0bfe7d1eedd37 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Sun, 26 Apr 2015 16:29:53 +0200
Subject: Fix segfault in module_info for deleted modules

Add a check to protect from segfault when erlang:get_module_info/1/2 is
called on a deleted module (i.e. with no current code). Also refactor
erts_module_info_0/1 to avoid repeated calls to erts_active_code_ix() and
remove some obsolete comments. Add test for module_info on deleted modules.
---
 erts/emulator/beam/beam_load.c | 181 ++++++++++++++---------------------------
 1 file changed, 63 insertions(+), 118 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0d40201934..41ce1659d4 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -525,13 +525,16 @@ static void new_literal_patch(LoaderState* stp, int pos);
 static void new_string_patch(LoaderState* stp, int pos);
 static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size);
 static int genopargcompare(GenOpArg* a, GenOpArg* b);
-static Eterm exported_from_module(Process* p, Eterm mod);
-static Eterm functions_in_module(Process* p, Eterm mod);
-static Eterm attributes_for_module(Process* p, Eterm mod);
-static Eterm compilation_info_for_module(Process* p, Eterm mod);
-static Eterm md5_of_module(Process* p, Eterm mod);
-static Eterm has_native(Process* p, Eterm mod);
-static Eterm native_addresses(Process* p, Eterm mod);
+static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix,
+                             BeamInstr* code, Eterm module, Eterm what);
+static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix,
+                                  Eterm mod);
+static Eterm functions_in_module(Process* p, BeamInstr* code);
+static Eterm attributes_for_module(Process* p, BeamInstr* code);
+static Eterm compilation_info_for_module(Process* p, BeamInstr* code);
+static Eterm md5_of_module(Process* p, BeamInstr* code);
+static Eterm has_native(BeamInstr* code);
+static Eterm native_addresses(Process* p, BeamInstr* code);
 int patch_funentries(Eterm Patchlist);
 int patch(Eterm Addresses, Uint fe);
 static int safe_mul(UWord a, UWord b, UWord* resp);
@@ -5389,6 +5392,9 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
 Eterm
 erts_module_info_0(Process* p, Eterm module)
 {
+    Module* modp;
+    ErtsCodeIndex code_ix = erts_active_code_ix();
+    BeamInstr* code;
     Eterm *hp;
     Eterm list = NIL;
     Eterm tup;
@@ -5397,12 +5403,18 @@ erts_module_info_0(Process* p, Eterm module)
 	return THE_NON_VALUE;
     }
 
-    if (erts_get_module(module, erts_active_code_ix()) == NULL) {
+    modp = erts_get_module(module, code_ix);
+    if (modp == NULL) {
 	return THE_NON_VALUE;
     }
 
+    code = modp->curr.code;
+    if (code == NULL) {
+        return THE_NON_VALUE;
+    }
+
 #define BUILD_INFO(What) \
-    tup = erts_module_info_1(p, module, What); \
+    tup = get_module_info(p, code_ix, code, module, What); \
     hp = HAlloc(p, 5); \
     tup = TUPLE2(hp, What, tup); \
     hp += 3; \
@@ -5422,23 +5434,48 @@ erts_module_info_0(Process* p, Eterm module)
 
 Eterm
 erts_module_info_1(Process* p, Eterm module, Eterm what)
+{
+    Module* modp;
+    ErtsCodeIndex code_ix = erts_active_code_ix();
+    BeamInstr* code;
+
+    if (is_not_atom(module)) {
+        return THE_NON_VALUE;
+    }
+
+    modp = erts_get_module(module, code_ix);
+    if (modp == NULL) {
+        return THE_NON_VALUE;
+    }
+
+    code = modp->curr.code;
+    if (code == NULL) {
+        return THE_NON_VALUE;
+    }
+
+    return get_module_info(p, code_ix, code, module, what);
+}
+
+static Eterm
+get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code,
+                Eterm module, Eterm what)
 {
     if (what == am_module) {
 	return module;
     } else if (what == am_md5) {
-	return md5_of_module(p, module);
+	return md5_of_module(p, code);
     } else if (what == am_exports) {
-	return exported_from_module(p, module);
+	return exported_from_module(p, code_ix, module);
     } else if (what == am_functions) {
-	return functions_in_module(p, module);
+	return functions_in_module(p, code);
     } else if (what == am_attributes) {
-	return attributes_for_module(p, module);
+	return attributes_for_module(p, code);
     } else if (what == am_compile) {
-	return compilation_info_for_module(p, module);
+	return compilation_info_for_module(p, code);
     } else if (what == am_native_addresses) {
-	return native_addresses(p, module);
+	return native_addresses(p, code);
     } else if (what == am_native) {
-	return has_native(p, module);
+	return has_native(code);
     }
     return THE_NON_VALUE;
 }
@@ -5446,16 +5483,12 @@ erts_module_info_1(Process* p, Eterm module, Eterm what)
 /*
  * Builds a list of all functions in the given module:
  *     [{Name, Arity},...]
- *
- * Returns a tagged term, or 0 on error.
  */
 
 Eterm
 functions_in_module(Process* p, /* Process whose heap to use. */
-		     Eterm mod) /* Tagged atom for module. */
+		    BeamInstr* code)
 {
-    Module* modp;
-    BeamInstr* code;
     int i;
     Uint num_functions;
     Uint need;
@@ -5463,15 +5496,6 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     Eterm* hp_end;
     Eterm result = NIL;
 
-    if (is_not_atom(mod)) {
-	return THE_NON_VALUE;
-    }
-
-    modp = erts_get_module(mod, erts_active_code_ix());
-    if (modp == NULL) {
-	return THE_NON_VALUE;
-    }
-    code = modp->curr.code;
     num_functions = code[MI_NUM_FUNCTIONS];
     need = 5*num_functions;
     hp = HAlloc(p, need);
@@ -5503,23 +5527,12 @@ functions_in_module(Process* p, /* Process whose heap to use. */
  */
 
 static Eterm
-has_native(Process* p, Eterm mod)
+has_native(BeamInstr *code)
 {
     Eterm result = am_false;
 #ifdef HIPE
-    Module* modp;
-
-    if (is_not_atom(mod)) {
-        return THE_NON_VALUE;
-    }
-
-    modp = erts_get_module(mod, erts_active_code_ix());
-    if (modp == NULL) {
-        return THE_NON_VALUE;
-    }
-
-    if (erts_is_module_native(modp->curr.code)) {
-      result = am_true;
+    if (erts_is_module_native(code)) {
+        result = am_true;
     }
 #endif
     return result;
@@ -5548,15 +5561,11 @@ erts_is_module_native(BeamInstr* code)
 /*
  * Builds a list of all functions including native addresses.
  *     [{Name,Arity,NativeAddress},...]
- *
- * Returns a tagged term, or 0 on error.
  */
 
 static Eterm
-native_addresses(Process* p, Eterm mod)
+native_addresses(Process* p, BeamInstr* code)
 {
-    Module* modp;
-    BeamInstr* code;
     int i;
     Eterm* hp;
     Uint num_functions;
@@ -5564,16 +5573,6 @@ native_addresses(Process* p, Eterm mod)
     Eterm* hp_end;
     Eterm result = NIL;
 
-    if (is_not_atom(mod)) {
-	return THE_NON_VALUE;
-    }
-
-    modp = erts_get_module(mod, erts_active_code_ix());
-    if (modp == NULL) {
-	return THE_NON_VALUE;
-    }
-
-    code = modp->curr.code;
     num_functions = code[MI_NUM_FUNCTIONS];
     need = (6+BIG_UINT_HEAP_SIZE)*num_functions;
     hp = HAlloc(p, need);
@@ -5600,25 +5599,18 @@ native_addresses(Process* p, Eterm mod)
 /*
  * Builds a list of all exported functions in the given module:
  *     [{Name, Arity},...]
- *
- * Returns a tagged term, or 0 on error.
  */
 
 Eterm
 exported_from_module(Process* p, /* Process whose heap to use. */
+                     ErtsCodeIndex code_ix,
 		     Eterm mod) /* Tagged atom for module. */
 {
     int i;
     Eterm* hp = NULL;
     Eterm* hend = NULL;
     Eterm result = NIL;
-    ErtsCodeIndex code_ix;
-
-    if (is_not_atom(mod)) {
-	return THE_NON_VALUE;
-    }
 
-    code_ix = erts_active_code_ix();
     for (i = 0; i < export_list_size(code_ix); i++) {
 	Export* ep = export_list(i,code_ix);
 	
@@ -5648,31 +5640,17 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 
 /*
  * Returns a list of all attributes for the module.
- *
- * Returns a tagged term, or 0 on error.
  */
 
 Eterm
 attributes_for_module(Process* p, /* Process whose heap to use. */
-		      Eterm mod) /* Tagged atom for module. */
-
+                      BeamInstr* code)
 {
-    Module* modp;
-    BeamInstr* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
     Eterm* end;
 
-    if (is_not_atom(mod)) {
-	return THE_NON_VALUE;
-    }
-
-    modp = erts_get_module(mod, erts_active_code_ix());
-    if (modp == NULL) {
-	return THE_NON_VALUE;
-    }
-    code = modp->curr.code;
     ext = (byte *) code[MI_ATTR_PTR];
     if (ext != NULL) {
 	hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]);
@@ -5688,30 +5666,17 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
 
 /*
  * Returns a list containing compilation information.
- *
- * Returns a tagged term, or 0 on error.
  */
 
 Eterm
 compilation_info_for_module(Process* p, /* Process whose heap to use. */
-			    Eterm mod) /* Tagged atom for module. */
+                            BeamInstr* code)
 {
-    Module* modp;
-    BeamInstr* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
     Eterm* end;
 
-    if (is_not_atom(mod)) {
-	return THE_NON_VALUE;
-    }
-
-    modp = erts_get_module(mod, erts_active_code_ix());
-    if (modp == NULL) {
-	return THE_NON_VALUE;
-    }
-    code = modp->curr.code;
     ext = (byte *) code[MI_COMPILE_PTR];
     if (ext != NULL) {
 	hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]);
@@ -5727,33 +5692,13 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 
 /*
  * Returns the MD5 checksum for a module
- *
- * Returns a tagged term, or 0 on error.
  */
 
 Eterm
 md5_of_module(Process* p, /* Process whose heap to use. */
-              Eterm mod) /* Tagged atom for module. */
+              BeamInstr* code)
 {
-    Module* modp;
-    BeamInstr* code;
-    Eterm res = NIL;
-
-    if (is_not_atom(mod)) {
-	return THE_NON_VALUE;
-    }
-
-    modp = erts_get_module(mod, erts_active_code_ix());
-    if (modp == NULL) {
-	return THE_NON_VALUE;
-    }
-    code = modp->curr.code;
-    if (code[MI_MD5_PTR] != 0) {
-      res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE);
-    } else {
-      res = am_undefined;
-    }
-    return res;
+    return new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE);
 }
 
 /*
-- 
cgit v1.2.3


From fe1c0d26d4e6180b79fc8497b827ac2ef1f471d5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 5 Jun 2015 17:25:22 +0200
Subject: Delayed node table GC

---
 erts/emulator/beam/dist.c            |   2 +-
 erts/emulator/beam/erl_bif_info.c    |  20 ++++
 erts/emulator/beam/erl_init.c        |  36 +++++--
 erts/emulator/beam/erl_node_tables.c | 202 +++++++++++++++++++++++++++++------
 erts/emulator/beam/erl_node_tables.h |  16 ++-
 5 files changed, 233 insertions(+), 43 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 142fcb3c00..cfdede793c 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2522,7 +2522,7 @@ info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected)
 
   erts_print(to, arg, "Name: %T", dep->sysname);
 #ifdef DEBUG
-  erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 1));
+  erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0));
 #endif
   erts_print(to, arg, "\n");
   if (!connected && is_nil(dep->cid)) {
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index ae8a5c266a..23fc4f915e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2687,6 +2687,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
 	res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit);
 	BIF_RET(res);
+    } else if (ERTS_IS_ATOM_STR("delayed_node_table_gc", BIF_ARG_1)) {
+	Uint hsz = 0;
+	Uint dntgc = erts_delayed_node_table_gc();
+	if (dntgc == ERTS_NODE_TAB_DELAY_GC_INFINITY)
+	    BIF_RET(am_infinity);
+ 	(void) erts_bld_uint(NULL, &hsz, dntgc);
+	hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
+	res = erts_bld_uint(&hp, NULL, dntgc);
+	BIF_RET(res);
     } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) {
 	BIF_RET(erts_get_ethread_info(BIF_P));
     }
@@ -4069,6 +4078,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2);
 	    BIF_RET(res ? am_true : am_false);
 	}
+	else if (ERTS_IS_ATOM_STR("node_tab_delayed_delete", BIF_ARG_1)) {
+	    /* node_container_SUITE */
+	    Sint64 msecs;
+	    if (term_to_Sint64(BIF_ARG_2, &msecs)) {
+		/* Negative value restore original value... */
+		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		erts_debug_test_node_tab_delayed_delete(msecs);
+		erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		BIF_RET(am_ok);
+	    }
+	}
     }
 
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 988ff0e2b5..a2d2a3befc 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -141,7 +141,8 @@ static void erl_init(int ncpu,
 		     int port_tab_sz_ignore_files,
 		     int legacy_port_tab,
 		     int time_correction,
-		     ErtsTimeWarpMode time_warp_mode);
+		     ErtsTimeWarpMode time_warp_mode,
+		     int node_tab_delete_delay);
 
 static erts_atomic_t exiting;
 
@@ -314,7 +315,8 @@ erts_short_init(void)
 	     0,
 	     0,
 	     time_correction,
-	     time_warp_mode);
+	     time_warp_mode,
+	     ERTS_NODE_TAB_DELAY_GC_DEFAULT);
     erts_initialized = 1;
 }
 
@@ -326,7 +328,8 @@ erl_init(int ncpu,
 	 int port_tab_sz_ignore_files,
 	 int legacy_port_tab,
 	 int time_correction,
-	 ErtsTimeWarpMode time_warp_mode)
+	 ErtsTimeWarpMode time_warp_mode,
+	 int node_tab_delete_delay)
 {
     init_benchmarking();
 
@@ -367,7 +370,7 @@ erl_init(int ncpu,
     erts_init_binary(); /* Must be after init_emulator() */
     erts_bp_init();
     init_db(); /* Must be after init_emulator */
-    erts_init_node_tables();
+    erts_init_node_tables(node_tab_delete_delay);
     init_dist();
     erl_drv_thr_init();
     erts_init_async();
@@ -629,6 +632,9 @@ void erts_usage(void)
     erts_fprintf(stderr, "               see error_logger documentation for details\n");
     erts_fprintf(stderr, "-zdbbl size    set the distribution buffer busy limit in kilobytes\n");
     erts_fprintf(stderr, "               valid range is [1-%d]\n", INT_MAX/1024);
+    erts_fprintf(stderr, "-zdntgc time   set delayed node table gc in seconds\n");
+    erts_fprintf(stderr, "               valid values are infinity or intergers in the range [0-%d]\n",
+		 ERTS_NODE_TAB_DELAY_GC_MAX);
     erts_fprintf(stderr, "\n");
     erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n");
     erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n");
@@ -1219,6 +1225,7 @@ erl_start(int argc, char **argv)
     int legacy_port_tab = 0;
     int time_correction;
     ErtsTimeWarpMode time_warp_mode;
+    int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT;
 
     set_default_time_adj(&time_correction,
 			 &time_warp_mode);
@@ -2004,9 +2011,9 @@ erl_start(int argc, char **argv)
 
 	case 'z': {
 	    char *sub_param = argv[i]+2;
-	    int new_limit;
 
 	    if (has_prefix("dbbl", sub_param)) {
+		int new_limit;
 		arg = get_arg(sub_param+4, argv[i+1], &i);
 		new_limit = atoi(arg);
 		if (new_limit < 1 || INT_MAX/1024 < new_limit) {
@@ -2015,6 +2022,22 @@ erl_start(int argc, char **argv)
 		} else {
 		    erts_dist_buf_busy_limit = new_limit*1024;
 		}
+	    }
+	    else if (has_prefix("dntgc", sub_param)) {
+		long secs;
+
+		arg = get_arg(sub_param+5, argv[i+1], &i);
+		if (sys_strcmp(arg, "infinity") == 0)
+		    secs = ERTS_NODE_TAB_DELAY_GC_INFINITY;
+		else {
+		    errno = 0;
+		    secs = strtol(arg, NULL, 10);
+		    if (errno != 0 || secs < 0 || ERTS_NODE_TAB_DELAY_GC_MAX < secs) {
+			erts_fprintf(stderr, "Invalid delayed node table gc: %ld\n", secs);
+			erts_usage();
+		    }
+		}
+		node_tab_delete_delay = (int) secs;
 	    } else {
 		erts_fprintf(stderr, "bad -z option %s\n", argv[i]);
 		erts_usage();
@@ -2088,7 +2111,8 @@ erl_start(int argc, char **argv)
 	     port_tab_sz_ignore_files,
 	     legacy_port_tab,
 	     time_correction,
-	     time_warp_mode);
+	     time_warp_mode,
+	     node_tab_delete_delay);
 
     load_preloaded();
     erts_end_staging_code_ix();
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 900fdb3ab5..c2acaea0bc 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -51,6 +51,9 @@ static Uint dist_entries;
 
 static int references_atoms_need_init = 1;
 
+static ErtsMonotonicTime orig_node_tab_delete_delay;
+static ErtsMonotonicTime node_tab_delete_delay;
+
 /* -- The distribution table ---------------------------------------------- */
 
 #ifdef DEBUG
@@ -290,21 +293,46 @@ DistEntry *erts_find_dist_entry(Eterm sysname)
     return res;
 }
 
-void erts_delete_dist_entry(DistEntry *dep)
+static void try_delete_dist_entry(void *vdep)
+{
+    DistEntry *dep = (DistEntry *) vdep;
+    erts_aint_t refc;
+
+    erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+    /*
+     * Another thread might have looked up this dist entry after
+     * we decided to delete it (refc became zero). If so, the other
+     * thread incremented refc twice. Once for the new reference
+     * and once for this thread.
+     *
+     * If refc reach -1, noone has used the entry since we
+     * set up the timer. Delete the entry.
+     *
+     * If refc reach 0, the entry is currently not in use
+     * but has been used since we set up the timer. Set up a
+     * new timer.
+     *
+     * If refc > 0, the entry is in use. Keep the entry.
+     */
+    refc = erts_refc_dectest(&dep->refc, -1);
+    if (refc == -1)
+	(void) hash_erase(&erts_dist_table, (void *) dep);
+    erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+
+    if (refc == 0)
+	erts_schedule_delete_dist_entry(dep);
+}
+
+void erts_schedule_delete_dist_entry(DistEntry *dep)
 {
     ASSERT(dep != erts_this_dist_entry);
-    if(dep != erts_this_dist_entry) {
-	erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
-	/*
-	 * Another thread might have looked up this dist entry after
-	 * we decided to delete it (refc became zero). If so, the other
-	 * thread incremented refc twice. Once for the new reference
-	 * and once for this thread. Therefore, delete dist entry if
-	 * refc is 0 or -1 after a decrement.
-	 */
-	if (erts_refc_dectest(&dep->refc, -1) <= 0)
-	    (void) hash_erase(&erts_dist_table, (void *) dep);
-	erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+    if (dep != erts_this_dist_entry) {
+	if (node_tab_delete_delay == 0)
+	    try_delete_dist_entry((void *) dep);
+	else if (node_tab_delete_delay > 0)
+	    erts_start_timer_callback(node_tab_delete_delay,
+				      try_delete_dist_entry,
+				      (void *) dep);
     }
 }
 
@@ -609,21 +637,46 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation)
     return res;
 }
 
-void erts_delete_node(ErlNode *enp)
+static void try_delete_node(void *venp)
+{
+    ErlNode *enp = (ErlNode *) venp;
+    erts_aint_t refc;
+
+    erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
+    /*
+     * Another thread might have looked up this node after we
+     * decided to delete it (refc became zero). If so, the other
+     * thread incremented refc twice. Once for the new reference
+     * and once for this thread.
+     *
+     * If refc reach -1, noone has used the entry since we
+     * set up the timer. Delete the entry.
+     *
+     * If refc reach 0, the entry is currently not in use
+     * but has been used since we set up the timer. Set up a
+     * new timer.
+     *
+     * If refc > 0, the entry is in use. Keep the entry.
+     */
+    refc = erts_refc_dectest(&enp->refc, -1);
+    if (refc == -1)
+	(void) hash_erase(&erts_node_table, (void *) enp);
+    erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+
+    if (refc == 0)
+	erts_schedule_delete_node(enp);
+}
+
+void erts_schedule_delete_node(ErlNode *enp)
 {
     ASSERT(enp != erts_this_node);
-    if(enp != erts_this_node) {
-	erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
-	/*
-	 * Another thread might have looked up this node after we
-	 * decided to delete it (refc became zero). If so, the other
-	 * thread incremented refc twice. Once for the new reference
-	 * and once for this thread. Therefore, delete node if refc
-	 * is 0 or -1 after a decrement.
-	 */
-	if (erts_refc_dectest(&enp->refc, -1) <= 0)
-	    (void) hash_erase(&erts_node_table, (void *) enp);
-	erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+    if (enp != erts_this_node) {
+	if (node_tab_delete_delay == 0)
+	    try_delete_node((void *) enp);
+	else if (node_tab_delete_delay > 0)
+	    erts_start_timer_callback(node_tab_delete_delay,
+				      try_delete_node,
+				      (void *) enp);
     }
 }
 
@@ -651,7 +704,7 @@ static void print_node(void *venp, void *vpndp)
 	erts_print(pndp->to, pndp->to_arg, " %d", enp->creation);
 #ifdef DEBUG
 	erts_print(pndp->to, pndp->to_arg, " (refc=%ld)",
-		   erts_refc_read(&enp->refc, 1));
+		   erts_refc_read(&enp->refc, 0));
 #endif
 	pndp->no_sysname++;
     }
@@ -714,11 +767,28 @@ erts_set_this_node(Eterm sysname, Uint creation)
 
 }
 
-void erts_init_node_tables(void)
+Uint
+erts_delayed_node_table_gc(void)
+{
+    if (node_tab_delete_delay < 0)
+	return (Uint) ERTS_NODE_TAB_DELAY_GC_INFINITY;
+    if (node_tab_delete_delay == 0)
+	return (Uint) 0;
+    return (Uint) ((node_tab_delete_delay-1)/1000 + 1);
+}
+
+void erts_init_node_tables(int dd_sec)
 {
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     HashFunctions f;
 
+    if (dd_sec == ERTS_NODE_TAB_DELAY_GC_INFINITY)
+	node_tab_delete_delay = (ErtsMonotonicTime) -1;
+    else
+	node_tab_delete_delay = ((ErtsMonotonicTime) dd_sec)*1000;
+
+    orig_node_tab_delete_delay = node_tab_delete_delay;
+
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
@@ -847,6 +917,7 @@ static Eterm AM_dist_references;
 static Eterm AM_node_references;
 static Eterm AM_system;
 static Eterm AM_timer;
+static Eterm AM_delayed_delete_timer;
 
 static void setup_reference_table(void);
 static Eterm reference_table_term(Uint **hpp, Uint *szp);
@@ -881,8 +952,10 @@ typedef struct dist_referrer_ {
     int heap_ref;
     int node_ref;
     int ctrl_ref;
+    int system_ref;
     Eterm id;
     Uint creation;
+    Uint id_heap[ID_HEAP_SIZE];
 } DistReferrer;
 
 typedef struct {
@@ -931,6 +1004,7 @@ erts_get_node_and_dist_references(struct process *proc)
 	INIT_AM(node_references);
 	INIT_AM(timer);
 	INIT_AM(system);
+	INIT_AM(delayed_delete_timer);
 	references_atoms_need_init = 0;
     }
 
@@ -988,17 +1062,25 @@ insert_dist_referrer(ReferredDist *referred_dist,
 					  sizeof(DistReferrer));
 	drp->next = referred_dist->referrers;
 	referred_dist->referrers = drp;
-	drp->id = id;
+	if(IS_CONST(id))
+	    drp->id = id;
+	else {
+	    Uint *hp = &drp->id_heap[0];
+	    ASSERT(is_tuple(id));
+	    drp->id = copy_struct(id, size_object(id), &hp, NULL);
+	}
 	drp->creation = creation;
 	drp->heap_ref = 0;
 	drp->node_ref = 0;
 	drp->ctrl_ref = 0;
+	drp->system_ref = 0;
     }
 
     switch (type) {
     case NODE_REF:	drp->node_ref++;	break;
     case CTRL_REF:	drp->ctrl_ref++;	break;
     case HEAP_REF:	drp->heap_ref++;	break;
+    case SYSTEM_REF:	drp->system_ref++;	break;
     default:		ASSERT(0);
     }
 }
@@ -1260,6 +1342,33 @@ insert_sys_msg(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp)
 }
 #endif
 
+static void
+insert_delayed_delete_node(void *state,
+			   ErtsMonotonicTime timeout_pos,
+			   void *vnp)
+{
+    DeclareTmpHeapNoproc(heap,3);
+    UseTmpHeapNoproc(3);
+    insert_node((ErlNode *) vnp,
+		SYSTEM_REF,
+		TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer));
+    UnUseTmpHeapNoproc(3);
+}
+
+static void
+insert_delayed_delete_dist_entry(void *state,
+				 ErtsMonotonicTime timeout_pos,
+				 void *vdep)
+{
+    DeclareTmpHeapNoproc(heap,3);
+    UseTmpHeapNoproc(3);
+    insert_dist_entry((DistEntry *) vdep,
+		      SYSTEM_REF,
+		      TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer),
+		      0);
+    UnUseTmpHeapNoproc(3);
+}
+
 static void
 setup_reference_table(void)
 {
@@ -1288,6 +1397,13 @@ setup_reference_table(void)
     /* Go through the hole system, and build a table of all references
        to ErlNode and DistEntry structures */
 
+    erts_debug_callback_timer_foreach(try_delete_node,
+				      insert_delayed_delete_node,
+				      NULL);
+    erts_debug_callback_timer_foreach(try_delete_dist_entry,
+				      insert_delayed_delete_dist_entry,
+				      NULL);
+
     UseTmpHeapNoproc(3);
     insert_node(erts_this_node,
 		SYSTEM_REF,
@@ -1601,7 +1717,7 @@ reference_table_term(Uint **hpp, Uint *szp)
 
 	tup = MK_2TUP(referred_nodes[i].node->sysname,
 		      MK_UINT(referred_nodes[i].node->creation));
-	tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 1)), nril);
+	tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril);
 	nl = MK_CONS(tup, nl);
     }
 
@@ -1624,6 +1740,10 @@ reference_table_term(Uint **hpp, Uint *szp)
 		tup = MK_2TUP(AM_heap, MK_UINT(drp->heap_ref));
 		drl = MK_CONS(tup, drl);
 	    }
+	    if(drp->system_ref) {
+		tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref));
+		drl = MK_CONS(tup, drl);
+	    }
 
 	    if (is_internal_pid(drp->id)) {
 		ASSERT(!drp->node_ref);
@@ -1633,6 +1753,14 @@ reference_table_term(Uint **hpp, Uint *szp)
 		ASSERT(drp->ctrl_ref && !drp->node_ref);
 		tup = MK_2TUP(AM_port, drp->id);
 	    }
+	    else if (is_tuple(drp->id)) {
+		Eterm *t;
+		ASSERT(drp->system_ref && !drp->node_ref
+		       && !drp->ctrl_ref && !drp->heap_ref);
+		t = tuple_val(drp->id);
+		ASSERT(2 == arityval(t[0]));
+		tup = MK_2TUP(t[1], t[2]);
+	    }
 	    else {
 		ASSERT(!drp->ctrl_ref && drp->node_ref);
 		ASSERT(is_atom(drp->id));
@@ -1650,7 +1778,7 @@ reference_table_term(Uint **hpp, Uint *szp)
 
 	/* DistList = [{Dist, Refc, ReferenceIdList}] */
 	tup = MK_3TUP(referred_dists[i].dist->sysname,
-		      MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 1)),
+		      MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)),
 		      dril);
 	dl = MK_CONS(tup, dl);
     }
@@ -1705,3 +1833,15 @@ delete_reference_table(void)
     }
 }
 
+void
+erts_debug_test_node_tab_delayed_delete(Sint64 millisecs)
+{
+    erts_smp_thr_progress_block();
+
+    if (millisecs < 0)
+	node_tab_delete_delay = orig_node_tab_delete_delay;
+    else
+	node_tab_delete_delay = millisecs;
+
+    erts_smp_thr_progress_unblock();
+}
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index af60071ea5..54c5cd1d11 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -46,6 +46,10 @@
 #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__
 #include "erl_port_task.h"
 #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__
+
+#define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60)
+#define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000)
+#define ERTS_NODE_TAB_DELAY_GC_INFINITY (ERTS_NODE_TAB_DELAY_GC_MAX+1)
  
 #define ERST_INTERNAL_CHANNEL_NO 0
 
@@ -166,20 +170,21 @@ extern DistEntry *erts_this_dist_entry;
 extern ErlNode *erts_this_node;
 extern char *erts_this_node_sysname; /* must match erl_node_tables.c */
 
+Uint erts_delayed_node_table_gc(void);
 DistEntry *erts_channel_no_to_dist_entry(Uint);
 DistEntry *erts_sysname_to_connected_dist_entry(Eterm);
 DistEntry *erts_find_or_insert_dist_entry(Eterm);
 DistEntry *erts_find_dist_entry(Eterm);
-void erts_delete_dist_entry(DistEntry *);
+void erts_schedule_delete_dist_entry(DistEntry *);
 Uint erts_dist_table_size(void);
 void erts_dist_table_info(int, void *);
 void erts_set_dist_entry_not_connected(DistEntry *);
 void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint);
 ErlNode *erts_find_or_insert_node(Eterm, Uint);
-void erts_delete_node(ErlNode *);
+void erts_schedule_delete_node(ErlNode *);
 void erts_set_this_node(Eterm, Uint);
 Uint erts_node_table_size(void);
-void erts_init_node_tables(void);
+void erts_init_node_tables(int);
 void erts_node_table_info(int, void *);
 void erts_print_node_info(int, void *, Eterm, int*, int*);
 Eterm erts_get_node_and_dist_references(struct process *);
@@ -204,7 +209,7 @@ erts_deref_dist_entry(DistEntry *dep)
 {
     ASSERT(dep);
     if (erts_refc_dectest(&dep->refc, 0) == 0)
-	erts_delete_dist_entry(dep);
+	erts_schedule_delete_dist_entry(dep);
 }
 
 ERTS_GLB_INLINE void
@@ -212,7 +217,7 @@ erts_deref_node_entry(ErlNode *np)
 {
     ASSERT(np);
     if (erts_refc_dectest(&np->refc, 0) == 0)
-	erts_delete_node(np);
+	erts_schedule_delete_node(np);
 }
 
 ERTS_GLB_INLINE void
@@ -253,5 +258,6 @@ erts_smp_de_links_unlock(DistEntry *dep)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs);
 
 #endif
-- 
cgit v1.2.3


From e6fce01ceb4e9ad27aade58f69857a5d9eb0d734 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 10 Jun 2015 20:13:33 +0200
Subject: Fix error checking for +zdntgc flag

---
 erts/emulator/beam/erl_init.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index a2d2a3befc..574a49dc7a 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2030,10 +2030,12 @@ erl_start(int argc, char **argv)
 		if (sys_strcmp(arg, "infinity") == 0)
 		    secs = ERTS_NODE_TAB_DELAY_GC_INFINITY;
 		else {
+		    char *endptr;
 		    errno = 0;
-		    secs = strtol(arg, NULL, 10);
-		    if (errno != 0 || secs < 0 || ERTS_NODE_TAB_DELAY_GC_MAX < secs) {
-			erts_fprintf(stderr, "Invalid delayed node table gc: %ld\n", secs);
+		    secs = strtol(arg, &endptr, 10);
+		    if (errno != 0 || *arg == '\0' || *endptr != '\0'
+			|| secs < 0 || ERTS_NODE_TAB_DELAY_GC_MAX < secs) {
+			erts_fprintf(stderr, "Invalid delayed node table gc: %s\n", arg);
 			erts_usage();
 		    }
 		}
-- 
cgit v1.2.3


From c09561dc51be736943a32862a41a3eaef2e41b83 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 3 Jun 2015 16:19:30 +0200
Subject: erts: Yield in maps:merge

---
 erts/emulator/beam/atom.names |   1 +
 erts/emulator/beam/erl_init.c |   1 +
 erts/emulator/beam/erl_map.c  | 255 ++++++++++++++++++++++++++++++------------
 erts/emulator/beam/global.h   |  13 ++-
 4 files changed, 196 insertions(+), 74 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 5ec1409adf..74b42c647e 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -350,6 +350,7 @@ atom message
 atom message_binary
 atom message_queue_len
 atom messages
+atom merge_trap
 atom meta
 atom meta_match_spec
 atom micro_seconds
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 988ff0e2b5..98ab9f598f 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -379,6 +379,7 @@ erl_init(int ncpu,
     erts_init_bif_re();
     erts_init_unicode(); /* after RE to get access to PCRE unicode */
     erts_init_external();
+    erts_init_map();
     erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2);
     erts_late_init_process();
 #if HAVE_ERTS_MSEG
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 5802ec76ba..c1ad5658d9 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -32,6 +32,7 @@
 #include "erl_process.h"
 #include "error.h"
 #include "bif.h"
+#include "erl_binary.h"
 
 #include "erl_map.h"
 
@@ -79,8 +80,12 @@ typedef struct {
 
 
 static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
-static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args);
+static BIF_RETTYPE map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
+struct HashmapMergeContext_;
+static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args,
+                                 struct HashmapMergeContext_*);
+static Export hashmap_merge_trap_export;
+static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -95,6 +100,15 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
 static int hxnodecmp(hxnode_t* a, hxnode_t* b);
 static int hxnodecmpkey(hxnode_t* a, hxnode_t* b);
 
+
+void erts_init_map(void) {
+    erts_init_trap_export(&hashmap_merge_trap_export,
+			  am_maps, am_merge_trap, 1,
+			  &maps_merge_trap_1);
+    return;
+}
+
+
 /* erlang:map_size/1
  * the corresponding instruction is implemented in:
  *     beam/erl_bif_guard.c
@@ -942,15 +956,15 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
 	    BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2));
 	} else if (is_hashmap(BIF_ARG_2)) {
 	    /* Will always become a tree */
-	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
+            return map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0);
 	}
 	BIF_P->fvalue = BIF_ARG_2;
     } else if (is_hashmap(BIF_ARG_1)) {
 	if (is_hashmap(BIF_ARG_2)) {
-	    BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
+	    return hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0, NULL);
 	} else if (is_flatmap(BIF_ARG_2)) {
 	    /* Will always become a tree */
-	    BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1));
+	    return map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1);
 	}
 	BIF_P->fvalue = BIF_ARG_2;
     } else {
@@ -1113,30 +1127,63 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
 
-    return hashmap_merge(p, res, tree, swap_args);
+    return hashmap_merge(p, res, tree, swap_args, NULL);
+}
+
+#define PSTACK_TYPE struct HashmapMergePStackType
+struct HashmapMergePStackType {
+    Eterm *srcA, *srcB;
+    Uint32 abm, bbm, rbm;        /* node bitmaps */
+    int keepA;
+    int ix;
+    Eterm array[16];
+};
+
+typedef struct HashmapMergeContext_ {
+    Uint size;  /* total key-value counter */
+    unsigned int lvl;
+    Eterm nodeA, nodeB;
+    Eterm res;
+    Eterm trap_bin;
+    ErtsPStack pstack;
+#ifdef DEBUG
+    Eterm dbg_map_A, dbg_map_B;
+#endif
+} HashmapMergeContext;
+
+static void hashmap_merge_ctx_destructor(Binary* ctx_bin)
+{
+    HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin);
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor);
+
+    PSTACK_DESTROY_SAVED(&ctx->pstack);
+}
+
+BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) {
+    Binary* ctx_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor);
+
+    return hashmap_merge(BIF_P, NIL, NIL, 0,
+                         (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin));
 }
 
 #define HALLOC_EXTRA 200
+#define MAP_MERGE_LOOP_FACTOR 8
 
-static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args) {
+static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B,
+                                 int swap_args, HashmapMergeContext* ctx) {
 #define PSTACK_TYPE struct HashmapMergePStackType
-    struct HashmapMergePStackType {
-	Eterm *srcA, *srcB;
-	Uint32 abm, bbm, rbm;        /* node bitmaps */
-	int keepA;
-	int ix;
-	Eterm array[16];
-    };
     PSTACK_DECLARE(s, 4);
-    struct HashmapMergePStackType* sp = PSTACK_PUSH(s);
-    Eterm *hp, *nhp;
-    Eterm hdrA, hdrB;
+    HashmapMergeContext local_ctx;
+    struct HashmapMergePStackType* sp;
     Uint32 ahx, bhx;
-    Uint size;  /* total key-value counter */
-    int keepA = swap_args;
-    unsigned int lvl = 0;
+    Eterm hdrA, hdrB;
+    Eterm *hp, *nhp;
+    Eterm trap_ret;
+    Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * MAP_MERGE_LOOP_FACTOR);
+    Sint reds =  initial_reds;
     DeclareTmpHeap(th,2,p);
-    Eterm res = THE_NON_VALUE;
     UseTmpHeap(2,p);
 
     /*
@@ -1144,52 +1191,72 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args)
      * and merge each pair of nodes.
      */
 
-    {
-	hashmap_head_t* a = (hashmap_head_t*) hashmap_val(nodeA);
-	hashmap_head_t* b = (hashmap_head_t*) hashmap_val(nodeB);
-	size = a->size + b->size;
+    PSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+
+    if (ctx == NULL) { /* first call */
+        hashmap_head_t* a = (hashmap_head_t*) hashmap_val(map_A);
+        hashmap_head_t* b = (hashmap_head_t*) hashmap_val(map_B);
+
+        sp = PSTACK_PUSH(s);
+        sp->keepA = swap_args;
+        local_ctx.size = a->size + b->size;
+        local_ctx.lvl = 0;
+        local_ctx.nodeA = map_A;
+        local_ctx.nodeB = map_B;
+        local_ctx.res = THE_NON_VALUE;
+    #ifdef DEBUG
+        local_ctx.dbg_map_A = map_A;
+        local_ctx.dbg_map_B = map_B;
+        local_ctx.trap_bin = THE_NON_VALUE;
+    #endif
+        ctx = &local_ctx;
+    }
+    else {
+        PSTACK_RESTORE(s, &ctx->pstack);
+        sp = PSTACK_TOP(s);
+        goto resume_from_trap;
     }
 
 recurse:
 
-    if (primary_tag(nodeA) == TAG_PRIMARY_BOXED &&
-	primary_tag(nodeB) == TAG_PRIMARY_LIST) {
+    if (primary_tag(ctx->nodeA) == TAG_PRIMARY_BOXED &&
+	primary_tag(ctx->nodeB) == TAG_PRIMARY_LIST) {
 	/* Avoid implementing this combination by switching places */
-	Eterm tmp = nodeA;
-	nodeA = nodeB;
-	nodeB = tmp;
-	keepA = !keepA;
+	Eterm tmp = ctx->nodeA;
+	ctx->nodeA = ctx->nodeB;
+	ctx->nodeB = tmp;
+	sp->keepA = !sp->keepA;
     }
 
-    switch (primary_tag(nodeA)) {
+    switch (primary_tag(ctx->nodeA)) {
     case TAG_PRIMARY_LIST: {
-	sp->srcA = list_val(nodeA);
-	switch (primary_tag(nodeB)) {
+	Eterm keyA = CAR(list_val(ctx->nodeA));
+	switch (primary_tag(ctx->nodeB)) {
 	case TAG_PRIMARY_LIST: { /* LEAF + LEAF */
-	    sp->srcB = list_val(nodeB);
+	    Eterm keyB = CAR(list_val(ctx->nodeB));
 
-	    if (EQ(CAR(sp->srcA), CAR(sp->srcB))) {
-		--size;
-		res = keepA ? nodeA : nodeB;
+	    if (EQ(keyA, keyB)) {
+		--ctx->size;
+		ctx->res = sp->keepA ? ctx->nodeA : ctx->nodeB;
 	    } else {
-		ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
-		bhx = hashmap_restore_hash(th, lvl, CAR(sp->srcB));
+		ahx = hashmap_restore_hash(th, ctx->lvl, keyA);
+		bhx = hashmap_restore_hash(th, ctx->lvl, keyB);
 		sp->abm = 1 << hashmap_index(ahx);
 		sp->bbm = 1 << hashmap_index(bhx);
 
-		sp->srcA = &nodeA;
-		sp->srcB = &nodeB;
+		sp->srcA = &ctx->nodeA;
+		sp->srcB = &ctx->nodeB;
 	    }
 	    break;
 	}
 	case TAG_PRIMARY_BOXED: { /* LEAF + NODE */
-	    sp->srcB = boxed_val(nodeB);
+	    sp->srcB = boxed_val(ctx->nodeB);
 	    ASSERT(is_header(*sp->srcB));
 	    hdrB = *sp->srcB++;
 
-	    ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA));
+	    ahx = hashmap_restore_hash(th, ctx->lvl, keyA);
 	    sp->abm = 1 << hashmap_index(ahx);
-	    sp->srcA = &nodeA;
+	    sp->srcA = &ctx->nodeA;
 	    switch(hdrB & _HEADER_MAP_SUBTAG_MASK) {
 	    case HAMT_SUBTAG_HEAD_ARRAY:
                 sp->srcB++;
@@ -1202,26 +1269,26 @@ recurse:
 		break;
 
 	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
+		erl_exit(ERTS_ABORT_EXIT, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
 		break;
 	    }
 	    break;
 	}
 	default:
-	    erl_exit(1, "bad primary tag %ld\r\n", nodeB);
+	    erl_exit(ERTS_ABORT_EXIT, "bad primary tag %ld\r\n", ctx->nodeB);
 	}
 	break;
     }
     case TAG_PRIMARY_BOXED: { /* NODE + NODE */
-	sp->srcA = boxed_val(nodeA);
+	sp->srcA = boxed_val(ctx->nodeA);
 	hdrA = *sp->srcA++;
 	ASSERT(is_header(hdrA));
 	switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
 	case HAMT_SUBTAG_HEAD_ARRAY: {
             sp->srcA++;
-	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
+	    ASSERT(primary_tag(ctx->nodeB) == TAG_PRIMARY_BOXED);
 	    sp->abm = 0xffff;
-	    sp->srcB = boxed_val(nodeB);
+	    sp->srcB = boxed_val(ctx->nodeB);
 	    hdrB = *sp->srcB++;
 	    ASSERT(is_header(hdrB));
 	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
@@ -1240,9 +1307,9 @@ recurse:
 	}
 	case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++;
 	case HAMT_SUBTAG_NODE_BITMAP: {
-	    ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED);
+	    ASSERT(primary_tag(ctx->nodeB) == TAG_PRIMARY_BOXED);
 	    sp->abm = MAP_HEADER_VAL(hdrA);
-	    sp->srcB = boxed_val(nodeB);
+	    sp->srcB = boxed_val(ctx->nodeB);
 	    hdrB = *sp->srcB++;
 	    ASSERT(is_header(hdrB));
 	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
@@ -1256,40 +1323,44 @@ recurse:
 		break;
 
 	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
+		erl_exit(ERTS_ABORT_EXIT, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
 	    }
 	    break;
 	}
 	default:
-	    erl_exit(1, "bad primary tag %ld\r\n", nodeA);
+            erl_exit(ERTS_ABORT_EXIT, "bad primary tag %ld\r\n", ctx->nodeA);
 	}
 	break;
     }
     default:
-	erl_exit(1, "bad primary tag %ld\r\n", nodeA);
+	erl_exit(ERTS_ABORT_EXIT, "bad primary tag %ld\r\n", ctx->nodeA);
     }
 
     for (;;) {
-	if (is_value(res)) { /* We have a complete (sub-)tree or leaf */
-	    if (lvl == 0)
+	if (is_value(ctx->res)) { /* We have a complete (sub-)tree or leaf */
+	    if (ctx->lvl == 0)
 		break;
 
 	    /* Pop from stack and continue build parent node */
-	    lvl--;
+	    ctx->lvl--;
 	    sp = PSTACK_POP(s);
-	    sp->array[sp->ix++] = res;
-	    res = THE_NON_VALUE;
+	    sp->array[sp->ix++] = ctx->res;
+	    ctx->res = THE_NON_VALUE;
 	    if (sp->rbm) {
 		sp->srcA++;
 		sp->srcB++;
-		keepA = sp->keepA;
 	    }
 	} else { /* Start build a node */
 	    sp->ix = 0;
 	    sp->rbm = sp->abm | sp->bbm;
-	    ASSERT(!(sp->rbm == 0 && lvl > 0));
+	    ASSERT(!(sp->rbm == 0 && ctx->lvl > 0));
 	}
 
+        if (--reds <= 0) {
+            goto trap;
+        }
+resume_from_trap:
+
 	while (sp->rbm) {
 	    Uint32 next = sp->rbm & (sp->rbm-1);
 	    Uint32 bit = sp->rbm ^ next;
@@ -1297,13 +1368,12 @@ recurse:
 	    if (sp->abm & bit) {
 		if (sp->bbm & bit) {
 		    /* Bit clash. Push and resolve by recursive merge */
-		    if (sp->rbm) {
-			sp->keepA = keepA;
-		    }
-		    nodeA = *sp->srcA;
-		    nodeB = *sp->srcB;
-		    lvl++;
+                    int keepA = sp->keepA;
+		    ctx->nodeA = *sp->srcA;
+		    ctx->nodeB= *sp->srcB;
+		    ctx->lvl++;
 		    sp = PSTACK_PUSH(s);
+                    sp->keepA = keepA;
 		    goto recurse;
 		} else {
 		    sp->array[sp->ix++] = *sp->srcA++;
@@ -1315,23 +1385,62 @@ recurse:
 	}
 
 	ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm));
-	if (lvl == 0) {
+	if (ctx->lvl == 0) {
 	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
 	    hp = nhp;
 	    *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
 		     : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm));
-	    *hp++ = size;
+	    *hp++ = ctx->size;
 	} else {
 	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
 	    hp = nhp;
 	    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm);
 	}
-	memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
-	res = make_boxed(nhp);
+	sys_memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
+	ctx->res = make_boxed(nhp);
+    }
+
+    /* Done */
+
+    if (ctx != &local_ctx) {
+        ASSERT(ctx->trap_bin != THE_NON_VALUE);
+        ASSERT(p->flags & F_DISABLE_GC);
+        erts_set_gc_state(p, 1);
+    }
+    else {
+        ASSERT(ctx->trap_bin == THE_NON_VALUE);
+        ASSERT(!(p->flags & F_DISABLE_GC));
     }
     PSTACK_DESTROY(s);
     UnUseTmpHeap(2,p);
-    return res;
+    BUMP_REDS(p, (initial_reds - reds) / MAP_MERGE_LOOP_FACTOR);
+    return ctx->res;
+
+trap:  /* Yield */
+
+    if (ctx == &local_ctx) {
+        Binary* ctx_b = erts_create_magic_binary(sizeof(HashmapMergeContext),
+                                                 hashmap_merge_ctx_destructor);
+        ctx = ERTS_MAGIC_BIN_DATA(ctx_b);
+        sys_memcpy(ctx, &local_ctx, sizeof(HashmapMergeContext));
+        hp = HAlloc(p, PROC_BIN_SIZE);
+        ASSERT(ctx->trap_bin == THE_NON_VALUE);
+        ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), ctx_b);
+
+        erts_set_gc_state(p, 0);
+    }
+    else {
+        ASSERT(ctx->trap_bin != THE_NON_VALUE);
+        ASSERT(p->flags & F_DISABLE_GC);
+    }
+
+    PSTACK_SAVE(s, &ctx->pstack);
+
+    BUMP_ALL_REDS(p);
+    ERTS_BIF_PREP_TRAP1(trap_ret, &hashmap_merge_trap_export,
+                        p, ctx->trap_bin);
+    UnUseTmpHeap(2,p);
+    return trap_ret;
 }
 
 static int hash_cmp(Uint32 ha, Uint32 hb)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 07806c823c..14d42599a1 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -817,7 +817,7 @@ do {\
 	UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\
 	(dst)->pstart = erts_alloc(s.alloc_type,\
 				   sizeof(PSTK_DEF_STACK(s)));\
-	memcpy((dst)->pstart, s.pstart, _pbytes);\
+	sys_memcpy((dst)->pstart, s.pstart, _pbytes);\
 	(dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\
 	(dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\
 	(dst)->alloc_type = s.alloc_type;\
@@ -838,6 +838,14 @@ do {						        \
     ASSERT(s.psp < s.pend);			        \
 } while (0)
 
+#define PSTACK_DESTROY_SAVED(pstack)\
+do {\
+    if ((pstack)->pstart) {\
+	erts_free((pstack)->alloc_type, (pstack)->pstart);\
+	(pstack)->pstart = NULL;\
+    }\
+} while(0)
+
 
 /* binary.c */
 
@@ -1126,6 +1134,9 @@ Sint erts_binary_set_loop_limit(Sint limit);
 /* external.c */
 void erts_init_external(void);
 
+/* erl_map.c */
+void erts_init_map(void);
+
 /* erl_unicode.c */
 void erts_init_unicode(void);
 Sint erts_unicode_set_loop_limit(Sint limit);
-- 
cgit v1.2.3


From f7ef9fb1679fcd46c48ec5f8a968f7e053b3d4ed Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 8 Jun 2015 15:50:46 +0200
Subject: erts: Optimize maps:merge

to be better at reusing entire hashmap sub-trees.

Sub-tree reuse is detected in three cases:

1. The sub-tree top node does not exist at all in the other map.
   Already implemented before this commit.

2. The exact same sub-tree exist in both maps.
   Must calculate nr of keys in tree to get total size right.

3. We detect that a sub-tree only contains stuff from one of the maps.

There is still one case we don't detect. If A and B leafs have equal
keys we could also compare the values. If values are equal, further
node reuse could propagate up toward the root (by 'mix'==0).
The downside would be potentially expensive value comparisons.
---
 erts/emulator/beam/erl_map.c | 300 ++++++++++++++++++++++---------------------
 1 file changed, 154 insertions(+), 146 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index c1ad5658d9..3e78731d20 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -86,6 +86,7 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_
                                  struct HashmapMergeContext_*);
 static Export hashmap_merge_trap_export;
 static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1);
+static Uint hashmap_subtree_size(Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -1132,18 +1133,18 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
 
 #define PSTACK_TYPE struct HashmapMergePStackType
 struct HashmapMergePStackType {
+    Eterm nodeA, nodeB;
     Eterm *srcA, *srcB;
-    Uint32 abm, bbm, rbm;        /* node bitmaps */
-    int keepA;
+    Uint32 abm, bbm, rbm; /* node bitmaps */
+    int mix;       /* &1: there are unique A stuff in node
+                    * &2: there are unique B stuff in node */
     int ix;
-    Eterm array[16];
+    Eterm array[16];   /* temp node construction area */
 };
 
 typedef struct HashmapMergeContext_ {
     Uint size;  /* total key-value counter */
     unsigned int lvl;
-    Eterm nodeA, nodeB;
-    Eterm res;
     Eterm trap_bin;
     ErtsPStack pstack;
 #ifdef DEBUG
@@ -1177,7 +1178,8 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B,
     PSTACK_DECLARE(s, 4);
     HashmapMergeContext local_ctx;
     struct HashmapMergePStackType* sp;
-    Uint32 ahx, bhx;
+    Uint32 hx;
+    Eterm res = THE_NON_VALUE;
     Eterm hdrA, hdrB;
     Eterm *hp, *nhp;
     Eterm trap_ret;
@@ -1198,12 +1200,11 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B,
         hashmap_head_t* b = (hashmap_head_t*) hashmap_val(map_B);
 
         sp = PSTACK_PUSH(s);
-        sp->keepA = swap_args;
+        sp->srcA = swap_args ? &map_B : &map_A;
+        sp->srcB = swap_args ? &map_A : &map_B;
+        sp->mix = 0;
         local_ctx.size = a->size + b->size;
         local_ctx.lvl = 0;
-        local_ctx.nodeA = map_A;
-        local_ctx.nodeB = map_B;
-        local_ctx.res = THE_NON_VALUE;
     #ifdef DEBUG
         local_ctx.dbg_map_A = map_A;
         local_ctx.dbg_map_B = map_B;
@@ -1219,133 +1220,97 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B,
 
 recurse:
 
-    if (primary_tag(ctx->nodeA) == TAG_PRIMARY_BOXED &&
-	primary_tag(ctx->nodeB) == TAG_PRIMARY_LIST) {
-	/* Avoid implementing this combination by switching places */
-	Eterm tmp = ctx->nodeA;
-	ctx->nodeA = ctx->nodeB;
-	ctx->nodeB = tmp;
-	sp->keepA = !sp->keepA;
-    }
-
-    switch (primary_tag(ctx->nodeA)) {
-    case TAG_PRIMARY_LIST: {
-	Eterm keyA = CAR(list_val(ctx->nodeA));
-	switch (primary_tag(ctx->nodeB)) {
-	case TAG_PRIMARY_LIST: { /* LEAF + LEAF */
-	    Eterm keyB = CAR(list_val(ctx->nodeB));
-
-	    if (EQ(keyA, keyB)) {
-		--ctx->size;
-		ctx->res = sp->keepA ? ctx->nodeA : ctx->nodeB;
-	    } else {
-		ahx = hashmap_restore_hash(th, ctx->lvl, keyA);
-		bhx = hashmap_restore_hash(th, ctx->lvl, keyB);
-		sp->abm = 1 << hashmap_index(ahx);
-		sp->bbm = 1 << hashmap_index(bhx);
+    sp->nodeA = *sp->srcA;
+    sp->nodeB = *sp->srcB;
 
-		sp->srcA = &ctx->nodeA;
-		sp->srcB = &ctx->nodeB;
-	    }
-	    break;
-	}
-	case TAG_PRIMARY_BOXED: { /* LEAF + NODE */
-	    sp->srcB = boxed_val(ctx->nodeB);
-	    ASSERT(is_header(*sp->srcB));
-	    hdrB = *sp->srcB++;
-
-	    ahx = hashmap_restore_hash(th, ctx->lvl, keyA);
-	    sp->abm = 1 << hashmap_index(ahx);
-	    sp->srcA = &ctx->nodeA;
-	    switch(hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-                sp->srcB++;
-		sp->bbm = 0xffff;
-		break;
+    if (sp->nodeA == sp->nodeB) {
+        res = sp->nodeA;
+        ctx->size -= is_list(sp->nodeB) ? 1 : hashmap_subtree_size(sp->nodeB);
+    }
+    else {
+        if (is_list(sp->nodeA)) { /* A is LEAF */
+            Eterm keyA = CAR(list_val(sp->nodeA));
+
+            if (is_list(sp->nodeB)) { /* LEAF + LEAF */
+                Eterm keyB = CAR(list_val(sp->nodeB));
+
+                if (EQ(keyA, keyB)) {
+                    --ctx->size;
+                    res = sp->nodeB;
+                    sp->mix = 2;   /* We assume values differ.
+                                      + Don't spend time comparing big values.
+                                      - Might waste some heap space for internal
+                                        nodes that could otherwise be reused. */
+                    goto merge_nodes;
+                }
+            }
+            hx = hashmap_restore_hash(th, ctx->lvl, keyA);
+            sp->abm = 1 << hashmap_index(hx);
+            /* keep srcA pointing at the leaf */
+        }
+        else { /* A is NODE */
+            sp->srcA = boxed_val(sp->nodeA);
+            hdrA = *sp->srcA++;
+            ASSERT(is_header(hdrA));
+            switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_HEAD_ARRAY: {
+                sp->srcA++;
+                sp->abm = 0xffff;
+                break;
+            }
+            case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++;
+            case HAMT_SUBTAG_NODE_BITMAP: {
+                sp->abm = MAP_HEADER_VAL(hdrA);
+                break;
+            }
+            default:
+                erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA);
+            }
+        }
 
-	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sp->bbm = MAP_HEADER_VAL(hdrB);
-		break;
+        if (is_list(sp->nodeB)) { /* B is LEAF */
+            Eterm keyB = CAR(list_val(sp->nodeB));
 
-	    default:
-		erl_exit(ERTS_ABORT_EXIT, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
-		break;
-	    }
-	    break;
-	}
-	default:
-	    erl_exit(ERTS_ABORT_EXIT, "bad primary tag %ld\r\n", ctx->nodeB);
-	}
-	break;
-    }
-    case TAG_PRIMARY_BOXED: { /* NODE + NODE */
-	sp->srcA = boxed_val(ctx->nodeA);
-	hdrA = *sp->srcA++;
-	ASSERT(is_header(hdrA));
-	switch (hdrA & _HEADER_MAP_SUBTAG_MASK) {
-	case HAMT_SUBTAG_HEAD_ARRAY: {
-            sp->srcA++;
-	    ASSERT(primary_tag(ctx->nodeB) == TAG_PRIMARY_BOXED);
-	    sp->abm = 0xffff;
-	    sp->srcB = boxed_val(ctx->nodeB);
-	    hdrB = *sp->srcB++;
-	    ASSERT(is_header(hdrB));
-	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-                sp->srcB++;
-		sp->bbm = 0xffff;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sp->bbm = MAP_HEADER_VAL(hdrB);
-		break;
-	    default:
-		erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
-	    }
-	    break;
-	}
-	case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++;
-	case HAMT_SUBTAG_NODE_BITMAP: {
-	    ASSERT(primary_tag(ctx->nodeB) == TAG_PRIMARY_BOXED);
-	    sp->abm = MAP_HEADER_VAL(hdrA);
-	    sp->srcB = boxed_val(ctx->nodeB);
-	    hdrB = *sp->srcB++;
-	    ASSERT(is_header(hdrB));
-	    switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
+            hx = hashmap_restore_hash(th, ctx->lvl, keyB);
+            sp->bbm = 1 << hashmap_index(hx);
+            /* keep srcB pointing at the leaf */
+        }
+        else { /* B is NODE */
+            sp->srcB = boxed_val(sp->nodeB);
+            hdrB = *sp->srcB++;
+            ASSERT(is_header(hdrB));
+            switch (hdrB & _HEADER_MAP_SUBTAG_MASK) {
+            case HAMT_SUBTAG_HEAD_ARRAY: {
                 sp->srcB++;
-		sp->bbm = 0xffff;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sp->bbm = MAP_HEADER_VAL(hdrB);
-		break;
-
-	    default:
-		erl_exit(ERTS_ABORT_EXIT, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK);
-	    }
-	    break;
-	}
-	default:
-            erl_exit(ERTS_ABORT_EXIT, "bad primary tag %ld\r\n", ctx->nodeA);
-	}
-	break;
-    }
-    default:
-	erl_exit(ERTS_ABORT_EXIT, "bad primary tag %ld\r\n", ctx->nodeA);
+                sp->bbm = 0xffff;
+                break;
+            }
+            case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++;
+            case HAMT_SUBTAG_NODE_BITMAP: {
+                sp->bbm = MAP_HEADER_VAL(hdrB);
+                break;
+            }
+            default:
+                erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB);
+            }
+        }
     }
 
+merge_nodes:
+
     for (;;) {
-	if (is_value(ctx->res)) { /* We have a complete (sub-)tree or leaf */
+	if (is_value(res)) { /* We have a complete (sub-)tree or leaf */
+            int child_mix;
 	    if (ctx->lvl == 0)
 		break;
 
 	    /* Pop from stack and continue build parent node */
 	    ctx->lvl--;
+            child_mix = sp->mix;
 	    sp = PSTACK_POP(s);
-	    sp->array[sp->ix++] = ctx->res;
-	    ctx->res = THE_NON_VALUE;
+	    sp->array[sp->ix++] = res;
+            sp->mix |= child_mix;
+	    res = THE_NON_VALUE;
 	    if (sp->rbm) {
 		sp->srcA++;
 		sp->srcB++;
@@ -1368,40 +1333,69 @@ resume_from_trap:
 	    if (sp->abm & bit) {
 		if (sp->bbm & bit) {
 		    /* Bit clash. Push and resolve by recursive merge */
-                    int keepA = sp->keepA;
-		    ctx->nodeA = *sp->srcA;
-		    ctx->nodeB= *sp->srcB;
+		    Eterm* srcA = sp->srcA;
+		    Eterm* srcB = sp->srcB;
 		    ctx->lvl++;
 		    sp = PSTACK_PUSH(s);
-                    sp->keepA = keepA;
+                    sp->srcA = srcA;
+                    sp->srcB = srcB;
+                    sp->mix = 0;
 		    goto recurse;
 		} else {
 		    sp->array[sp->ix++] = *sp->srcA++;
+                    sp->mix |= 1;
 		}
 	    } else {
 		ASSERT(sp->bbm & bit);
 		sp->array[sp->ix++] = *sp->srcB++;
+                sp->mix |=  2;
 	    }
 	}
 
-	ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm));
-	if (ctx->lvl == 0) {
-	    nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
-	    hp = nhp;
-	    *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
-		     : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm));
-	    *hp++ = ctx->size;
-	} else {
-	    nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
-	    hp = nhp;
-	    *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm);
-	}
-	sys_memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
-	ctx->res = make_boxed(nhp);
+        switch (sp->mix) {
+        case 0: /* Nodes A and B contain the *EXACT* same sub-trees
+                   => fall through and reuse nodeA */
+
+        case 1: /* Only unique A stuff => reuse nodeA */
+            res = sp->nodeA;
+            break;
+
+        case 2: /* Only unique B stuff => reuse nodeB */
+            res = sp->nodeB;
+            break;
+
+        case 3: /* We have a mix => must build new node */
+            ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm));
+            if (ctx->lvl == 0) {
+                nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
+                hp = nhp;
+                *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY
+                         : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm));
+                *hp++ = ctx->size;
+            } else {
+                nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA);
+                hp = nhp;
+                *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm);
+            }
+            sys_memcpy(hp, sp->array, sp->ix * sizeof(Eterm));
+            res = make_boxed(nhp);
+            break;
+        default:
+            erl_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix);
+        }
     }
 
     /* Done */
 
+#ifdef DEBUG
+    {
+        Eterm *head = hashmap_val(res);
+        Uint size = head[1];
+        Uint real_size = hashmap_subtree_size(res);
+        ASSERT(size == real_size);
+    }
+#endif
+
     if (ctx != &local_ctx) {
         ASSERT(ctx->trap_bin != THE_NON_VALUE);
         ASSERT(p->flags & F_DISABLE_GC);
@@ -1414,7 +1408,7 @@ resume_from_trap:
     PSTACK_DESTROY(s);
     UnUseTmpHeap(2,p);
     BUMP_REDS(p, (initial_reds - reds) / MAP_MERGE_LOOP_FACTOR);
-    return ctx->res;
+    return res;
 
 trap:  /* Yield */
 
@@ -1443,6 +1437,19 @@ trap:  /* Yield */
     return trap_ret;
 }
 
+static Uint hashmap_subtree_size(Eterm node) {
+    DECLARE_WSTACK(stack);
+    Uint size = 0;
+
+    hashmap_iterator_init(&stack, node, 0);
+    while (hashmap_iterator_next(&stack)) {
+        size++;
+    }
+    DESTROY_WSTACK(stack);
+    return size;
+}
+
+
 static int hash_cmp(Uint32 ha, Uint32 hb)
 {
     int i;
@@ -1865,10 +1872,11 @@ void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) {
 	sz = 16;
 	break;
     case HAMT_SUBTAG_HEAD_BITMAP:
-	sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+    case HAMT_SUBTAG_NODE_BITMAP:
+        sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
 	break;
     default:
-	erl_exit(1, "bad header");
+	erl_exit(ERTS_ABORT_EXIT, "bad header");
     }
 
     WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE,  /* end marker */
@@ -1905,7 +1913,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
 		ASSERT(sz < 17);
 		break;
 	    default:
-		erl_exit(1, "bad header");
+		erl_exit(ERTS_ABORT_EXIT, "bad header");
 	    }
 
 	    idx++;
-- 
cgit v1.2.3


From 02d0ce034598297565f2b35ecc3d1af121787f33 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 2 Jun 2015 19:23:49 +0200
Subject: erts: Remove hashmap probabilistic heap overestimation

by adding a dynamic heap factory.

"binary_to_term" is now a hybrid solution with both
a call to decoded_size() to calculate needed heap space
AND possible dynamic allocation of more heap space
if needed for big maps.

The heap size returned from decoded_size() is guaranteed
to be sufficient for all term heap data except for hashmap
nodes. All hashmap nodes are created at the end of dec_term()
by invoking the heap factory interface that may allocate more
heap space on process heap or in fragments.

With this commit it is no longer guaranteed that a message
is confined to only one heap fragment.
---
 erts/emulator/beam/beam_emu.c         |   3 +-
 erts/emulator/beam/beam_load.c        | 184 +++++++++-----------
 erts/emulator/beam/big.h              |   1 +
 erts/emulator/beam/copy.c             |  13 +-
 erts/emulator/beam/dist.c             |   4 +-
 erts/emulator/beam/erl_bif_info.c     |  58 ++++---
 erts/emulator/beam/erl_db_hash.c      |   1 -
 erts/emulator/beam/erl_db_util.c      |  25 ++-
 erts/emulator/beam/erl_gc.c           |  93 +++++-----
 erts/emulator/beam/erl_map.c          |  27 +--
 erts/emulator/beam/erl_map.h          |  14 +-
 erts/emulator/beam/erl_message.c      | 307 +++++++++++++++++++++++++++++-----
 erts/emulator/beam/erl_message.h      |  85 ++++++----
 erts/emulator/beam/erl_process_lock.c |   2 +
 erts/emulator/beam/external.c         | 291 +++++++++++++++-----------------
 erts/emulator/beam/external.h         |   9 +-
 erts/emulator/beam/io.c               | 247 ++++++++++++++-------------
 17 files changed, 807 insertions(+), 557 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a21622f424..2d4bf4240c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6653,8 +6653,9 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
 
 	p->htop = mhp;
 
-        factory.p = p;
+        erts_factory_proc_init(&factory, p);
         res = erts_hashmap_from_array(&factory, thp, n/2, 0);
+        erts_factory_close(&factory);
         if (p->mbuf) {
             Uint live = Arg(2);
             reg[live] = res;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0d40201934..8a8ad0d7a3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -205,10 +205,7 @@ typedef struct {
 
 typedef struct {
     Eterm term;			/* The tagged term (in the heap). */
-    Uint heap_size;		/* (Exact) size on the heap. */
-    SWord offset;		/* Offset from temporary location to final. */
-    ErlOffHeap off_heap;	/* Start of linked list of ProcBins. */
-    Eterm* heap;		/* Heap for term. */
+    ErlHeapFragment* heap_frags;
 } Literal;
 
 /*
@@ -477,6 +474,8 @@ typedef struct LoaderState {
 
 
 static void free_loader_state(Binary* magic);
+static ErlHeapFragment* new_literal_fragment(Uint size);
+static void free_literal_fragment(ErlHeapFragment*);
 static void loader_state_dtor(Binary* magic);
 static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 			     Eterm group_leader, Eterm module,
@@ -883,6 +882,28 @@ free_loader_state(Binary* magic)
     }
 }
 
+static ErlHeapFragment* new_literal_fragment(Uint size)
+{
+    ErlHeapFragment* bp;
+    bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE,
+					    ERTS_HEAP_FRAG_SIZE(size));
+    ERTS_INIT_HEAP_FRAG(bp, size);
+    return bp;
+}
+
+static void free_literal_fragment(ErlHeapFragment* bp)
+{
+    ASSERT(bp != NULL);
+    do {
+	ErlHeapFragment* next_bp = bp->next;
+
+	erts_cleanup_offheap(&bp->off_heap);
+	ERTS_HEAP_FREE(ERTS_ALC_T_PREPARED_CODE, (void *) bp,
+		       ERTS_HEAP_FRAG_SIZE(bp->size));
+	bp = next_bp;
+    }while (bp != NULL);
+}
+
 /*
  * This destructor function can safely be called multiple times.
  */
@@ -922,10 +943,9 @@ loader_state_dtor(Binary* magic)
     if (stp->literals != 0) {
 	int i;
 	for (i = 0; i < stp->num_literals; i++) {
-	    if (stp->literals[i].heap != 0) {
-		erts_free(ERTS_ALC_T_PREPARED_CODE,
-			  (void *) stp->literals[i].heap);
-		stp->literals[i].heap = 0;
+	    if (stp->literals[i].heap_frags != 0) {
+                free_literal_fragment(stp->literals[i].heap_frags);
+		stp->literals[i].heap_frags = 0;
 	    }
 	}
 	erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals);
@@ -1450,6 +1470,7 @@ read_lambda_table(LoaderState* stp)
     return 0;
 }
 
+
 static int
 read_literal_table(LoaderState* stp)
 {
@@ -1471,7 +1492,7 @@ read_literal_table(LoaderState* stp)
     stp->allocated_literals = stp->num_literals;
 
     for (i = 0; i < stp->num_literals; i++) {
-	stp->literals[i].heap = 0;
+	stp->literals[i].heap_frags = 0;
     }
 
     for (i = 0; i < stp->num_literals; i++) {
@@ -1479,28 +1500,38 @@ read_literal_table(LoaderState* stp)
 	Sint heap_size;
 	byte* p;
 	Eterm val;
-	Eterm* hp;
+        ErtsHeapFactory factory;
 
 	GetInt(stp, 4, sz);	/* Size of external term format. */
 	GetString(stp, p, sz);
 	if ((heap_size = erts_decode_ext_size(p, sz)) < 0) {
 	    LoadError1(stp, "literal %d: bad external format", i);
 	}
-	hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
-						heap_size*sizeof(Eterm));
-	stp->literals[i].off_heap.first = 0;
-	stp->literals[i].off_heap.overhead = 0;
-	val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p);
-	stp->literals[i].heap_size = hp - stp->literals[i].heap;
-	if (stp->literals[i].heap_size > heap_size) {
-	    erl_exit(1, "overrun by %d word(s) for literal heap, term %d",
-		     stp->literals[i].heap_size - heap_size, i);
-	}
-	if (is_non_value(val)) {
-	    LoadError1(stp, "literal %d: bad external format", i);
-	}
-	stp->literals[i].term = val;
-	stp->total_literal_size += stp->literals[i].heap_size;
+
+        if (heap_size > 0) {
+            erts_factory_message_init(&factory, NULL, NULL,
+                                      new_literal_fragment(heap_size));
+	    factory.alloc_type = ERTS_ALC_T_PREPARED_CODE;
+            val = erts_decode_ext(&factory, &p);
+
+            if (is_non_value(val)) {
+                LoadError1(stp, "literal %d: bad external format", i);
+            }
+            erts_factory_close(&factory);
+            stp->literals[i].heap_frags = factory.heap_frags;
+            stp->total_literal_size += erts_used_frag_sz(factory.heap_frags);
+        }
+        else {
+            erts_factory_dummy_init(&factory);
+            val = erts_decode_ext(&factory, &p);
+            if (is_non_value(val)) {
+                LoadError1(stp, "literal %d: bad external format", i);
+            }
+            ASSERT(is_immed(val));
+            stp->literals[i].heap_frags = NULL;
+        }
+        stp->literals[i].term = val;
+
     }
     erts_free(ERTS_ALC_T_TMP, uncompressed);
     return 1;
@@ -4370,8 +4401,9 @@ freeze_code(LoaderState* stp)
 	Uint* low;
 	Uint* high;
 	LiteralPatch* lp;
-	struct erl_off_heap_header* off_heap = 0;
-	struct erl_off_heap_header** off_heap_last = &off_heap;
+        ErlOffHeap code_off_heap;
+
+        ERTS_INIT_OFF_HEAP(&code_off_heap);
 
 	low = (Uint *) (code+stp->ci);
 	high = low + stp->total_literal_size;
@@ -4379,73 +4411,21 @@ freeze_code(LoaderState* stp)
 	code[MI_LITERALS_END] = (BeamInstr) high;
 	ptr = low;
 	for (i = 0; i < stp->num_literals; i++) {
-	    SWord offset;
-	    struct erl_off_heap_header* t_off_heap;
-
-	    sys_memcpy(ptr, stp->literals[i].heap,
-		       stp->literals[i].heap_size*sizeof(Eterm));
-	    offset = ptr - stp->literals[i].heap;
-	    stp->literals[i].offset = offset;
-	    high = ptr + stp->literals[i].heap_size;
-	    while (ptr < high) {
-		Eterm val = *ptr;
-		switch (primary_tag(val)) {
-		case TAG_PRIMARY_LIST:
-		case TAG_PRIMARY_BOXED:
-		    *ptr++ = offset_ptr(val, offset);
-		    break;
-		case TAG_PRIMARY_HEADER:
-		    if (header_is_transparent(val)) {
-			ptr++;
-		    } else {
-			if (thing_subtag(val) == REFC_BINARY_SUBTAG) {
-			    struct erl_off_heap_header* oh;
-
-			    oh = (struct erl_off_heap_header*) ptr;
-			    if (oh->next) {
-				Eterm** uptr = (Eterm **) (void *) &oh->next;
-				*uptr += offset;
-			    }
-			}
-			ptr += 1 + thing_arityval(val);
-		    }
-		    break;
-		default:
-		    ptr++;
-		    break;
-		}
-	    }
-	    ASSERT(ptr == high);
-
-	    /*
-	     * Re-link the off_heap list for this term onto the
-	     * off_heap list for the entire module.
-	     */
-	    t_off_heap = stp->literals[i].off_heap.first;
-	    if (t_off_heap) {
-		t_off_heap = (struct erl_off_heap_header *)
-		    offset_ptr((UWord) t_off_heap, offset);
-		while (t_off_heap) {
-		    *off_heap_last = t_off_heap;
-		    off_heap_last = &t_off_heap->next;
-		    t_off_heap = t_off_heap->next;
-		}
-	    }
+            if (stp->literals[i].heap_frags) {
+                move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags,
+                                 &stp->literals[i].term, 1);
+            }
+            else ASSERT(is_immed(stp->literals[i].term));
 	}
-	code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap;
+        code[MI_LITERALS_OFF_HEAP] = (BeamInstr) code_off_heap.first;
 	lp = stp->literal_patches;
 	while (lp != 0) {
 	    BeamInstr* op_ptr;
-	    Uint literal;
 	    Literal* lit;
 
 	    op_ptr = code + lp->pos;
 	    lit = &stp->literals[op_ptr[0]];
-	    literal = lit->term;
-	    if (is_boxed(literal) || is_list(literal)) {
-		literal = offset_ptr(literal, lit->offset);
-	    }
-	    op_ptr[0] = literal;
+	    op_ptr[0] = lit->term;
 	    lp = lp->next;
 	}
 	literal_end += stp->total_literal_size;
@@ -5376,13 +5356,9 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
 
     stp->total_literal_size += heap_size;
     lit = stp->literals + stp->num_literals;
-    lit->offset = 0;
-    lit->heap_size = heap_size;
-    lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm));
-    lit->term = make_boxed(lit->heap);
-    lit->off_heap.first = 0;
-    lit->off_heap.overhead = 0;
-    *hpp = lit->heap;
+    lit->heap_frags = new_literal_fragment(heap_size);
+    lit->term = make_boxed(lit->heap_frags->mem);
+    *hpp = lit->heap_frags->mem;
     return stp->num_literals++;
 }
 
@@ -5659,10 +5635,8 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
 {
     Module* modp;
     BeamInstr* code;
-    Eterm* hp;
     byte* ext;
     Eterm result = NIL;
-    Eterm* end;
 
     if (is_not_atom(mod)) {
 	return THE_NON_VALUE;
@@ -5675,13 +5649,12 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
     code = modp->curr.code;
     ext = (byte *) code[MI_ATTR_PTR];
     if (ext != NULL) {
-	hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]);
-	end = hp + code[MI_ATTR_SIZE_ON_HEAP];
-	result = erts_decode_ext(&hp, &MSO(p), &ext);
+	ErtsHeapFactory factory;
+	erts_factory_proc_prealloc_init(&factory, p, code[MI_ATTR_SIZE_ON_HEAP]);
+	result = erts_decode_ext(&factory, &ext);
 	if (is_value(result)) {
-	    ASSERT(hp <= end);
+	    erts_factory_close(&factory);
 	}
-        HRelease(p,end,hp);
     }
     return result;
 }
@@ -5698,10 +5671,8 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 {
     Module* modp;
     BeamInstr* code;
-    Eterm* hp;
     byte* ext;
     Eterm result = NIL;
-    Eterm* end;
 
     if (is_not_atom(mod)) {
 	return THE_NON_VALUE;
@@ -5714,13 +5685,12 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
     code = modp->curr.code;
     ext = (byte *) code[MI_COMPILE_PTR];
     if (ext != NULL) {
-	hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]);
-	end = hp + code[MI_COMPILE_SIZE_ON_HEAP];
-	result = erts_decode_ext(&hp, &MSO(p), &ext);
+	ErtsHeapFactory factory;
+	erts_factory_proc_prealloc_init(&factory, p, code[MI_COMPILE_SIZE_ON_HEAP]);
+	result = erts_decode_ext(&factory, &ext);
 	if (is_value(result)) {
-	    ASSERT(hp <= end);
+	    erts_factory_close(&factory);
 	}
-        HRelease(p,end,hp);
     }
     return result;
 }
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 4e4611de16..5b5550da43 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -85,6 +85,7 @@ typedef Uint  dsize_t;	 /* Vector size type */
 
 /* The heap size needed for a bignum */
 #define BIG_NEED_SIZE(x)  ((x) + 1)
+#define BIG_NEED_FOR_BITS(bits) BIG_NEED_SIZE(((bits)-1)/D_EXP + 1)
 
 #define BIG_UINT_HEAP_SIZE (1 + 1)	/* always, since sizeof(Uint) <= sizeof(Eterm) */
 
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 850606dd86..cddadb346c 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -34,7 +34,7 @@
 #include "erl_bits.h"
 #include "dtrace-wrapper.h"
 
-static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*);
+static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*);
 
 /*
  *  Copy object "obj" to process p.
@@ -661,8 +661,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
     unsigned i;
 
     for (bp=first; bp!=NULL; bp=bp->next) {
-	move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
-	OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+	move_one_frag(hpp, bp, off_heap);
     }
     hp_end = *hpp;
     for (hp=hp_start; hp<hp_end; ++hp) {
@@ -698,10 +697,10 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 }
 
 static void
-move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
+move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap)
 {
-    Eterm* ptr = src;
-    Eterm* end = ptr + src_sz;
+    Eterm* ptr = frag->mem;
+    Eterm* end = ptr + frag->used_size;
     Eterm dummy_ref;
     Eterm* hp = *hpp;
 
@@ -732,5 +731,7 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
 	}
     }
     *hpp = hp;
+    OH_OVERHEAD(off_heap, frag->off_heap.overhead);
+    frag->off_heap.first = NULL;
 }
 
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 142fcb3c00..ae85659522 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1149,6 +1149,7 @@ int erts_net_message(Port *prt,
     DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE);
     Eterm* ctl = ctl_default;
     ErlOffHeap off_heap;
+    ErtsHeapFactory factory;
     Eterm* hp;
     Sint type;
     Eterm token;
@@ -1225,7 +1226,8 @@ int erts_net_message(Port *prt,
     }
     hp = ctl;
 
-    arg = erts_decode_dist_ext(&hp, &off_heap, &ede);
+    erts_factory_static_init(&factory, ctl, ctl_len, &off_heap);
+    arg = erts_decode_dist_ext(&factory, &ede);
     if (is_non_value(arg)) {
 #ifdef ERTS_DIST_MSG_DBG
 	erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n");
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f74aea80a7..15173bd05e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1106,19 +1106,18 @@ process_info_aux(Process *BIF_P,
 		    heap_need += mq[i].copy_struct_size;
 		}
 		else {
-		    mq[i].copy_struct_size = 0;
-		    if (mp->data.attached)
-			heap_need += erts_msg_attached_data_size(mp);
+		    mq[i].copy_struct_size = mp->data.attached ?
+                        erts_msg_attached_data_size(mp) : 0;
 		}
 		i++;
 	    }
 
-	    hp = HAlloc(BIF_P, heap_need);
-	    hp_end = hp + heap_need;
-	    ASSERT(i == n);
-	    for (i--; i >= 0; i--) {
-		Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
-		if (rp != BIF_P) {
+            if (rp != BIF_P) {
+                hp = HAlloc(BIF_P, heap_need);
+                hp_end = hp + heap_need;
+                ASSERT(i == n);
+                for (i--; i >= 0; i--) {
+                    Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
 		    if (is_value(msg)) {
 			if (mq[i].copy_struct_size)
 			    msg = copy_struct(msg,
@@ -1152,9 +1151,9 @@ process_info_aux(Process *BIF_P,
 			}
 			else {
 			    /* Make our copy of the message */
-			    ASSERT(size_object(msg) == hfp->used_size);
+			    ASSERT(size_object(msg) == erts_used_frag_sz(hfp));
 			    msg = copy_struct(msg,
-					      hfp->used_size,
+					      erts_used_frag_sz(hfp),
 					      &hp,
 					      &MSO(BIF_P));
 			}
@@ -1164,27 +1163,38 @@ process_info_aux(Process *BIF_P,
 			remove_bad_messages = 1;
 			continue;
 		    }
+                    res = CONS(hp, msg, res);
+                    hp += 2;
 		}
-		else {
+                HRelease(BIF_P, hp_end, hp+3);
+            }
+	    else {
+                for (i--; i >= 0; i--) {
+                    ErtsHeapFactory factory;
+                    Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
+
+                    erts_factory_proc_prealloc_init(&factory, BIF_P,
+                                                    mq[i].copy_struct_size+2);
 		    if (mq[i].msgp->data.attached) {
 			/* Decode it on the heap */
-			erts_move_msg_attached_data_to_heap(&hp,
-							    &MSO(BIF_P),
+			erts_move_msg_attached_data_to_heap(&factory,
 							    mq[i].msgp);
 			msg = ERL_MESSAGE_TERM(mq[i].msgp);
 			ASSERT(!mq[i].msgp->data.attached);
-			if (is_non_value(msg)) {
-			    /* Bad distribution message; ignore */
-			    remove_bad_messages = 1;
-			    continue;
-			}
-		    }
+                    }
+                    if (is_value(msg)) {
+                        hp = erts_produce_heap(&factory, 2, 0);
+                        res = CONS(hp, msg, res);
+                    }
+                    else {
+                        /* Bad distribution message; ignore */
+                        remove_bad_messages = 1;
+                        continue;
+                    }
+                    erts_factory_close(&factory);
 		}
-		    
-		res = CONS(hp, msg, res);
-		hp += 2;
+                hp = HAlloc(BIF_P, 3);
 	    }
-	    HRelease(BIF_P, hp_end, hp+3);
 	    erts_free(ERTS_ALC_T_TMP, mq);
 	    if (remove_bad_messages) {
 		ErlMessage **mpp;
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 383ee7c430..6e50e9c5b4 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1049,7 +1049,6 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 			Eterm copy = db_copy_element_from_ets(&tb->common, p,
 							      &b->dbterm, ndex, &hp, 2);
 			elem_list = CONS(hp, copy, elem_list);
-			hp += 2;
 		    }
 		    b = b->next;
 		}
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index c6c3c55a7e..d47ff03a30 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2175,11 +2175,12 @@ restart:
             {
                 ErtsHeapFactory factory;
                 Uint ix;
-                factory.p = build_proc;
                 for (ix = 0; ix < 2*n; ix++){
                     ehp[ix] = esp[ix];
                 }
+                erts_factory_proc_init(&factory, build_proc);
                 t = erts_hashmap_from_array(&factory, ehp, n, 0);
+                erts_factory_close(&factory);
             }
             *esp++ = t;
             break;
@@ -3192,6 +3193,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
 {
     Eterm* hp = *hpp;
     int i, arity = arityval(bp->tpl[0]);
+    ErtsHeapFactory factory;
 
     hp[0] = bp->tpl[0];
     *hpp += arity + 1;
@@ -3199,17 +3201,23 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
     hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos],
 				     size_object_rel(bp->tpl[tb->keypos], bp->tpl),
 				     hpp, off_heap, bp->tpl, NULL);
+
+    erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap);
+
     for (i=arity; i>0; i--) {
 	if (i != tb->keypos) {
 	    if (is_immed(bp->tpl[i])) {
 		hp[i] = bp->tpl[i];
 	    }
 	    else {
-		hp[i] = erts_decode_ext_ets(hpp, off_heap,
+		hp[i] = erts_decode_ext_ets(&factory,
 					    elem2ext(bp->tpl, i));
 	    }
 	}
     }
+    *hpp = factory.hp;
+    erts_factory_close(&factory);
+
     ASSERT((*hpp - hp) <= bp->size);
 #ifdef DEBUG_CLONE
     ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone));
@@ -3228,12 +3236,13 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p,
     if (tb->compress && pos != tb->keypos) {
 	byte* ext = elem2ext(obj->tpl, pos);
 	Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra;
-	Eterm* hp = HAlloc(p, sz);
-	Eterm* endp = hp + sz;
-	Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext);
-	*hpp = hp;
-	hp += extra;
-	HRelease(p, endp, hp);
+	Eterm copy;
+        ErtsHeapFactory factory;
+
+        erts_factory_proc_prealloc_init(&factory, p, sz);
+        copy = erts_decode_ext_ets(&factory, ext);
+	*hpp = erts_produce_heap(&factory, extra, 0);
+        erts_factory_close(&factory);
 #ifdef DEBUG_CLONE
 	ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone));
 #endif
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 14e7dde778..a456689b5c 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -108,8 +108,7 @@ static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 			     char* src, Uint src_size);
 static Eterm* collect_heap_frags(Process* p, Eterm* heap,
 				 Eterm* htop, Eterm* objv, int nobj);
-static Uint adjust_after_fullsweep(Process *p, Uint size_before,
-				   int need, Eterm *objv, int nobj);
+static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
 static void sweep_off_heap(Process *p, int fullsweep);
@@ -868,29 +867,37 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	ErlMessage *msgp;
 	Uint size_after;
 	Uint need_after;
-	Uint stack_size = STACK_SZ_ON_HEAP(p);
-	Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
-	Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p));
-	Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0);
+	const Uint stack_size = STACK_SZ_ON_HEAP(p);
+	const Uint size_before = MBUF_SIZE(p) + (HEAP_TOP(p) - HEAP_START(p));
+	Uint new_sz = HEAP_SIZE(p) + MBUF_SIZE(p) + combined_message_size(p);
+        new_sz = next_heap_size(p, new_sz, 0);
 
         do_minor(p, new_sz, objv, nobj);
 
-	/*
+        size_after = HEAP_TOP(p) - HEAP_START(p);
+        *recl += (size_before - size_after);
+
+        /*
 	 * Copy newly received message onto the end of the new heap.
 	 */
-	ErtsGcQuickSanityCheck(p);
-	for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-	    if (msgp->data.attached) {
-		erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
-		ErtsGcQuickSanityCheck(p);
-	    }
-	}
+        ErtsGcQuickSanityCheck(p);
+        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+            if (msgp->data.attached) {
+                ErtsHeapFactory factory;
+                erts_factory_proc_prealloc_init(&factory, p,
+                                                erts_msg_attached_data_size(msgp));
+                erts_move_msg_attached_data_to_heap(&factory, msgp);
+                erts_factory_close(&factory);
+                ErtsGcQuickSanityCheck(p);
+            }
+        }
 	ErtsGcQuickSanityCheck(p);
 
         GEN_GCS(p)++;
-        size_after = HEAP_TOP(p) - HEAP_START(p);
-        need_after = size_after + need + stack_size;
-        *recl += (size_before - size_after);
+        need_after = ((HEAP_TOP(p) - HEAP_START(p))
+                      + erts_used_frag_sz(MBUF(p))
+                      + need
+                      + stack_size);
 	
         /*
          * Excessively large heaps should be shrunk, but
@@ -925,6 +932,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
             }
 
             ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
+            ASSERT(MBUF(p) == NULL);
 	    return 1;		/* We are done. */
         }
 
@@ -933,6 +941,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	     * The heap size turned out to be just right. We are done.
 	     */
             ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
+            ASSERT(MBUF(p) == NULL);
             return 1;
 	}
     }
@@ -1212,7 +1221,9 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 {
     Rootset rootset;
     Roots* roots;
-    Uint size_before;
+    const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p))
+                              + (OLD_HTOP(p) - OLD_HEAP(p))
+                              + MBUF_SIZE(p));
     Eterm* n_heap;
     Eterm* n_htop;
     char* src = (char *) HEAP_START(p);
@@ -1221,25 +1232,15 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
     Uint n;
     Uint new_sz;
-    Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
-
-    size_before = fragments + (HEAP_TOP(p) - HEAP_START(p))
-                            + (OLD_HTOP(p) - OLD_HEAP(p));
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
      * to receive all live data.
      */
 
-    new_sz = HEAP_SIZE(p) + fragments + (OLD_HTOP(p) - OLD_HEAP(p));
-    /*
-     * We used to do
-     *
-     * new_sz += STACK_SZ_ON_HEAP(p);
-     *
-     * here for no obvious reason. (The stack size is already counted once
-     * in HEAP_SIZE(p).)
-     */
+    new_sz = (HEAP_SIZE(p) + MBUF_SIZE(p)
+              + combined_message_size(p)
+              + (OLD_HTOP(p) - OLD_HEAP(p)));
     new_sz = next_heap_size(p, new_sz, 0);
 
     /*
@@ -1432,20 +1433,27 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 
     ErtsGcQuickSanityCheck(p);
 
+    *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+
     {
 	ErlMessage *msgp;
+
 	/*
 	 * Copy newly received message onto the end of the new heap.
 	 */
-	for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
 	    if (msgp->data.attached) {
-		erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp);
+                ErtsHeapFactory factory;
+                erts_factory_proc_prealloc_init(&factory, p,
+                                                erts_msg_attached_data_size(msgp));
+		erts_move_msg_attached_data_to_heap(&factory, msgp);
+                erts_factory_close(&factory);
 		ErtsGcQuickSanityCheck(p);
 	    }
 	}
     }
 
-    *recl += adjust_after_fullsweep(p, size_before, need, objv, nobj);
+    adjust_after_fullsweep(p, need, objv, nobj);
 
 #ifdef HARDDEBUG
     disallow_heap_frag_ref_in_heap(p);
@@ -1456,21 +1464,17 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     return 1;			/* We are done. */
 }
 
-static Uint
-adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj)
+static void
+adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
 {
-    Uint wanted, sz, size_after, need_after;
+    Uint wanted, sz, need_after;
     Uint stack_size = STACK_SZ_ON_HEAP(p);
-    Uint reclaimed_now;
-
-    size_after = (HEAP_TOP(p) - HEAP_START(p));
-    reclaimed_now = (size_before - size_after);
     
     /*
      * Resize the heap if needed.
      */
     
-    need_after = size_after + need + stack_size;
+    need_after = (HEAP_TOP(p) - HEAP_START(p)) + need + stack_size;
     if (HEAP_SIZE(p) < need_after) {
         /* Too small - grow to match requested need */
         sz = next_heap_size(p, need_after, 0);
@@ -1493,8 +1497,6 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int
             shrink_new_heap(p, sz, objv, nobj);
         }
     }
-
-    return reclaimed_now;
 }
 
 /*
@@ -1942,7 +1944,8 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
      * until next GC.  
      */ 
     qb = MBUF(p);
-    while (qb != NULL) {      
+    while (qb != NULL) {
+        ASSERT(!qb->off_heap.first);  /* process fragments use the MSO(p) list */
 	frag_size = qb->used_size * sizeof(Eterm);
 	if (frag_size != 0) {
 	    frag_begin = (char *) qb->mem;
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index a1bd39dbc8..09fc9a2ad6 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -410,8 +410,9 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) {
     }
     UnUseTmpHeap(2,p);
 
-    factory.p = p;
+    erts_factory_proc_init(&factory, p);
     res = hashmap_from_unsorted_array(&factory, hxns, size, 0);
+    erts_factory_close(&factory);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -515,8 +516,9 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
 	hxns[i].i    = i;
     }
 
-    factory.p = p;
+    erts_factory_proc_init(&factory, p);
     res = hashmap_from_unsorted_array(&factory, hxns, sz, 0);
+    erts_factory_close(&factory);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1063,8 +1065,9 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) {
 	    hxns[i].i    = i;
 	}
 
-        factory.p = p;
+        erts_factory_proc_init(&factory, p);
 	res = hashmap_from_unsorted_array(&factory, hxns, n, 0);
+	erts_factory_close(&factory);
 
 	erts_free(ERTS_ALC_T_TMP, (void *) hxns);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -1107,8 +1110,9 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
 	hxns[i].i    = i;
     }
 
-    factory.p = p;
+    erts_factory_proc_init(&factory, p);
     res = hashmap_from_unsorted_array(&factory, hxns, n, 0);
+    erts_factory_close(&factory);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
@@ -2504,6 +2508,9 @@ int erts_validate_and_sort_flatmap(flatmap_t* mp)
     return 1;
 }
 
+#if 0 /* Can't get myself to remove this beautiful piece of code
+         for probabilistic overestimation of nr of nodes in a hashmap */
+
 /* Really rough estimate of sqrt(x)
  * Guaranteed not to be less than sqrt(x)
  */
@@ -2525,7 +2532,10 @@ static int int_sqrt_ceiling(Uint x)
     }
 }
 
-Uint hashmap_over_estimated_heap_size(Uint k)
+/* May not be enough if hashing is broken (not uniform)
+ * or if hell freezes over.
+ */
+Uint hashmap_overestimated_node_count(Uint k)
 {
     /* k is nr of key-value pairs.
        N(k) is expected nr of nodes in hamt.
@@ -2539,12 +2549,9 @@ Uint hashmap_over_estimated_heap_size(Uint k)
        by 15 std.devs above the average, which gives a probability for overrun
        less than 1.0e-49 (same magnitude as a git SHA1 collision).
      */
-    Uint max_nodes = 2*k/5 + (15/3)*int_sqrt_ceiling(k);
-    return (k*2 +     /* leaf cons cells */
-	    k +       /* leaf list terms */
-	    max_nodes*2); /* headers + parent boxed terms */
+    return 2*k/5 + 1 + (15/3)*int_sqrt_ceiling(k);
 }
-
+#endif
 
 BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) {
     if (is_hashmap(BIF_ARG_1)) {
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 2cc6768bfc..b5941c5c9a 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -91,7 +91,6 @@ Eterm  erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 			      Uint *upsz, struct ErtsEStack_ *sp);
 
 int    erts_validate_and_sort_flatmap(flatmap_t* map);
-Uint   hashmap_over_estimated_heap_size(Uint n);
 void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse);
 Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
 Eterm* hashmap_iterator_prev(struct ErtsWStack_* s);
@@ -191,5 +190,18 @@ typedef struct hashmap_head_s {
 
 #define hashmap_index(hash)      (((Uint32)hash) & 0xf)
 
+/* hashmap heap size:
+   [one cons cell + one list term in parent node] per key
+   [one header + one boxed term in parent node] per inner node
+   [one header + one size word] for root node
+*/
+#define HASHMAP_HEAP_SIZE(KEYS,NODES) ((KEYS)*3 + (NODES)*2)
+#ifdef DEBUG
+#  define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (KEYS)
+#else
+#  define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (2*(KEYS)/5)
+#endif
+#define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \
+        HASHMAP_HEAP_SIZE(KEYS,HASHMAP_ESTIMATED_NODE_COUNT(KEYS))
 
 #endif
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ccfc2e6458..13e084ce10 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -237,8 +237,7 @@ erts_msg_distext2heap(Process *pp,
     Eterm msg;
     Uint tok_sz = 0;
     Eterm *hp = NULL;
-    Eterm *hp_end = NULL;
-    ErlOffHeap *ohp;
+    ErtsHeapFactory factory;
     Sint sz;
 
     *bpp = NULL;
@@ -250,36 +249,26 @@ erts_msg_distext2heap(Process *pp,
 	tok_sz = heap_frag->used_size;
 	sz += tok_sz;
     }
-    if (pp)
+    if (pp) {
+	ErlOffHeap *ohp;
 	hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp);
+    }
     else {
 	*bpp = new_message_buffer(sz);
 	hp = (*bpp)->mem;
-	ohp = &(*bpp)->off_heap;
     }
-    hp_end = hp + sz;
-    msg = erts_decode_dist_ext(&hp, ohp, dist_extp);
+    erts_factory_message_init(&factory, pp, hp, *bpp);
+    msg = erts_decode_dist_ext(&factory, dist_extp);
     if (is_non_value(msg))
 	goto decode_error;
     if (is_not_nil(*tokenp)) {
 	ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
-	*tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp);
+	hp = erts_produce_heap(&factory, tok_sz, 0);
+	*tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap);
 	erts_cleanup_offheap(&heap_frag->off_heap);
     }
     erts_free_dist_ext_copy(dist_extp);
-    if (hp_end != hp) {
-	if (!(*bpp)) {
-	    HRelease(pp, hp_end, hp);
-	}
-	else {
-	    Uint final_size = hp - &(*bpp)->mem[0];
-	    Eterm brefs[2] = {msg, *tokenp};
-	    ASSERT(sz - (hp_end - hp) == final_size);
-	    *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2);
-	    msg = brefs[0];
-	    *tokenp = brefs[1];
-	}
-    }
+    erts_factory_close(&factory);
     return msg;
 
  decode_error:
@@ -288,13 +277,7 @@ erts_msg_distext2heap(Process *pp,
 	erts_cleanup_offheap(&heap_frag->off_heap);
     }
     erts_free_dist_ext_copy(dist_extp);
-    if (*bpp) {
-	free_message_buffer(*bpp);
-	*bpp = NULL;
-    }    
-    else if (hp) {
-	HRelease(pp, hp_end, hp);
-    }
+    *bpp = NULL;
     return THE_NON_VALUE;
  }
 
@@ -851,10 +834,11 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
 }
 
 void
-erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *msg)
+erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory,
+				    ErlMessage *msg)
 {
     if (is_value(ERL_MESSAGE_TERM(msg)))
-	erts_move_msg_mbuf_to_heap(hpp, ohp, msg);
+	erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg);
     else if (msg->data.dist_ext) {
 	ASSERT(msg->data.dist_ext->heap_size >= 0);
 	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
@@ -862,12 +846,11 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms
 	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
 	    ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg),
 						 heap_frag->used_size,
-						 hpp,
-						 ohp);
+						 &factory->hp,
+						 factory->off_heap);
 	    erts_cleanup_offheap(&heap_frag->off_heap);
 	}
-	ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(hpp,
-						     ohp,
+	ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory,
 						     msg->data.dist_ext);
 	erts_free_dist_ext_copy(msg->data.dist_ext);
 	msg->data.dist_ext = NULL;
@@ -1134,15 +1117,263 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
     }
 }
 
+void erts_factory_proc_init(ErtsHeapFactory* factory,
+			    Process* p)
+{
+    erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p));
+}
+
+void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory,
+				     Process* p,
+				     Sint size)
+{
+    factory->mode     = FACTORY_HALLOC;
+    factory->p        = p;
+    factory->hp_start = HAlloc(p, size);
+    factory->hp       = factory->hp_start;
+    factory->hp_end   = factory->hp_start + size;
+    factory->off_heap = &p->off_heap;
+    factory->off_heap_saved.first    = p->off_heap.first;
+    factory->off_heap_saved.overhead = p->off_heap.overhead;
+    factory->heap_frags_saved = p->mbuf;
+    factory->heap_frags = NULL; /* not used */
+    factory->alloc_type = 0; /* not used */
+}
+
+void erts_factory_message_init(ErtsHeapFactory* factory,
+			       Process* rp,
+			       Eterm* hp,
+			       ErlHeapFragment* bp)
+{
+    if (bp) {
+	factory->mode     = FACTORY_HEAP_FRAGS;
+	factory->p        = NULL;
+	factory->hp_start = bp->mem;
+	factory->hp       = hp ? hp : bp->mem;
+	factory->hp_end   = bp->mem + bp->alloc_size;
+	factory->off_heap = &bp->off_heap;
+        factory->heap_frags = bp;
+        factory->heap_frags_saved = bp;
+        factory->alloc_type = ERTS_ALC_T_HEAP_FRAG;
+        ASSERT(!bp->next);
+    }
+    else {
+	factory->mode     = FACTORY_HALLOC;
+	factory->p        = rp;
+	factory->hp_start = hp;
+	factory->hp       = hp;
+	factory->hp_end   = HEAP_TOP(rp);
+	factory->off_heap = &rp->off_heap;
+        factory->heap_frags_saved = rp->mbuf;
+        factory->heap_frags = NULL; /* not used */
+        factory->alloc_type = 0; /* not used */
+    }
+    factory->off_heap_saved.first    = factory->off_heap->first;
+    factory->off_heap_saved.overhead = factory->off_heap->overhead;
+
+    ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end);
+}
+
+void erts_factory_static_init(ErtsHeapFactory* factory,
+			     Eterm* hp,
+			     Uint size,
+			     ErlOffHeap* off_heap)
+{
+    factory->mode     = FACTORY_STATIC;
+    factory->hp_start = hp;
+    factory->hp       = hp;
+    factory->hp_end   = hp + size;
+    factory->off_heap = off_heap;
+    factory->off_heap_saved.first    = factory->off_heap->first;
+    factory->off_heap_saved.overhead = factory->off_heap->overhead;
+}
+
+/* When we know the term is an immediate and need no heap.
+*/
+void erts_factory_dummy_init(ErtsHeapFactory* factory)
+{
+    factory->mode = FACTORY_CLOSED;
+}
+
+static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra);
+
 Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
 {
     Eterm* res;
-    if (factory->p) {
-        res = HAllocX(factory->p, need, xtra);
-    } else {
-        res = factory->hp;
-        factory->hp += need;
+
+    ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
+    if (factory->hp + need > factory->hp_end) {
+	reserve_heap(factory, need, xtra);
     }
+    res = factory->hp;
+    factory->hp += need;
     return res;
 }
 
+Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need)
+{
+    ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
+    if (factory->hp + need > factory->hp_end) {
+	reserve_heap(factory, need, 200);
+    }
+    return factory->hp;
+}
+
+static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
+{
+    ErlHeapFragment* bp;
+
+    switch (factory->mode) {
+    case FACTORY_HALLOC:
+	HRelease(factory->p, factory->hp_end, factory->hp);
+	factory->hp     = HAllocX(factory->p, need, xtra);
+	factory->hp_end = factory->hp + need;
+	return;
+
+    case FACTORY_HEAP_FRAGS:
+	bp = factory->heap_frags;
+
+        if (bp) {
+	    ASSERT(factory->hp > bp->mem);
+	    ASSERT(factory->hp <= factory->hp_end);
+	    ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
+
+	    bp->used_size = factory->hp - bp->mem;
+        }
+	bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type,
+						ERTS_HEAP_FRAG_SIZE(need+xtra));
+	bp->next = factory->heap_frags;
+	factory->heap_frags = bp;
+	bp->alloc_size = need + xtra;
+	bp->used_size = need;
+	bp->off_heap.first = NULL;
+	bp->off_heap.overhead = 0;
+
+	factory->hp     = bp->mem;
+	factory->hp_end = bp->mem + bp->alloc_size;
+	return;
+
+    case FACTORY_STATIC:
+    case FACTORY_CLOSED:
+    default:
+	ASSERT(!"Invalid factory mode");
+    }
+}
+
+void erts_factory_close(ErtsHeapFactory* factory)
+{
+    ErlHeapFragment* bp;
+
+    switch (factory->mode) {
+    case FACTORY_HALLOC:
+	HRelease(factory->p, factory->hp_end, factory->hp);
+	break;
+
+    case FACTORY_HEAP_FRAGS:
+	bp = factory->heap_frags;
+
+	if (bp) {
+	    ASSERT(factory->hp >= bp->mem);
+	    ASSERT(factory->hp <= factory->hp_end);
+	    ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
+
+	    bp->used_size = factory->hp - bp->mem;
+        }
+	break;
+    case FACTORY_STATIC: break;
+    case FACTORY_CLOSED: break;
+    default:
+	ASSERT(!"Invalid factory mode");
+    }
+    factory->mode = FACTORY_CLOSED;
+}
+
+void erts_factory_undo(ErtsHeapFactory* factory)
+{
+    ErlHeapFragment* bp;
+    struct erl_off_heap_header *hdr, **hdr_nextp;
+
+    switch (factory->mode) {
+    case FACTORY_HALLOC:
+    case FACTORY_STATIC:
+	/* Cleanup off-heap
+	 */
+	hdr_nextp = NULL;
+        for (hdr = factory->off_heap->first;
+	     hdr != factory->off_heap_saved.first;
+	     hdr = hdr->next) {
+
+	    hdr_nextp = &hdr->next;
+        }
+
+        if (hdr_nextp != NULL) {
+	    *hdr_nextp = NULL;
+	    erts_cleanup_offheap(factory->off_heap);
+	    factory->off_heap->first    = factory->off_heap_saved.first;
+	    factory->off_heap->overhead = factory->off_heap_saved.overhead;
+        }
+
+        if (factory->mode == FACTORY_HALLOC) {
+            /* Free heap frags
+             */
+            bp = factory->p->mbuf;
+            if (bp != factory->heap_frags_saved) {
+                do {
+                    ErlHeapFragment *next_bp = bp->next;
+                    ASSERT(bp->off_heap.first == NULL);
+                    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
+                                   ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
+                    bp = next_bp;
+                } while (bp != factory->heap_frags_saved);
+
+                factory->p->mbuf = bp;
+            }
+
+            /* Rollback heap top
+	     */
+            if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */
+                ASSERT(factory->hp_start >= HEAP_START(factory->p));
+                ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p));
+
+                HEAP_TOP(factory->p) = factory->hp_start;
+            }
+            else {
+                ASSERT(factory->heap_frags_saved == factory->p->mbuf);
+                if (factory->hp_start == factory->heap_frags_saved->mem) {
+                    factory->p->mbuf = factory->p->mbuf->next;
+                    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
+                                   ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
+                }
+                else if (factory->hp_start != factory->hp_end) {
+                    unsigned remains = factory->hp_start - factory->heap_frags_saved->mem;
+                    ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size);
+                    factory->heap_frags_saved->used_size = remains;
+                }
+            }
+        }
+        break;
+
+    case FACTORY_HEAP_FRAGS:
+        bp = factory->heap_frags;
+        do {
+            ErlHeapFragment* next_bp = bp->next;
+
+            erts_cleanup_offheap(&bp->off_heap);
+            ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
+                           ERTS_HEAP_FRAG_SIZE(bp->size));
+            bp = next_bp;
+        }while (bp != NULL);
+	break;
+
+    case FACTORY_CLOSED: break;
+    default:
+	ASSERT(!"Invalid factory mode");
+    }
+    factory->mode = FACTORY_CLOSED;
+#ifdef DEBUG
+    factory->p = NULL;
+    factory->hp = NULL;
+    factory->heap_frags = NULL;
+#endif
+}
+
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 1e1dafee90..39946f2a14 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -51,6 +51,44 @@ typedef struct erl_off_heap {
 	(OHP)->first = NULL;			\
 	(OHP)->overhead = 0;			\
     } while (0)
+
+typedef struct {
+    enum {
+        FACTORY_CLOSED = 0,
+        FACTORY_HALLOC,
+        FACTORY_HEAP_FRAGS,
+        FACTORY_STATIC
+    } mode;
+    Process* p;
+    Eterm* hp_start;
+    Eterm* hp;
+    Eterm* hp_end;
+    struct erl_heap_fragment* heap_frags;
+    struct erl_heap_fragment* heap_frags_saved;
+    ErlOffHeap* off_heap;
+    ErlOffHeap off_heap_saved;
+    Uint32 alloc_type;
+} ErtsHeapFactory;
+
+void erts_factory_proc_init(ErtsHeapFactory*, Process*);
+void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size);
+void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*);
+void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*);
+void erts_factory_dummy_init(ErtsHeapFactory*);
+
+Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
+Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need);
+void erts_factory_close(ErtsHeapFactory*);
+void erts_factory_undo(ErtsHeapFactory*);
+
+#ifdef CHECK_FOR_HOLES
+# define ERTS_FACTORY_HOLE_CHECK(f) do {    \
+        /*if ((f)->p) erts_check_for_holes((f)->p);*/ \
+    } while (0)
+#else
+# define ERTS_FACTORY_HOLE_CHECK(p)
+#endif
+
 #include "external.h"
 #include "erl_process.h"
 
@@ -68,21 +106,6 @@ struct erl_heap_fragment {
     Eterm mem[1];		/* Data */
 };
 
-typedef struct {
-    Process* p;
-    Eterm* hp;
-} ErtsHeapFactory;
-
-Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
-#ifdef CHECK_FOR_HOLES
-# define ERTS_FACTORY_HOLE_CHECK(f) do {    \
-        if ((f)->p) erts_check_for_holes((f)->p); \
-    } while (0)
-#else
-# define ERTS_FACTORY_HOLE_CHECK(p)
-#endif
-
-
 typedef struct erl_mesg {
     struct erl_mesg* next;	/* Next message */
     union {
@@ -139,7 +162,7 @@ typedef struct {
     *(p)->msg.last = (mp); \
     (p)->msg.last = &(mp)->next; \
     (p)->msg.len++; \
-} while(0)
+} while (0)
 
 
 #ifdef ERTS_SMP
@@ -212,17 +235,23 @@ do {									\
 do {									\
     if ((M)->data.attached) {						\
 	Uint need__ = erts_msg_attached_data_size((M));			\
+        { SWPO ; }							\
  	if ((ST) - (HT) >= need__) {					\
-	    Uint *htop__ = (HT);					\
-	    erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\
-	    ASSERT(htop__ - (HT) <= need__);				\
-	    (HT) = htop__;						\
+            ErtsHeapFactory factory__;                                  \
+            erts_factory_proc_prealloc_init(&factory__, (P), need__);   \
+	    erts_move_msg_attached_data_to_heap(&factory__, (M));       \
+            erts_factory_close(&factory__);                             \
+            if ((P)->mbuf != NULL) {                                    \
+                /* Heap was exhausted by messages. This is a rare case */   \
+                /* that can currently (OTP 18) only happen if hamts are */  \
+                /* far exceeding the estimated heap size. Do GC. */         \
+                (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
+            }								\
 	}								\
 	else {								\
-	    { SWPO ; }							\
 	    (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
-	    { SWPI ; }							\
 	}								\
+        { SWPI ; }							\
 	ASSERT(!(M)->data.attached);					\
     }									\
 } while (0)
@@ -266,23 +295,21 @@ void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
 void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *);
 
 Uint erts_msg_attached_data_size_aux(ErlMessage *msg);
-void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *);
-
+void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *);
 Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **,
 			    Eterm *, ErtsDistExternal *);
 
 void erts_cleanup_offheap(ErlOffHeap *offheap);
 
 
-ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg);
+ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg)
+ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp)
 {
-    const ErlHeapFragment *bp;
     Uint sz = 0;
-    for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) {
+    for ( ; bp!=NULL; bp=bp->next) {
 	sz += bp->used_size;
     }
     return sz;
@@ -292,7 +319,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
 {
     ASSERT(msg->data.attached);
     if (is_value(ERL_MESSAGE_TERM(msg)))
-	return erts_msg_used_frag_sz(msg);
+	return erts_used_frag_sz(msg->data.heap_frag);
     else if (msg->data.dist_ext->heap_size < 0)
 	return erts_msg_attached_data_size_aux(msg);
     else {
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index fff267ff2a..b28bc498f6 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1003,7 +1003,9 @@ erts_pid2proc_opt(Process *c_p,
 void
 erts_proc_lock_init(Process *p)
 {
+#if ERTS_PROC_LOCK_OWN_IMPL || defined(ERTS_PROC_LOCK_DEBUG)
     int i;
+#endif
 #if ERTS_PROC_LOCK_OWN_IMPL
     /* We always start with all locks locked */
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index fe48298155..0a69172980 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -94,9 +94,9 @@ static Uint is_external_string(Eterm obj, int* p_is_string);
 static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 struct B2TContext_t;
-static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*);
+static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
-static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
+static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*);
 static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*);
 static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1);
 
@@ -930,8 +930,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size)
 ** on return hpp is updated to point after allocated data
 */
 Eterm
-erts_decode_dist_ext(Eterm** hpp,
-		     ErlOffHeap* off_heap,
+erts_decode_dist_ext(ErtsHeapFactory* factory,
 		     ErtsDistExternal *edep)
 {
     Eterm obj;
@@ -951,7 +950,7 @@ erts_decode_dist_ext(Eterm** hpp,
 	    goto error;
 	ep++;
     }
-    ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL);
+    ep = dec_term(edep, factory, ep, &obj, NULL);
     if (!ep)
 	goto error;
 
@@ -960,19 +959,22 @@ erts_decode_dist_ext(Eterm** hpp,
     return obj;
 
  error:
+    erts_factory_undo(factory);
 
     bad_dist_ext(edep);
 
     return THE_NON_VALUE;
 }
 
-Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
+Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext)
 {
     Eterm obj;
     byte *ep = *ext;
-    if (*ep++ != VERSION_MAGIC)
+    if (*ep++ != VERSION_MAGIC) {
+        erts_factory_undo(factory);
 	return THE_NON_VALUE;
-    ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL);
+    }
+    ep = dec_term(NULL, factory, ep, &obj, NULL);
     if (!ep) {
 #ifdef DEBUG
 	bin_write(ERTS_PRINT_STDERR,NULL,*ext,500);
@@ -983,10 +985,10 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
     return obj;
 }
 
-Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
+Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext)
 {
     Eterm obj;
-    ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL);
+    ext = dec_term(NULL, factory, ext, &obj, NULL);
     ASSERT(ext);
     return obj;
 }
@@ -995,9 +997,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
 
 BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
 {
+    ErtsHeapFactory factory;
     Eterm res;
-    Eterm *hp;
-    Eterm *hendp;
     Sint hsz;
     ErtsDistExternal ede;
     Eterm *tp;
@@ -1044,12 +1045,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
     if (hsz < 0)
 	goto badarg;
 
-    hp = HAlloc(BIF_P, (Uint) hsz);
-    hendp = hp + hsz;
-
-    res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede);
-
-    HRelease(BIF_P, hendp, hp);
+    erts_factory_proc_prealloc_init(&factory, BIF_P, hsz);
+    res = erts_decode_dist_ext(&factory, &ede);
+    erts_factory_close(&factory);
 
     if (is_value(res))
 	BIF_RET(res);
@@ -1177,13 +1175,11 @@ typedef struct {
     byte*  ep;
     Eterm  res;
     Eterm* next;
-    Eterm* hp_start;
-    Eterm* hp;
-    Eterm* hp_end;
+    ErtsHeapFactory factory;
     int remaining_n;
     char* remaining_bytes;
     Eterm* maps_list;
-    struct dec_term_hamt_placeholder* hamt_list;
+    ErtsPStack hamt_array;
 } B2TDecodeContext;
 
 typedef struct {
@@ -1307,10 +1303,12 @@ binary2term_abort(ErtsBinary2TermState *state)
 }
 
 static ERTS_INLINE Eterm
-binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
+binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state,
+		   ErtsHeapFactory* factory)
 {
     Eterm res;
-    if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL))
+
+    if (!dec_term(edep, factory, state->extp, &res, NULL))
 	res = THE_NON_VALUE;
     if (state->exttmp) {
 	state->exttmp = 0;
@@ -1343,9 +1341,9 @@ erts_binary2term_abort(ErtsBinary2TermState *state)
 }
 
 Eterm
-erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp)
+erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory)
 {
-    return binary2term_create(NULL,state, hpp, ohp);
+    return binary2term_create(NULL,state, factory);
 }
 
 static void b2t_destroy_context(B2TContext* context)
@@ -1354,8 +1352,21 @@ static void b2t_destroy_context(B2TContext* context)
                                          ERTS_ALC_T_EXT_TERM_DATA);
     context->aligned_alloc = NULL;
     binary2term_abort(&context->b2ts);
-    if (context->state == B2TUncompressChunk) {
+    switch (context->state) {
+    case B2TUncompressChunk:
 	erl_zlib_inflate_finish(&context->u.uc.stream);
+	break;
+    case B2TDecode:
+    case B2TDecodeList:
+    case B2TDecodeTuple:
+    case B2TDecodeString:
+    case B2TDecodeBinary:
+	if (context->u.dc.hamt_array.pstart) {
+	    erts_free(context->u.dc.hamt_array.alloc_type,
+		      context->u.dc.hamt_array.pstart);
+	}
+	break;
+    default:;
     }
 }
 
@@ -1506,11 +1517,9 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
             ctx->u.dc.ep = ctx->b2ts.extp;
             ctx->u.dc.res = (Eterm) (UWord) NULL;
             ctx->u.dc.next = &ctx->u.dc.res;
-            ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size);
-            ctx->u.dc.hp       = ctx->u.dc.hp_start;
-            ctx->u.dc.hp_end   = ctx->u.dc.hp_start + ctx->heap_size;
+	    erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size);
 	    ctx->u.dc.maps_list = NULL;
-	    ctx->u.dc.hamt_list = NULL;
+	    ctx->u.dc.hamt_array.pstart = NULL;
             ctx->state = B2TDecode;
             /*fall through*/
 	case B2TDecode:
@@ -1520,11 +1529,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
         case B2TDecodeBinary: {
 	    ErtsDistExternal fakedep;
             fakedep.flags = ctx->flags;
-            dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx);
+            dec_term(&fakedep, NULL, NULL, NULL, ctx);
             break;
 	}
         case B2TDecodeFail:
-            HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start);
             /*fall through*/
         case B2TBadArg:
             BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION);
@@ -1549,11 +1557,11 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
         case B2TDone:
             b2t_destroy_context(ctx);
 
-            if (ctx->u.dc.hp > ctx->u.dc.hp_end) {
+            if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) {
                 erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
-                         __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end);
+                         __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end);
             }
-            HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp);
+	    erts_factory_close(&ctx->u.dc.factory);
 
             if (!is_first_call) {
                 erts_set_gc_state(p, 1);
@@ -2247,7 +2255,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
 }
 
 static byte*
-dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
+dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp)
 {
     Eterm sysname;
     Uint data;
@@ -2286,15 +2294,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
     if(node == erts_this_node) {
 	*objp = make_internal_pid(data);
     } else {
-	ExternalThing *etp = (ExternalThing *) *hpp;
-	*hpp += EXTERNAL_THING_HEAD_SIZE + 1;
+	ExternalThing *etp = (ExternalThing *) factory->hp;
+	factory->hp += EXTERNAL_THING_HEAD_SIZE + 1;
 
 	etp->header = make_external_pid_header(1);
-	etp->next = off_heap->first;
+	etp->next = factory->off_heap->first;
 	etp->node = node;
 	etp->data.ui[0] = data;
 
-	off_heap->first = (struct erl_off_heap_header*) etp;
+	factory->off_heap->first = (struct erl_off_heap_header*) etp;
 	*objp = make_external_pid(etp);
     }
     return ep;
@@ -2905,69 +2913,43 @@ is_external_string(Eterm list, int* p_is_string)
     return len;
 }
 
-/* Assumes that the ones to undo are preluding the list. */ 
-static void
-undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
-{
-    const Uint area_sz = (end - start) * sizeof(Eterm);
-    struct erl_off_heap_header* hdr;
-    struct erl_off_heap_header** hdr_nextp = NULL;
-
-    for (hdr = off_heap->first; ; hdr=hdr->next) {
-	if (!in_area(hdr, start, area_sz)) {
-	    if (hdr_nextp != NULL) {
-		*hdr_nextp = NULL;
-		erts_cleanup_offheap(off_heap);
-		off_heap->first = hdr;
-	    }
-	    break;
-	}
-	hdr_nextp = &hdr->next;
-    }    
-
-    /* Assert that the ones to undo were indeed preluding the list. */ 
-#ifdef DEBUG
-    for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) {
-	ASSERT(!in_area(hdr, start, area_sz));
-    }    
-#endif /* DEBUG */
-}
 
-struct dec_term_hamt_placeholder
+struct dec_term_hamt
 {
-    struct dec_term_hamt_placeholder* next;
     Eterm* objp; /* write result here */
     Uint size;   /* nr of leafs */
-    Eterm leafs[1];
+    Eterm* leaf_array;
 };
 
-#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \
-    (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm))
 
 /* Decode term from external format into *objp.
-** On failure return NULL and *hpp will be unchanged.
+** On failure calls erts_factory_undo() and returns NULL
 */
 static byte*
-dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
-         Eterm* objp, B2TContext* ctx)
+dec_term(ErtsDistExternal *edep,
+	 ErtsHeapFactory* factory,
+	 byte* ep,
+         Eterm* objp,
+	 B2TContext* ctx)
 {
-    Eterm* hp_saved;
+#define PSTACK_TYPE struct dec_term_hamt
+    PSTACK_DECLARE(hamt_array, 5);
     int n;
     ErtsAtomEncoding char_enc;
     register Eterm* hp;        /* Please don't take the address of hp */
     Eterm *maps_list;   /* for preprocessing of small maps */
-    struct dec_term_hamt_placeholder* hamt_list;   /* for preprocessing of big maps */
     Eterm* next;
     SWord reds;
+#ifdef DEBUG
+    Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp;
+#endif
 
     if (ctx) {
-        hp_saved = ctx->u.dc.hp_start;
         reds     = ctx->reds;
         next     = ctx->u.dc.next;
         ep       = ctx->u.dc.ep;
-        hpp      = &ctx->u.dc.hp;
+	factory  = &ctx->u.dc.factory;
 	maps_list = ctx->u.dc.maps_list;
-        hamt_list = ctx->u.dc.hamt_list;
 
         if (ctx->state != B2TDecode) {
             int n_limit = reds;
@@ -3012,7 +2994,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                 break;
 
             case B2TDecodeString:
-                hp = *hpp;
+                hp = factory->hp;
                 hp[-1] = make_list(hp);  /* overwrite the premature NIL */
                 while (n-- > 0) {
                     hp[0] = make_small(*ep++);
@@ -3020,7 +3002,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                     hp += 2;
                 }
                 hp[-1] = NIL;
-                *hpp = hp;
+		factory->hp = hp;
                 break;
 
             case B2TDecodeBinary:
@@ -3042,16 +3024,18 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
                 return NULL;
             }
         }
+	PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK);
+	if (ctx->u.dc.hamt_array.pstart) {
+	    PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array);
+	}
     }
     else {
-        hp_saved = *hpp;
         reds = ERTS_SWORD_MAX;
         next = objp;
         *next = (Eterm) (UWord) NULL;
 	maps_list = NULL;
-        hamt_list = NULL;
     }
-    hp = *hpp;
+    hp = factory->hp;
 
     while (next != NULL) {
 
@@ -3288,9 +3272,9 @@ dec_term_atom_common:
 		break;
 	    }
 	case PID_EXT:
-	    *hpp = hp;
-	    ep = dec_pid(edep, hpp, ep, off_heap, objp);
-	    hp = *hpp;
+	    factory->hp = hp;
+	    ep = dec_pid(edep, factory, ep, objp);
+	    hp = factory->hp;
 	    if (ep == NULL) {
 		goto error;
 	    }
@@ -3324,11 +3308,11 @@ dec_term_atom_common:
 		    hp += EXTERNAL_THING_HEAD_SIZE + 1;
 		    
 		    etp->header = make_external_port_header(1);
-		    etp->next = off_heap->first;
+		    etp->next = factory->off_heap->first;
 		    etp->node = node;
 		    etp->data.ui[0] = num;
 
-		    off_heap->first = (struct erl_off_heap_header*)etp;
+		    factory->off_heap->first = (struct erl_off_heap_header*)etp;
 		    *objp = make_external_port(etp);
 		}
 
@@ -3408,10 +3392,10 @@ dec_term_atom_common:
 #else
 		    etp->header = make_external_ref_header(ref_words);
 #endif
-		    etp->next = off_heap->first;
+		    etp->next = factory->off_heap->first;
 		    etp->node = node;
 
-		    off_heap->first = (struct erl_off_heap_header*)etp;
+		    factory->off_heap->first = (struct erl_off_heap_header*)etp;
 		    *objp = make_external_ref(etp);
 		    ref_num = &(etp->data.ui32[0]);
 		}
@@ -3451,9 +3435,9 @@ dec_term_atom_common:
 		    hp += PROC_BIN_SIZE;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
-		    pb->next = off_heap->first;
-		    off_heap->first = (struct erl_off_heap_header*)pb;
-                    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+		    pb->next = factory->off_heap->first;
+		    factory->off_heap->first = (struct erl_off_heap_header*)pb;
+		    OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -3503,9 +3487,9 @@ dec_term_atom_common:
 		    pb = (ProcBin *) hp;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
-		    pb->next = off_heap->first;
-		    off_heap->first = (struct erl_off_heap_header*)pb;
-                    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+		    pb->next = factory->off_heap->first;
+		    factory->off_heap->first = (struct erl_off_heap_header*)pb;
+		    OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -3557,9 +3541,9 @@ dec_term_atom_common:
 		if ((ep = dec_atom(edep, ep, &name)) == NULL) {
 		    goto error;
 		}
-		*hpp = hp;
-		ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL);
-		hp = *hpp;
+		factory->hp = hp;
+		ep = dec_term(edep, factory, ep, &temp, NULL);
+		hp = factory->hp;
 		if (ep == NULL) {
 		    goto error;
 		}
@@ -3631,15 +3615,11 @@ dec_term_atom_common:
                     }
                 }
                 else {  /* Make hamt */
-                    struct dec_term_hamt_placeholder* holder =
-                        (struct dec_term_hamt_placeholder*) hp;
-
-                    holder->next = hamt_list;
-                    hamt_list    = holder;
-                    holder->objp = objp;
-                    holder->size = size;
+                    struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array);
 
-                    hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE;
+                    hamt->objp = objp;
+                    hamt->size = size;
+                    hamt->leaf_array = hp;
 
                     for (n = size; n; n--) {
                         CDR(hp) = (Eterm) COMPRESS_POINTER(next);
@@ -3681,9 +3661,9 @@ dec_term_atom_common:
 		if ((ep = dec_atom(edep, ep, &module)) == NULL) {
 		    goto error;
 		}
-		*hpp = hp;
+		factory->hp = hp;
 		/* Index */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+		if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3692,7 +3672,7 @@ dec_term_atom_common:
 		old_index = unsigned_val(temp);
 
 		/* Uniq */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+		if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3704,8 +3684,8 @@ dec_term_atom_common:
 		 * It is safe to link the fun into the fun list only when
 		 * no more validity tests can fail.
 		 */
-		funp->next = off_heap->first;
-		off_heap->first = (struct erl_off_heap_header*)funp;
+		funp->next = factory->off_heap->first;
+		factory->off_heap->first = (struct erl_off_heap_header*)funp;
 
 		funp->fe = erts_put_fun_entry2(module, old_uniq, old_index,
 					       uniq, index, arity);
@@ -3716,7 +3696,7 @@ dec_term_atom_common:
 		}
 		funp->native_address = funp->fe->native_address;
 #endif
-		hp = *hpp;
+		hp = factory->hp;
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
@@ -3742,14 +3722,14 @@ dec_term_atom_common:
 		ep += 4;
 		hp += ERL_FUN_SIZE;
 		hp += num_free;
-		*hpp = hp;
+		factory->hp = hp;
 		funp->thing_word = HEADER_FUN;
 		funp->num_free = num_free;
 		*objp = make_fun(funp);
 
 		/* Creator pid */
 		if (*ep != PID_EXT 
-		    || (ep = dec_pid(edep, hpp, ++ep, off_heap,
+		    || (ep = dec_pid(edep, factory, ++ep,
 				     &funp->creator))==NULL) { 
 		    goto error;
 		}
@@ -3760,7 +3740,7 @@ dec_term_atom_common:
 		}
 
 		/* Index */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+		if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3769,7 +3749,7 @@ dec_term_atom_common:
 		old_index = unsigned_val(temp);
 
 		/* Uniq */
-		if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) {
+		if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
 		    goto error;
 		}
 		if (!is_small(temp)) {
@@ -3780,8 +3760,8 @@ dec_term_atom_common:
 		 * It is safe to link the fun into the fun list only when
 		 * no more validity tests can fail.
 		 */
-		funp->next = off_heap->first;
-		off_heap->first = (struct erl_off_heap_header*)funp;
+		funp->next = factory->off_heap->first;
+		factory->off_heap->first = (struct erl_off_heap_header*)funp;
 		old_uniq = unsigned_val(temp);
 
 		funp->fe = erts_put_fun_entry(module, old_uniq, old_index);
@@ -3789,7 +3769,7 @@ dec_term_atom_common:
 #ifdef HIPE
 		funp->native_address = funp->fe->native_address;
 #endif
-		hp = *hpp;
+		hp = factory->hp;
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
@@ -3823,9 +3803,9 @@ dec_term_atom_common:
 
 		erts_refc_inc(&pb->val->refc, 1);
 		hp += PROC_BIN_SIZE;
-		pb->next = off_heap->first;
-		off_heap->first = (struct erl_off_heap_header*)pb;
-                OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+		pb->next = factory->off_heap->first;
+		factory->off_heap->first = (struct erl_off_heap_header*)pb;
+		OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
 		pb->flags = 0;
 		*objp = make_binary(pb);
 		break;
@@ -3841,9 +3821,9 @@ dec_term_atom_common:
 
 		erts_refc_inc(&pb->val->refc, 1);
 		hp += PROC_BIN_SIZE;
-		pb->next = off_heap->first;
-		off_heap->first = (struct erl_off_heap_header*)pb;
-                OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+		pb->next = factory->off_heap->first;
+		factory->off_heap->first = (struct erl_off_heap_header*)pb;
+                OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
 		pb->flags = 0;
 
 		sub = (ErlSubBin*)hp;
@@ -3869,9 +3849,11 @@ dec_term_atom_common:
                 if (next || ctx->state != B2TDecode) {
                     ctx->u.dc.ep = ep;
                     ctx->u.dc.next = next;
-                    ctx->u.dc.hp = hp;
+                    ctx->u.dc.factory.hp = hp;
 		    ctx->u.dc.maps_list = maps_list;
-		    ctx->u.dc.hamt_list = hamt_list;
+		    if (!PSTACK_IS_EMPTY(hamt_array)) {
+			PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array);
+		    }
                     ctx->reds = 0;
                     return NULL;
                 }
@@ -3894,40 +3876,36 @@ dec_term_atom_common:
 	maps_list  = next;
     }
 
-    /* Iterate through all the hamts and build tree nodes.
+    ASSERT(hp <= factory->hp_end
+           || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp)));
+    factory->hp = hp;
+    /*
+     * From here on factory may produce (more) heap fragments
      */
-    if (hamt_list) {
-	ErtsHeapFactory factory;
 
-	factory.p = NULL;
-        factory.hp = hp;
-	/* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
+    if (!PSTACK_IS_EMPTY(hamt_array)) {
+	do {
+	    struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array);
 
-        do {
-	    struct dec_term_hamt_placeholder* hamt = hamt_list;
-	    *hamt->objp = erts_hashmap_from_array(&factory,
-						  hamt->leafs,
+	    *hamt->objp = erts_hashmap_from_array(factory,
+						  hamt->leaf_array,
 						  hamt->size,
 						  1);
 	    if (is_non_value(*hamt->objp))
-		goto error;
+		goto error_hamt;
 
-	    hamt_list = hamt->next;
-
-	    /* Yes, we waste a couple of heap words per hamt
-	       for the temporary placeholder */
-	    *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1);
-        } while (hamt_list);
-
-	hp = factory.hp;
+	    (void) PSTACK_POP(hamt_array);
+	} while (!PSTACK_IS_EMPTY(hamt_array));
+	PSTACK_DESTROY(hamt_array);
     }
 
+    ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL);
+
     if (ctx) {
         ctx->state = B2TDone;
 	ctx->reds = reds;
     }
 
-    *hpp = hp;
     return ep;
 
 error:
@@ -3935,11 +3913,12 @@ error:
      * Must unlink all off-heap objects that may have been
      * linked into the process. 
      */
-    if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */
-	hp = *hpp;   /* the largest must be the freshest */
+    if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */
+	factory->hp = hp;   /* the largest must be the freshest */
     }
-    undo_offheap_in_area(off_heap, hp_saved, hp);
-    *hpp = hp_saved;
+error_hamt:
+    erts_factory_undo(factory);
+    PSTACK_DESTROY(hamt_array);
     if (ctx) {
 	ctx->state = B2TDecodeFail;
 	ctx->reds = reds;
@@ -4465,7 +4444,7 @@ init_done:
             if (n <= MAP_SMALL_MAP_LIMIT) {
                 heap_size += 3 + n + 1 + n;
             } else {
-                heap_size += hashmap_over_estimated_heap_size(n);
+                heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n);
             }
 	    break;
 	case STRING_EXT:
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 50fcfa04d6..508ab8dc17 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -148,6 +148,7 @@ typedef struct {
     byte *extp;
     int exttmp;
     Uint extsize;
+    Uint heap_size;
 } ErtsBinary2TermState;
 
 
@@ -185,18 +186,18 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *);
 int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
 			  DistEntry *, ErtsAtomCache *);
 Sint erts_decode_dist_ext_size(ErtsDistExternal *);
-Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *);
+Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *);
 
 Sint erts_decode_ext_size(byte*, Uint);
 Sint erts_decode_ext_size_ets(byte*, Uint);
-Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**);
-Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*);
+Eterm erts_decode_ext(ErtsHeapFactory*, byte**);
+Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*);
 
 Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags);
 
 Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint);
 void erts_binary2term_abort(ErtsBinary2TermState *);
-Eterm erts_binary2term_create(ErtsBinary2TermState *, Eterm **hpp, ErlOffHeap *);
+Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*);
 int erts_debug_max_atom_out_cache_index(void);
 int erts_debug_atom_to_out_cache_index(Eterm);
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index ccc7da265e..07a82b42d2 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1395,31 +1395,26 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp)
 static ERTS_INLINE void
 queue_port_sched_op_reply(Process *rp,
 			  ErtsProcLocks *rp_locksp,
-			  Eterm *hp_start,
-			  Eterm *hp,
-			  Uint h_size,
-			  ErlHeapFragment* bp,
+                          ErtsHeapFactory* factory,
 			  Uint32 *ref_num,
 			  Eterm msg)
 {
-    Eterm ref = make_internal_ref(hp);
+    Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0);
+    Eterm ref;
+
+    ref= make_internal_ref(hp);
     write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]);
     hp += REF_THING_SIZE;
 
     msg = TUPLE2(hp, ref, msg);
     hp += 3;
 
-    if (!bp) {
-	HRelease(rp, hp_start + h_size, hp);
-    }
-    else {
-	Uint used_h_size = hp - hp_start;
-	ASSERT(h_size >= used_h_size);
-	if (h_size > used_h_size)
-	    bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1);
-    }
+    /* SVERK: We used to call erts_resize_message_buffer here
+     * to maybe realloc message.
+     */
+    erts_factory_close(factory);
 
-    erts_queue_message(rp, rp_locksp, bp, msg, NIL);
+    erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL);
 }
 
 static void
@@ -1429,9 +1424,10 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
     if (rp) {
 	ErlOffHeap *ohp;
 	ErlHeapFragment* bp;
+        ErtsHeapFactory factory;
 	Eterm msg_copy;
 	Uint hsz, msg_sz;
-	Eterm *hp, *hp_start;
+	Eterm *hp;
 	ErtsProcLocks rp_locks = 0;
 
 	hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
@@ -1442,22 +1438,22 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
 	    hsz += msg_sz;
 	}
 
-	hp_start = hp = erts_alloc_message_heap(hsz,
+	hp = erts_alloc_message_heap(hsz,
 						&bp,
 						&ohp,
 						rp,
 						&rp_locks);
+        erts_factory_message_init(&factory, rp, hp, bp);
 	if (is_immed(msg))
 	    msg_copy = msg;
-	else
+	else {
 	    msg_copy = copy_struct(msg, msg_sz, &hp, ohp);
+            factory.hp = hp;
+        }
 
 	queue_port_sched_op_reply(rp,
 				  &rp_locks,
-				  hp_start,
-				  hp,
-				  hsz,
-				  bp,
+                                  &factory,
 				  ref_num,
 				  msg_copy);
 
@@ -3896,12 +3892,13 @@ port_sig_control(Port *prt,
 
 	if (res == ERTS_PORT_OP_DONE) {
 	    Eterm msg;
-	    Eterm *hp, *hp_start;
+	    Eterm *hp;
 	    ErlHeapFragment *bp;
 	    ErlOffHeap *ohp;
+            ErtsHeapFactory factory;
 	    Process *rp;
 	    ErtsProcLocks rp_locks = 0;
-	    Uint hsz;
+	    Uint hsz, rsz;
 	    int control_flags;
 
 	    rp = erts_proc_lookup_raw(sigdp->caller);
@@ -3910,17 +3907,19 @@ port_sig_control(Port *prt,
 
 	    control_flags = prt->control_flags;
 
-	    hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
-	    hsz += port_control_result_size(control_flags,
+	    rsz = port_control_result_size(control_flags,
 					    resp_bufp,
 					    &resp_size,
 					    &resp_buf[0]);
+	    hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+
 
-	    hp_start = hp = erts_alloc_message_heap(hsz,
+	    hp = erts_alloc_message_heap(hsz,
 						    &bp,
 						    &ohp,
 						    rp,
 						    &rp_locks);
+            erts_factory_message_init(&factory, rp, hp, bp);
 
 	    msg = write_port_control_result(control_flags,
 					    resp_bufp,
@@ -3929,13 +3928,11 @@ port_sig_control(Port *prt,
 					    &hp,
 					    bp,
 					    ohp);
+            factory.hp = hp;
 
 	    queue_port_sched_op_reply(rp,
 				      &rp_locks,
-				      hp_start,
-				      hp,
-				      hsz,
-				      bp,
+                                      &factory,
 				      sigdp->ref,
 				      msg);
 
@@ -4222,8 +4219,6 @@ port_sig_call(Port *prt,
 	if (res == ERTS_PORT_OP_DONE) {
 	    Eterm msg;
 	    Eterm *hp;
-	    ErlHeapFragment *bp;
-	    ErlOffHeap *ohp;
 	    Process *rp;
 	    ErtsProcLocks rp_locks = 0;
 	    Sint hsz;
@@ -4234,29 +4229,31 @@ port_sig_call(Port *prt,
 
 	    hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size);
 	    if (hsz >= 0) {
-		Eterm *hp_start;
+                ErlHeapFragment* bp;
+                ErlOffHeap* ohp;
+                ErtsHeapFactory factory;
 		byte *endp;
 
 		hsz += 3; /* ok tuple */
 		hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
 
-		hp_start = hp = erts_alloc_message_heap(hsz,
-							&bp,
-							&ohp,
-							rp,
-							&rp_locks);
+                hp = erts_alloc_message_heap(hsz,
+                                             &bp,
+                                             &ohp,
+                                             rp,
+                                             &rp_locks);
 		endp = (byte *) resp_bufp;
-		msg = erts_decode_ext(&hp, ohp, &endp);
+                erts_factory_message_init(&factory, rp, hp, bp);
+		msg = erts_decode_ext(&factory, &endp);
 		if (is_value(msg)) {
+                    hp = erts_produce_heap(&factory,
+                                           3,
+                                           ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE);
 		    msg = TUPLE2(hp, am_ok, msg);
-		    hp += 3;
 
 		    queue_port_sched_op_reply(rp,
 					      &rp_locks,
-					      hp_start,
-					      hp,
-					      hsz,
-					      bp,
+                                              &factory,
 					      sigdp->ref,
 					      msg);
 
@@ -4264,8 +4261,6 @@ port_sig_call(Port *prt,
 			erts_smp_proc_unlock(rp, rp_locks);
 		    goto done;
 		}
-		if (bp)
-		    free_message_buffer(bp);
 		if (rp_locks)
 		    erts_smp_proc_unlock(rp, rp_locks);
 	    }
@@ -4342,10 +4337,11 @@ erts_port_call(Process* c_p,
 	try_call_res = try_imm_drv_call(&try_call_state);
 	switch (try_call_res) {
 	case ERTS_TRY_IMM_DRV_CALL_OK: {
-	    Eterm *hp, *hp_end;
+            ErtsHeapFactory factory;
 	    Sint hsz;
 	    unsigned ret_flags = 0U;
 	    Eterm term;
+            Eterm* hp;
 
 	    res = call_driver_call(c_p->common.id,
 				   prt,
@@ -4365,15 +4361,14 @@ erts_port_call(Process* c_p,
 	    if (hsz < 0)
 		return ERTS_PORT_OP_BADARG;
 	    hsz += 3;
-	    hp = HAlloc(c_p, hsz);
-	    hp_end = hp + hsz;
+            erts_factory_proc_prealloc_init(&factory, c_p, hsz);
 	    endp = (byte *) resp_bufp;
-	    term = erts_decode_ext(&hp, &MSO(c_p), &endp);
+	    term = erts_decode_ext(&factory, &endp);
 	    if (term == THE_NON_VALUE)
 		return ERTS_PORT_OP_BADARG;
+            hp = erts_produce_heap(&factory,3,0);
 	    *retvalp = TUPLE2(hp, am_ok, term);
-	    hp += 3;
-	    HRelease(c_p, hp_end, hp);
+            erts_factory_close(&factory);
 	    if (resp_bufp != &resp_buf[0]
 		&& !(ret_flags & DRIVER_CALL_KEEP_BUFFER))
 		driver_free(resp_bufp);
@@ -4508,12 +4503,11 @@ port_sig_info(Port *prt,
 				    prt,
 				    sigdp->u.info.item);
 	if (is_value(value)) {
+            ErtsHeapFactory factory;
+            erts_factory_message_init(&factory, NULL, hp, bp);
 	    queue_port_sched_op_reply(rp,
 				      &rp_locks,
-				      hp_start,
-				      hp,
-				      hsz,
-				      bp,
+                                      &factory,
 				      sigdp->ref,
 				      value);
 	}
@@ -5106,22 +5100,22 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp)
 static int
 driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 {
+#define HEAP_EXTRA 200
 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0)
     Uint need = 0;
     int depth = 0;
     int res;
-    Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL;
     ErlDrvTermData* ptr;
     ErlDrvTermData* ptr_end;
     DECLARE_ESTACK(stack); 
     Eterm mess = NIL;		/* keeps compiler happy */
     Process* rp = NULL;
-    ErlHeapFragment *bp = NULL;
-    ErlOffHeap *ohp;
+    ErtsHeapFactory factory;
     ErtsProcLocks rp_locks = 0;
     struct b2t_states__ b2t;
     int scheduler = 1; /* Silence erroneous warning... */
 
+    factory.mode = FACTORY_CLOSED;
     init_b2t_states(&b2t);
 
     /*
@@ -5285,9 +5279,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 #ifdef DEBUG
 	    b2t.org_ext[b2t.ix] = ext;
 #endif
-	    hsz = erts_binary2term_prepare(&b2t.state[b2t.ix++], ext, size);
+	    hsz = erts_binary2term_prepare(&b2t.state[b2t.ix], ext, size);
 	    if (hsz < 0)
 		ERTS_DDT_FAIL; /* Invalid data */
+	    b2t.state[b2t.ix++].heap_size = hsz;
 	    need += hsz;
 	    ptr += 2;
 	    depth++;
@@ -5297,7 +5292,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
 	    if ((int) ptr[0] < 0) ERTS_DDT_FAIL;
             if (ptr[0] > MAP_SMALL_MAP_LIMIT) {
-                need += hashmap_over_estimated_heap_size(ptr[0]);
+                need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]);
             } else {
                 need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0];
             }
@@ -5336,8 +5331,14 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	goto done;
     }
 
-    hp_start = hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
-    hp_end = hp + need;
+
+    /* We could try to copy directly to destination heap
+     * if we knew there is no big maps.
+
+    erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
+    */
+    erts_factory_message_init(&factory, NULL, NULL,
+			      new_message_buffer(need));
 
     /*
      * Interpret the instructions and build the term.
@@ -5358,13 +5359,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_INT:  /* signed int argument */
 #if HALFWORD_HEAP
-	    mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]);
+	    erts_reserve_heap(&factory, BIG_NEED_SIZE(2));
+	    mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]);
 #else
+	    erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
 	    if (IS_SSMALL((Sint)ptr[0]))
 		mess = make_small((Sint)ptr[0]);
 	    else {
-		mess = small_to_big((Sint)ptr[0], hp);
-		hp += BIG_UINT_HEAP_SIZE;
+		mess = small_to_big((Sint)ptr[0], factory.hp);
+		factory.hp += BIG_UINT_HEAP_SIZE;
 	    }
 #endif
 	    ptr++;
@@ -5372,25 +5375,29 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_UINT:  /* unsigned int argument */
 #if HALFWORD_HEAP
-	    mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]);
+	    erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+	    mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]);
 #else
+	    erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
 	    if (IS_USMALL(0, (Uint)ptr[0]))
 		mess = make_small((Uint)ptr[0]);
 	    else {
-		mess = uint_to_big((Uint)ptr[0], hp);
-		hp += BIG_UINT_HEAP_SIZE;
+		mess = uint_to_big((Uint)ptr[0], factory.hp);
+		factory.hp += BIG_UINT_HEAP_SIZE;
 	    }
 #endif
 	    ptr++;
 	    break;
 
 	case ERL_DRV_INT64: /* pointer to unsigned 64-bit int argument */
-	    mess = erts_bld_sint64(&hp, NULL, *((Sint64 *) ptr[0]));
+	    erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+	    mess = erts_bld_sint64(&factory.hp, NULL, *((Sint64 *) ptr[0]));
 	    ptr++;
 	    break;
 
 	case ERL_DRV_UINT64: /* pointer to unsigned 64-bit int argument */
-	    mess = erts_bld_uint64(&hp, NULL, *((Uint64 *) ptr[0]));
+	    erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
+	    mess = erts_bld_uint64(&factory.hp, NULL, *((Uint64 *) ptr[0]));
 	    ptr++;
 	    break;
 
@@ -5407,8 +5414,8 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
 
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
-		ErlHeapBin* hbp = (ErlHeapBin *) hp;
-		hp += heap_bin_size(size);
+		ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
+								   heap_bin_size(size), HEAP_EXTRA);
 		hbp->thing_word = header_heap_bin(size);
 		hbp->size = size;
 		if (size > 0) {
@@ -5417,18 +5424,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		mess = make_binary(hbp);
 	    }
 	    else {
-		ProcBin* pb = (ProcBin *) hp;
+		ProcBin* pb = (ProcBin *) erts_produce_heap(&factory,
+							    PROC_BIN_SIZE, HEAP_EXTRA);
 		driver_binary_inc_refc(b);  /* caller will free binary */
 		pb->thing_word = HEADER_PROC_BIN;
 		pb->size = size;
-		pb->next = ohp->first;
-		ohp->first = (struct erl_off_heap_header*)pb;
+		pb->next = factory.off_heap->first;
+		factory.off_heap->first = (struct erl_off_heap_header*)pb;
 		pb->val = ErlDrvBinary2Binary(b);
 		pb->bytes = ((byte*) b->orig_bytes) + offset;
 		pb->flags = 0;
 		mess =  make_binary(pb);
-		hp += PROC_BIN_SIZE;
-		OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
+		OH_OVERHEAD(factory.off_heap, pb->size / sizeof(Eterm));
 	    }
 	    ptr += 3;
 	    break;
@@ -5441,8 +5448,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
 
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
-		ErlHeapBin* hbp = (ErlHeapBin *) hp;
-		hp += heap_bin_size(size);
+		ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
+								   heap_bin_size(size),
+								   HEAP_EXTRA);
 		hbp->thing_word = header_heap_bin(size);
 		hbp->size = size;
 		if (size > 0) {
@@ -5457,16 +5465,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		ASSERT(bufp);
 		erts_refc_init(&bp->refc, 1);
 		sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
-		pbp = (ProcBin *) hp;
-		hp += PROC_BIN_SIZE;
+		pbp = (ProcBin *) erts_produce_heap(&factory,
+						    PROC_BIN_SIZE, HEAP_EXTRA);
 		pbp->thing_word = HEADER_PROC_BIN;
 		pbp->size = size;
-		pbp->next = ohp->first;
-		ohp->first = (struct erl_off_heap_header*)pbp;
+		pbp->next = factory.off_heap->first;
+		factory.off_heap->first = (struct erl_off_heap_header*)pbp;
 		pbp->val = bp;
 		pbp->bytes = (byte*) bp->orig_bytes;
 		pbp->flags = 0;
-		OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
+		OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm));
 		mess = make_binary(pbp);
 	    }
 	    ptr += 2;
@@ -5475,13 +5483,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_STRING: /* char*, length */
 	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]);
-	    mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL);
+	    erts_reserve_heap(&factory, 2*ptr[1]);
+	    mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL);
 	    ptr += 2;
 	    break;
 
 	case ERL_DRV_STRING_CONS:  /* char*, length */
 	    mess = ESTACK_POP(stack);
-	    mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], mess);
+	    erts_reserve_heap(&factory, 2*ptr[1]);
+	    mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], mess);
 	    ptr += 2;
 	    break;
 
@@ -5490,11 +5500,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	    mess = ESTACK_POP(stack);
 	    i--;
+	    erts_reserve_heap(&factory, 2*i);
 	    while(i > 0) {
 		Eterm hd = ESTACK_POP(stack);
 
-		mess = CONS(hp, hd, mess);
-		hp += 2;
+		mess = CONS(factory.hp, hd, mess);
+		factory.hp += 2;
 		i--;
 	    }
 	    ptr++;
@@ -5503,13 +5514,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_TUPLE: { /* int */
 	    int size = (int)ptr[0];
-	    Eterm* tp = hp;
+	    Eterm* tp = erts_produce_heap(&factory, size+1, HEAP_EXTRA);
 
 	    *tp = make_arityval(size);
 	    mess = make_tuple(tp);
 
 	    tp += size;   /* point at last element */
-	    hp = tp+1;    /* advance "heap" pointer */
 
 	    while(size--) {
 		*tp-- = ESTACK_POP(stack);
@@ -5525,20 +5535,22 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	case ERL_DRV_FLOAT: { /* double * */
 	    FloatDef f;
+	    Eterm* fp = erts_produce_heap(&factory, FLOAT_SIZE_OBJECT, HEAP_EXTRA);
 
-	    mess = make_float(hp);
+	    mess = make_float(fp);
 	    f.fd = *((double *) ptr[0]);
             if (!erts_isfinite(f.fd))
                 ERTS_DDT_FAIL;
-            PUT_DOUBLE(f, hp);
-	    hp += FLOAT_SIZE_OBJECT;
+	    PUT_DOUBLE(f, fp);
 	    ptr++;
 	    break;
 	}
 
 	case ERL_DRV_EXT2TERM: /* char *ext, int size */
 	    ASSERT(b2t.org_ext[b2t.ix] == (byte *) ptr[0]);
-	    mess = erts_binary2term_create(&b2t.state[b2t.ix++], &hp, ohp);
+
+	    erts_reserve_heap(&factory, b2t.state[b2t.ix].heap_size);
+	    mess = erts_binary2term_create(&b2t.state[b2t.ix++], &factory);
 	    if (mess == THE_NON_VALUE)
 		ERTS_DDT_FAIL;
 	    ptr += 2;
@@ -5548,41 +5560,32 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    int size = (int)ptr[0];
             if (size > MAP_SMALL_MAP_LIMIT) {
                 int ix = 2*size;
-                ErtsHeapFactory factory;
-                Eterm* leafs = hp;
-
-                hp += 2*size;
-                while(ix--) { *--hp = ESTACK_POP(stack); }
+                Eterm* leafs;
 
-                hp += 2*size;
-                factory.p = NULL;
-                factory.hp = hp;
-                /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */
+		erts_produce_heap(&factory, ix, HEAP_EXTRA);
+		leafs = factory.hp;
+                while(ix--) { *--leafs = ESTACK_POP(stack); }
 
                 mess = erts_hashmap_from_array(&factory, leafs, size, 1);
-
                 if (is_non_value(mess))
                     ERTS_DDT_FAIL;
-
-                hp = factory.hp;
             } else {
-                Eterm* tp = hp;
                 Eterm* vp;
                 flatmap_t *mp;
+		Eterm* tp = erts_produce_heap(&factory,
+					      2*size + 1 + MAP_HEADER_FLATMAP_SZ,
+					      HEAP_EXTRA);
 
                 *tp = make_arityval(size);
 
-                hp += 1 + size;
-                mp = (flatmap_t*)hp;
+                mp = (flatmap_t*) (tp + 1 + size);
                 mp->thing_word = MAP_HEADER_FLATMAP;
                 mp->size = size;
                 mp->keys = make_tuple(tp);
                 mess = make_flatmap(mp);
 
-                hp += MAP_HEADER_FLATMAP_SZ + size;
-
                 tp += size;    /* point at last key */
-                vp = hp - 1;   /* point at last value */
+                vp = factory.hp - 1;   /* point at last value */
 
                 while(size--) {
                     *vp-- = ESTACK_POP(stack);
@@ -5605,25 +5608,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
     if (res > 0) {
 	mess = ESTACK_POP(stack);  /* get resulting value */
-	if (bp)
-	    bp = erts_resize_message_buffer(bp, hp - hp_start, &mess, 1);
-	else {
-	    ASSERT(hp);
-	    HRelease(rp, hp_end, hp);	    
-	}
+	erts_factory_close(&factory);
 	/* send message */
-	erts_queue_message(rp, &rp_locks, bp, mess, am_undefined);
+	erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined);
     }
     else {
 	if (b2t.ix > b2t.used)
 	    b2t.used = b2t.ix;
 	for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++)
 	    erts_binary2term_abort(&b2t.state[b2t.ix]);
-	if (bp)
-	    free_message_buffer(bp);
-	else if (hp) {
-	    HRelease(rp, hp_end, hp);
-	}
+	erts_factory_undo(&factory);
     }
     if (rp) {
 	if (rp_locks)
@@ -5635,6 +5629,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
     DESTROY_ESTACK(stack);
     return res;
 #undef ERTS_DDT_FAIL
+#undef HEAP_EXTRA
 }
 
 static ERTS_INLINE int
-- 
cgit v1.2.3


From 7c1ed37b4aa173181c4d45a2108f5eef4c36c41e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 1 Jun 2015 19:43:13 +0200
Subject: erts: Optimize driver_deliver_term

Try write directly to process heap (as before)
if the term is guaranteed not to contain any
big maps that may break the initial size estimation.
---
 erts/emulator/beam/io.c | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 07a82b42d2..fb030d61f9 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5108,12 +5108,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
     ErlDrvTermData* ptr;
     ErlDrvTermData* ptr_end;
     DECLARE_ESTACK(stack); 
-    Eterm mess = NIL;		/* keeps compiler happy */
+    Eterm mess;
     Process* rp = NULL;
     ErtsHeapFactory factory;
     ErtsProcLocks rp_locks = 0;
     struct b2t_states__ b2t;
-    int scheduler = 1; /* Silence erroneous warning... */
+    int scheduler;
+    int is_heap_need_limited = 1;
+
+    ERTS_UNDEF(mess,NIL);
+    ERTS_UNDEF(scheduler,1);
 
     factory.mode = FACTORY_CLOSED;
     init_b2t_states(&b2t);
@@ -5286,6 +5290,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    need += hsz;
 	    ptr += 2;
 	    depth++;
+            if (size > MAP_SMALL_MAP_LIMIT*3) { /* may contain big map */
+                is_heap_need_limited = 0;
+            }
 	    break;
 	}
 	case ERL_DRV_MAP: { /* int */
@@ -5293,6 +5300,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    if ((int) ptr[0] < 0) ERTS_DDT_FAIL;
             if (ptr[0] > MAP_SMALL_MAP_LIMIT) {
                 need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]);
+                is_heap_need_limited = 0;
             } else {
                 need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0];
             }
@@ -5331,14 +5339,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	goto done;
     }
 
-
-    /* We could try to copy directly to destination heap
-     * if we knew there is no big maps.
-
-    erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
-    */
-    erts_factory_message_init(&factory, NULL, NULL,
-			      new_message_buffer(need));
+    /* Try copy directly to destination heap if we know there are no big maps */
+    if (is_heap_need_limited) {
+        ErlOffHeap *ohp;
+        ErlHeapFragment* bp;
+        Eterm* hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
+        erts_factory_message_init(&factory, rp, hp, bp);
+    }
+    else {
+        erts_factory_message_init(&factory, NULL, NULL,
+                                  new_message_buffer(need));
+    }
 
     /*
      * Interpret the instructions and build the term.
-- 
cgit v1.2.3


From f5138184479bd16c0ee0a5e583479378451401c8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 10 Jun 2015 20:59:05 +0200
Subject: erts: Add erts_factory_trim_and_close

---
 erts/emulator/beam/erl_message.c | 25 +++++++++++++++++++++----
 erts/emulator/beam/erl_message.h |  1 +
 erts/emulator/beam/io.c          |  6 +-----
 3 files changed, 23 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 13e084ce10..f806d2c498 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -93,9 +93,6 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 #endif
     ErlHeapFragment* nbp;
 
-    /* ToDo: Make use of 'used_size' to avoid realloc
-	when shrinking just a few words */
-
 #ifdef DEBUG
     {
 	Uint off_sz = size < bp->used_size ? size : bp->used_size;
@@ -110,8 +107,10 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
     }
 #endif
 
-    if (size == bp->used_size)
+    if (size >= (bp->used_size - bp->used_size / 16)) {
+        bp->used_size = size;
 	return bp;
+    }
 
 #ifdef HARD_DEBUG
     dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size);
@@ -1288,6 +1287,24 @@ void erts_factory_close(ErtsHeapFactory* factory)
     factory->mode = FACTORY_CLOSED;
 }
 
+void erts_factory_trim_and_close(ErtsHeapFactory* factory,
+				 Eterm *brefs, Uint brefs_size)
+{
+    if (factory->mode == FACTORY_HEAP_FRAGS) {
+        ErlHeapFragment* bp = factory->heap_frags;
+        if (bp->next == NULL) {
+            Uint used_sz = factory->hp - bp->mem;
+            ASSERT(used_sz <= bp->alloc_size);
+            factory->heap_frags = erts_resize_message_buffer(bp, used_sz,
+                                                             brefs, brefs_size);
+            factory->mode = FACTORY_CLOSED;
+            return;
+        }
+        /*else we don't trim multi fragmented messages for now */
+    }
+    erts_factory_close(factory);
+}
+
 void erts_factory_undo(ErtsHeapFactory* factory)
 {
     ErlHeapFragment* bp;
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 39946f2a14..705ba5e506 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -79,6 +79,7 @@ void erts_factory_dummy_init(ErtsHeapFactory*);
 Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
 Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need);
 void erts_factory_close(ErtsHeapFactory*);
+void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size);
 void erts_factory_undo(ErtsHeapFactory*);
 
 #ifdef CHECK_FOR_HOLES
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index fb030d61f9..23f208c2eb 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1407,12 +1407,8 @@ queue_port_sched_op_reply(Process *rp,
     hp += REF_THING_SIZE;
 
     msg = TUPLE2(hp, ref, msg);
-    hp += 3;
 
-    /* SVERK: We used to call erts_resize_message_buffer here
-     * to maybe realloc message.
-     */
-    erts_factory_close(factory);
+    erts_factory_trim_and_close(factory, &msg, 1);
 
     erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL);
 }
-- 
cgit v1.2.3


From bfd575ddf985408494d4d3d4933eda2c9ee18326 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 19 May 2015 22:48:21 +0200
Subject: ETS busy wait option

Conflicts:
	erts/emulator/beam/erl_init.c
	erts/etc/common/erlexec.c
---
 erts/emulator/beam/erl_db.c      | 43 +++++++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_db.h      | 13 +++++++++++-
 erts/emulator/beam/erl_db_hash.c |  2 ++
 erts/emulator/beam/erl_init.c    | 41 +++++++++++++++++++++++++++++++++-----
 4 files changed, 92 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 2e2cb98354..844272c699 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -279,6 +279,8 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     if (use_frequent_read_lock)
 	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    if (erts_ets_rwmtx_spin_count >= 0)
+	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
 #endif
 #ifdef ERTS_SMP
     erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt,
@@ -2856,10 +2858,11 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
 ** External interface (NOT BIF's)
 */
 
+int erts_ets_rwmtx_spin_count = -1;
 
 /* Init the db */
 
-void init_db(void)
+void init_db(ErtsDbSpinCount db_spin_count)
 {
     DbTable init_tb;
     int i;
@@ -2868,10 +2871,48 @@ void init_db(void)
     size_t size;
 
 #ifdef ERTS_SMP
+    int max_spin_count = (1 << 15) - 1; /* internal limit */
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
+    switch (db_spin_count) {
+    case ERTS_DB_SPNCNT_NONE:
+	erts_ets_rwmtx_spin_count = 0;
+	break;
+    case ERTS_DB_SPNCNT_VERY_LOW:
+	erts_ets_rwmtx_spin_count = 100;
+	break;
+    case ERTS_DB_SPNCNT_LOW:
+	erts_ets_rwmtx_spin_count = 200;
+	erts_ets_rwmtx_spin_count += erts_no_schedulers * 50;
+	if (erts_ets_rwmtx_spin_count > 1000)
+	    erts_ets_rwmtx_spin_count = 1000;
+	break;
+    case ERTS_DB_SPNCNT_HIGH:
+	erts_ets_rwmtx_spin_count = 2000;
+	erts_ets_rwmtx_spin_count += erts_no_schedulers * 100;
+	if (erts_ets_rwmtx_spin_count > 15000)
+	    erts_ets_rwmtx_spin_count = 15000;
+	break;
+    case ERTS_DB_SPNCNT_VERY_HIGH:
+	erts_ets_rwmtx_spin_count = 15000;
+	erts_ets_rwmtx_spin_count += erts_no_schedulers * 500;
+	if (erts_ets_rwmtx_spin_count > max_spin_count)
+	    erts_ets_rwmtx_spin_count = max_spin_count;
+	break;
+    case ERTS_DB_SPNCNT_EXTREMELY_HIGH:
+	erts_ets_rwmtx_spin_count = max_spin_count;
+	break;
+    case ERTS_DB_SPNCNT_NORMAL:
+    default:
+	erts_ets_rwmtx_spin_count = -1;
+	break;
+    }
+
+    if (erts_ets_rwmtx_spin_count >= 0)
+	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
+
     meta_main_tab_locks =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES,
 					   sizeof(erts_meta_main_tab_lock_t)
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 5b4681fc90..5039b71108 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -61,7 +61,17 @@ union db_table {
 				"ERL_MAX_ETS_TABLES" */
 #define ERL_MAX_ETS_TABLES_ENV "ERL_MAX_ETS_TABLES"
 
-void init_db(void);
+typedef enum {
+    ERTS_DB_SPNCNT_NONE,
+    ERTS_DB_SPNCNT_VERY_LOW,
+    ERTS_DB_SPNCNT_LOW,
+    ERTS_DB_SPNCNT_NORMAL,
+    ERTS_DB_SPNCNT_HIGH,
+    ERTS_DB_SPNCNT_VERY_HIGH,
+    ERTS_DB_SPNCNT_EXTREMELY_HIGH
+} ErtsDbSpinCount;
+
+void init_db(ErtsDbSpinCount);
 int erts_db_process_exiting(Process *, ErtsProcLocks);
 void db_info(int, void *, int);
 void erts_db_foreach_table(void (*)(DbTable *, void *), void *);
@@ -69,6 +79,7 @@ void erts_db_foreach_offheap(DbTable *,
 			     void (*func)(ErlOffHeap *, void *),
 			     void *);
 
+extern int erts_ets_rwmtx_spin_count;
 extern int user_requested_db_max_tabs; /* set in erl_init */
 extern int erts_ets_realloc_always_moves;  /* set in erl_init */
 extern int erts_ets_always_compress;  /* set in erl_init */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 383ee7c430..f6ea57e513 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -670,6 +670,8 @@ int db_create_hash(Process *p, DbTable *tbl)
 	int i;
 	if (tb->common.type & DB_FREQ_READ)
 	    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+	if (erts_ets_rwmtx_spin_count >= 0)
+	    rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
 	tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ 
 							      (DbTable *) tb,
 							      sizeof(DbTableHashFineLocks));	    	    
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 574a49dc7a..c676682463 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -142,7 +142,8 @@ static void erl_init(int ncpu,
 		     int legacy_port_tab,
 		     int time_correction,
 		     ErtsTimeWarpMode time_warp_mode,
-		     int node_tab_delete_delay);
+		     int node_tab_delete_delay,
+		     ErtsDbSpinCount db_spin_count);
 
 static erts_atomic_t exiting;
 
@@ -316,7 +317,8 @@ erts_short_init(void)
 	     0,
 	     time_correction,
 	     time_warp_mode,
-	     ERTS_NODE_TAB_DELAY_GC_DEFAULT);
+	     ERTS_NODE_TAB_DELAY_GC_DEFAULT,
+	     ERTS_DB_SPNCNT_NORMAL);
     erts_initialized = 1;
 }
 
@@ -329,7 +331,8 @@ erl_init(int ncpu,
 	 int legacy_port_tab,
 	 int time_correction,
 	 ErtsTimeWarpMode time_warp_mode,
-	 int node_tab_delete_delay)
+	 int node_tab_delete_delay,
+	 ErtsDbSpinCount db_spin_count)
 {
     init_benchmarking();
 
@@ -369,7 +372,7 @@ erl_init(int ncpu,
     erts_ptab_init(); /* Must be after init_emulator() */
     erts_init_binary(); /* Must be after init_emulator() */
     erts_bp_init();
-    init_db(); /* Must be after init_emulator */
+    init_db(db_spin_count); /* Must be after init_emulator */
     erts_init_node_tables(node_tab_delete_delay);
     init_dist();
     erl_drv_thr_init();
@@ -635,6 +638,10 @@ void erts_usage(void)
     erts_fprintf(stderr, "-zdntgc time   set delayed node table gc in seconds\n");
     erts_fprintf(stderr, "               valid values are infinity or intergers in the range [0-%d]\n",
 		 ERTS_NODE_TAB_DELAY_GC_MAX);
+#if 0
+    erts_fprintf(stderr, "-zebwt  val    set ets busy wait threshold, valid values are:\n");
+    erts_fprintf(stderr, "               none|very_short|short|medium|long|very_long|extremely_long\n");
+#endif
     erts_fprintf(stderr, "\n");
     erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n");
     erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n");
@@ -1226,6 +1233,7 @@ erl_start(int argc, char **argv)
     int time_correction;
     ErtsTimeWarpMode time_warp_mode;
     int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT;
+    ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL;
 
     set_default_time_adj(&time_correction,
 			 &time_warp_mode);
@@ -2040,6 +2048,28 @@ erl_start(int argc, char **argv)
 		    }
 		}
 		node_tab_delete_delay = (int) secs;
+	    }
+	    else if (has_prefix("ebwt", sub_param)) {
+		arg = get_arg(sub_param+4, argv[i+1], &i);
+		if (sys_strcmp(arg, "none") == 0)
+		    db_spin_count = ERTS_DB_SPNCNT_NONE;
+		else if (sys_strcmp(arg, "very_short") == 0)
+		    db_spin_count = ERTS_DB_SPNCNT_VERY_LOW;
+		else if (sys_strcmp(arg, "short") == 0)
+		    db_spin_count = ERTS_DB_SPNCNT_LOW;
+		else if (sys_strcmp(arg, "medium") == 0)
+		    db_spin_count = ERTS_DB_SPNCNT_NORMAL;
+		else if (sys_strcmp(arg, "long") == 0)
+		    db_spin_count = ERTS_DB_SPNCNT_HIGH;
+		else if (sys_strcmp(arg, "very_long") == 0)
+		    db_spin_count = ERTS_DB_SPNCNT_VERY_HIGH;
+		else if (sys_strcmp(arg, "extremely_long") == 0)
+		    db_spin_count = ERTS_DB_SPNCNT_EXTREMELY_HIGH;
+		else {
+		    erts_fprintf(stderr,
+				 "Invalid ets busy wait threshold: %s\n", arg);
+		    erts_usage();
+		}
 	    } else {
 		erts_fprintf(stderr, "bad -z option %s\n", argv[i]);
 		erts_usage();
@@ -2114,7 +2144,8 @@ erl_start(int argc, char **argv)
 	     legacy_port_tab,
 	     time_correction,
 	     time_warp_mode,
-	     node_tab_delete_delay);
+	     node_tab_delete_delay,
+	     db_spin_count);
 
     load_preloaded();
     erts_end_staging_code_ix();
-- 
cgit v1.2.3


From d18dd1ba7f7c874b1cbc2806822d134ea1546781 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 15 Jun 2015 20:13:15 +0200
Subject: Unbreak global inlining

---
 erts/emulator/beam/sys.h | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index cd53069872..78eb0bfe2b 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -20,6 +20,22 @@
 #ifndef __SYS_H__
 #define __SYS_H__
 
+#ifdef ERTS_INLINE
+#  ifndef ERTS_CAN_INLINE
+#    define ERTS_CAN_INLINE 1
+#  endif
+#else
+#  if defined(__GNUC__)
+#    define ERTS_CAN_INLINE 1
+#    define ERTS_INLINE __inline__
+#  elif defined(__WIN32__)
+#    define ERTS_CAN_INLINE 1
+#    define ERTS_INLINE __inline
+#  else
+#    define ERTS_CAN_INLINE 0
+#    define ERTS_INLINE
+#  endif
+#endif
 
 #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
 #  undef ERTS_CAN_INLINE
@@ -94,23 +110,6 @@ typedef int ErtsSysFdType;
 typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #endif
 
-#ifdef ERTS_INLINE
-#  ifndef ERTS_CAN_INLINE
-#    define ERTS_CAN_INLINE 1
-#  endif
-#else
-#  if defined(__GNUC__)
-#    define ERTS_CAN_INLINE 1
-#    define ERTS_INLINE __inline__
-#  elif defined(__WIN32__)
-#    define ERTS_CAN_INLINE 1
-#    define ERTS_INLINE __inline
-#  else
-#    define ERTS_CAN_INLINE 0
-#    define ERTS_INLINE
-#  endif
-#endif
-
 #if !defined(__GNUC__)
 #  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
 #elif !defined(__GNUC_MINOR__)
-- 
cgit v1.2.3


From d80f10c1586718fc0097fa156fd6e9f8ec0f6597 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 16 Jun 2015 10:56:57 +0200
Subject: erts: Refactor LCNT

---
 erts/emulator/beam/erl_bif_info.c     |  80 +++-----
 erts/emulator/beam/erl_lock_count.c   | 348 +++++++++++++++++-----------------
 erts/emulator/beam/erl_lock_count.h   |  42 ++--
 erts/emulator/beam/erl_process_lock.c | 186 +++++++-----------
 erts/emulator/beam/io.c               |  54 +++---
 5 files changed, 321 insertions(+), 389 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 397c68e199..8a675302b2 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4296,59 +4296,41 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
 
 	BIF_RET(am_ok);
     } else if (is_tuple(BIF_ARG_1)) {
-	Eterm* tp = tuple_val(BIF_ARG_1);
+        Eterm* ptr = tuple_val(BIF_ARG_1);
+
+        if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) {
+            int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0;
+            if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) {
+                lock_opt = ERTS_LCNT_OPT_COPYSAVE;
+            } else if (ERTS_IS_ATOM_STR("process_locks", ptr[1])) {
+                lock_opt = ERTS_LCNT_OPT_PROCLOCK;
+            } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) {
+                lock_opt = ERTS_LCNT_OPT_PORTLOCK;
+            } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) {
+                lock_opt = ERTS_LCNT_OPT_SUSPEND;
+            } else if (ERTS_IS_ATOM_STR("location", ptr[1])) {
+                lock_opt = ERTS_LCNT_OPT_LOCATION;
+            } else {
+                BIF_ERROR(BIF_P, BADARG);
+            }
 
-	switch (arityval(tp[0])) {
-	    case 2: {
-		int opt = 0;
-		int val = 0;
-		if (ERTS_IS_ATOM_STR("copy_save", tp[1])) {
-		    opt = ERTS_LCNT_OPT_COPYSAVE;
-		} else if (ERTS_IS_ATOM_STR("process_locks", tp[1])) {
-		    opt = ERTS_LCNT_OPT_PROCLOCK;
-		} else if (ERTS_IS_ATOM_STR("port_locks", tp[1])) {
-		    opt = ERTS_LCNT_OPT_PORTLOCK;
-		} else if (ERTS_IS_ATOM_STR("suspend", tp[1])) {
-		    opt = ERTS_LCNT_OPT_SUSPEND;
-		} else if (ERTS_IS_ATOM_STR("location", tp[1])) {
-		    opt = ERTS_LCNT_OPT_LOCATION;
-		} else {
-		    BIF_ERROR(BIF_P, BADARG);
-		}
-		if (tp[2] == am_true) {
-		    val = 1;
-		} else if (tp[2] == am_false) {
-		    val = 0;
-		} else {
-		    BIF_ERROR(BIF_P, BADARG);
-		}
+            erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+            erts_smp_thr_progress_block();
 
-		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		erts_smp_thr_progress_block();
-		
-		if (val) {
-		    res = erts_lcnt_set_rt_opt(opt) ? am_true : am_false;
-		} else {
-		    res = erts_lcnt_clear_rt_opt(opt) ? am_true : am_false;
-		}
+            if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false;
+            else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false;
+            
 #ifdef ERTS_SMP
-		if (res != tp[2]) {
-		    if (opt == ERTS_LCNT_OPT_PORTLOCK) {
-			erts_lcnt_enable_io_lock_count(val);
-		    } else if (opt == ERTS_LCNT_OPT_PROCLOCK) {
-			erts_lcnt_enable_proc_lock_count(val);
-		    }
-		}
+            if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) {
+                erts_lcnt_enable_io_lock_count(enable);
+            } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) {
+                erts_lcnt_enable_proc_lock_count(enable);
+            }
 #endif
-		erts_smp_thr_progress_unblock();
-		erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		BIF_RET(res);
-		break;
-	    }
-     
-	    default:
-	    break;
-	}
+            erts_smp_thr_progress_unblock();
+            erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+            BIF_RET(res);
+        }
     } 
 
 #endif 
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index c6d8f4df95..c4956d8569 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -20,7 +20,7 @@
 /*
  * Description:	Statistics for locks.
  *
- * Author:	Björn-Egil Dahlberg
+ * Author:	Björn-Egil Dahlberg
  * Date:	2008-07-03
  */
 
@@ -49,7 +49,7 @@ const char *str_undefined = "undefined";
 
 static ethr_tsd_key lcnt_thr_data_key;
 static int lcnt_n_thr;
-static erts_lcnt_thread_data_t *lcnt_thread_data[4096]; 
+static erts_lcnt_thread_data_t *lcnt_thread_data[2048];
 
 /* local functions */
 
@@ -83,12 +83,12 @@ static ERTS_INLINE int lcnt_log2(Uint64 v) {
 
 static char* lcnt_lock_type(Uint16 flag) {
     switch(flag & ERTS_LCNT_LT_ALL) {
-    case ERTS_LCNT_LT_SPINLOCK:	  return "spinlock";
-    case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock";
-    case ERTS_LCNT_LT_MUTEX:      return "mutex";
-    case ERTS_LCNT_LT_RWMUTEX:	  return "rw_mutex";
-    case ERTS_LCNT_LT_PROCLOCK:	  return "proclock";
-    default:                      return "";
+        case ERTS_LCNT_LT_SPINLOCK:   return "spinlock";
+        case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock";
+        case ERTS_LCNT_LT_MUTEX:      return "mutex";
+        case ERTS_LCNT_LT_RWMUTEX:    return "rw_mutex";
+        case ERTS_LCNT_LT_PROCLOCK:   return "proclock";
+        default:                      return "";
     }
 }
 
@@ -116,15 +116,15 @@ static void lcnt_time(erts_lcnt_time_t *time) {
 static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) {
     long ds;
     long dns;
-    
+
     ds  = t1->s  - t0->s;
     dns = t1->ns - t0->ns;
-	
+
     /* the difference should not be able to get bigger than 1 sec in ns*/
-    
+
     if (dns < 0) {
-	ds  -= 1;
-	dns += 1000000000LL;
+        ds  -= 1;
+        dns += 1000000000LL;
     }
 
     ASSERT(ds >= 0);
@@ -145,7 +145,7 @@ static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) {
 
 static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) {
     erts_lcnt_thread_data_t *eltd;
-   
+
     eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t));
     if (!eltd) {
         ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
@@ -164,7 +164,6 @@ static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) {
     return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key);
 }
 
-
 /* debug */
 
 #if 0
@@ -183,15 +182,15 @@ static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) {
     type = lcnt_lock_type(lock->flag);
     r_state = ethr_atomic_read(&lock->r_state);
     w_state = ethr_atomic_read(&lock->w_state);
-    
+
     if (lock->flag & flag) {
         erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n",
-		action,
-		lock->name,
-		r_state,
-		w_state,
-		type,
-		lock->id);
+                action,
+                lock->name,
+                r_state,
+                w_state,
+                type,
+                lock->id);
     }
 }
 #endif
@@ -201,18 +200,18 @@ static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char
     erts_lcnt_lock_stats_t *stats = NULL;
 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
-	for (i = 0; i < lock->n_stats; i++) {
-	    if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) {
-		return &(lock->stats[i]);
-	    }
-	}
-	if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
-	    stats = &lock->stats[lock->n_stats];
-	    lock->n_stats++;
-	    stats->file = file;
-	    stats->line = line;
-	    return stats;
-	}
+        for (i = 0; i < lock->n_stats; i++) {
+            if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) {
+                return &(lock->stats[i]);
+            }
+        }
+        if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
+            stats = &lock->stats[lock->n_stats];
+            lock->n_stats++;
+            stats->file = file;
+            stats->line = line;
+            return stats;
+        }
     }
     return &lock->stats[0];
 }
@@ -222,53 +221,48 @@ static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *tim
     unsigned long r;
 
     if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) {
-	idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1;
+        idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1;
     } else {
-	r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT;
-	if (r) idx = lcnt_log2(r);
-	else idx = 0;
+        r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT;
+        if (r) idx = lcnt_log2(r);
+        else idx = 0;
     }
     hist->ns[idx]++;
 }
 
 static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict,
-	                      erts_lcnt_time_t *time_wait) {
-    
+        erts_lcnt_time_t *time_wait) {
+
     ethr_atomic_inc(&stats->tries);
 
     if (lock_in_conflict)
-	ethr_atomic_inc(&stats->colls);
+        ethr_atomic_inc(&stats->colls);
 
     if (time_wait) {
-	lcnt_time_add(&(stats->timer), time_wait);
-	stats->timer_n++;
-	lcnt_update_stats_hist(&stats->hist,time_wait);
+        lcnt_time_add(&(stats->timer), time_wait);
+        stats->timer_n++;
+        lcnt_update_stats_hist(&stats->hist,time_wait);
     }
 }
 
-/* 
- * interface 
- */
+/* interface */
 
 void erts_lcnt_init() {
     erts_lcnt_thread_data_t *eltd = NULL;
-    
+
     /* init lock */
     if (ethr_mutex_init(&lcnt_data_lock) != 0) abort();
 
     /* init tsd */    
     lcnt_n_thr = 0;
-
     ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data");
 
     lcnt_lock();
 
-    erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION;
-    
+    erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK;
     eltd = lcnt_thread_data_alloc();
-
     ethr_tsd_set(lcnt_thr_data_key, eltd);
-    
+
     /* init lcnt structure */
     erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t));
     if (!erts_lcnt_data) {
@@ -293,7 +287,7 @@ void erts_lcnt_late_init() {
 
 erts_lcnt_lock_list_t *erts_lcnt_list_init(void) {
     erts_lcnt_lock_list_t *list;
-    
+
     list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t));
     if (!list) {
         ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
@@ -307,14 +301,14 @@ erts_lcnt_lock_list_t *erts_lcnt_list_init(void) {
 /* only do this on the list with the deleted locks! */
 void erts_lcnt_list_clear(erts_lcnt_lock_list_t *list) {
     erts_lcnt_lock_t *lock = NULL,
-		     *next = NULL;
+                     *next = NULL;
 
     lock = list->head;
-    
+
     while(lock != NULL) {
-	next = lock->next;
-	free(lock);
-	lock = next;
+        next = lock->next;
+        free(lock);
+        lock = next;
     }
 
     list->head = NULL;
@@ -327,26 +321,25 @@ void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock)
 
     tail = list->tail;
     if (tail) {
-	tail->next = lock;
-	lock->prev = tail;
+        tail->next = lock;
+        lock->prev = tail;
     } else {
-	list->head = lock;
-	lock->prev = NULL;
-	ASSERT(!lock->next);
+        list->head = lock;
+        lock->prev = NULL;
+        ASSERT(!lock->next);
     }
     lock->next = NULL;
     list->tail = lock;
-    
+
     list->n++;
 }
 
 void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) {
-
     if (lock->next) lock->next->prev = lock->prev;
     if (lock->prev) lock->prev->next = lock->next;
     if (list->head == lock) list->head = lock->next;
     if (list->tail == lock) list->tail = lock->prev;
-    
+
     lock->prev = NULL;
     lock->next = NULL;
     list->n--;
@@ -364,12 +357,9 @@ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) {
 
 void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { 
     int i;
-    if (!name) {
-	lock->flag = 0;
-	return;
-    }
+    if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; }
     lcnt_lock();
-    
+
     lock->next = NULL;
     lock->prev = NULL;
     lock->flag = flag;
@@ -378,46 +368,55 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter
 
     ethr_atomic_init(&lock->r_state, 0);
     ethr_atomic_init(&lock->w_state, 0);
-    
 #ifdef DEBUG
     ethr_atomic_init(&lock->flowstate, 0);
 #endif
-    
+
     lock->n_stats = 1;
 
     for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
-	lcnt_clear_stats(&lock->stats[i]);
+        lcnt_clear_stats(&lock->stats[i]);
     }
 
     erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock);
     lcnt_unlock();
 }
-
+/* init empty, instead of zero struct */
+void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) {
+    lock->next = NULL;
+    lock->prev = NULL;
+    lock->flag = 0;
+    lock->name = NULL;
+    lock->id   = NIL;
+    ethr_atomic_init(&lock->r_state, 0);
+    ethr_atomic_init(&lock->w_state, 0);
+#ifdef DEBUG
+    ethr_atomic_init(&lock->flowstate, 0);
+#endif
+    lock->n_stats = 0;
+    sys_memzero(lock->stats, sizeof(lock->stats));
+}
+/* destroy lock */
 void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
-    erts_lcnt_lock_t *deleted_lock;
-
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
-
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
     lcnt_lock();
 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) {
-	/* copy structure and insert the copy */
-
-	deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t));
+        erts_lcnt_lock_t *deleted_lock;
+        /* copy structure and insert the copy */
+        deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t));
         if (!deleted_lock) {
             ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
         }
-	memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t));
-
-	deleted_lock->next = NULL;
-	deleted_lock->prev = NULL;
-
-	erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock);
+        memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t));
+        deleted_lock->next = NULL;
+        deleted_lock->prev = NULL;
+        erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock);
     }
     /* delete original */
     erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock);
-    lock->flag = 0;
-    
+    ERTS_LCNT_CLEAR_FLAG(lock);
+
     lcnt_unlock();
 }
 
@@ -426,16 +425,15 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
 void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
     erts_aint_t r_state = 0, w_state = 0;
     erts_lcnt_thread_data_t *eltd;
-    
+
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 
     eltd = lcnt_get_thread_data();
-
     ASSERT(eltd);
-    
+
     w_state = ethr_atomic_read(&lock->w_state);
-    
+
     if (option & ERTS_LCNT_LO_WRITE) {
         r_state = ethr_atomic_read(&lock->r_state);
         ethr_atomic_inc( &lock->w_state);
@@ -443,48 +441,47 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
     if (option & ERTS_LCNT_LO_READ) {
         ethr_atomic_inc( &lock->r_state);
     }
-    
+
     /* we cannot acquire w_lock if either w or r are taken */
     /* we cannot acquire r_lock if w_lock is taken */	
-    
+
     if ((w_state > 0) || (r_state > 0)) {
-	eltd->lock_in_conflict = 1;
-	if (eltd->timer_set == 0) {
-	    lcnt_time(&eltd->timer);
-	}
-	eltd->timer_set++;
+        eltd->lock_in_conflict = 1;
+        if (eltd->timer_set == 0) {
+            lcnt_time(&eltd->timer);
+        }
+        eltd->timer_set++;
     } else {
-	eltd->lock_in_conflict = 0;
+        eltd->lock_in_conflict = 0;
     }
 }
 
 void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
     erts_aint_t w_state;
     erts_lcnt_thread_data_t *eltd;
-    
+
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 
     w_state = ethr_atomic_read(&lock->w_state);
     ethr_atomic_inc(&lock->w_state);
-
     eltd = lcnt_get_thread_data();
 
     ASSERT(eltd);
 
     if (w_state > 0) {
-	eltd->lock_in_conflict = 1;
-	/* only set the timer if nobody else has it
-	 * This should only happen when proc_locks aquires several locks
-	 * 'atomicly'. All other locks will block the thread if w_state > 0
-	 * i.e. locked.
-	 */
-	if (eltd->timer_set == 0) {
-	    lcnt_time(&eltd->timer);
-	}
-	eltd->timer_set++;
+        eltd->lock_in_conflict = 1;
+        /* only set the timer if nobody else has it
+         * This should only happen when proc_locks aquires several locks
+         * 'atomicly'. All other locks will block the thread if w_state > 0
+         * i.e. locked.
+         */
+        if (eltd->timer_set == 0) {
+            lcnt_time(&eltd->timer);
+        }
+        eltd->timer_set++;
     } else {
-	eltd->lock_in_conflict = 0;
+        eltd->lock_in_conflict = 0;
     }
 }
 
@@ -493,16 +490,19 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
 void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) {
     /* should check if this thread was "waiting" */
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 
     ethr_atomic_dec(&lock->w_state);
 }
 
-/* erts_lcnt_lock_post
- * used when we get a lock (i.e. directly after a lock operation)
+/*
+ * erts_lcnt_lock_post
+ *
+ * Used when we get a lock (i.e. directly after a lock operation)
  * if the timer was set then we had to wait for the lock
  * lock_post will calculate the wait time.
  */
+
 void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) {
     erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0);
 }
@@ -517,31 +517,31 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
 #endif
 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
-    
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
+
 #ifdef DEBUG
     if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
-	flowstate = ethr_atomic_read(&lock->flowstate);
-	ASSERT(flowstate == 0);
-	ethr_atomic_inc(&lock->flowstate);
+        flowstate = ethr_atomic_read(&lock->flowstate);
+        ASSERT(flowstate == 0);
+        ethr_atomic_inc(&lock->flowstate);
     }
 #endif
-    
+
     eltd = lcnt_get_thread_data();
-    
+
     ASSERT(eltd);
 
     /* if lock was in conflict, time it */
     stats = lcnt_get_lock_stats(lock, file, line);
     if (eltd->timer_set) {
-	lcnt_time(&timer);
-	
-	lcnt_time_diff(&time_wait, &timer, &(eltd->timer));
-	lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait);
-	eltd->timer_set--;
-	ASSERT(eltd->timer_set >= 0);
+        lcnt_time(&timer);
+
+        lcnt_time_diff(&time_wait, &timer, &(eltd->timer));
+        lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait);
+        eltd->timer_set--;
+        ASSERT(eltd->timer_set >= 0);
     } else {
-	lcnt_update_stats(stats, eltd->lock_in_conflict, NULL);
+        lcnt_update_stats(stats, eltd->lock_in_conflict, NULL);
     }
 
 }
@@ -550,27 +550,28 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
 
 void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
     if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state);
     if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state);
 }
 
 void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
-#ifdef DEBUG
-    erts_aint_t w_state;  
-    erts_aint_t flowstate;
-#endif
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 #ifdef DEBUG
-    /* flowstate */
-    flowstate = ethr_atomic_read(&lock->flowstate);
-    ASSERT(flowstate == 1);
-    ethr_atomic_dec(&lock->flowstate);
-    
-    /* write state */
-    w_state = ethr_atomic_read(&lock->w_state);
-    ASSERT(w_state > 0);
+    {
+        erts_aint_t w_state;  
+        erts_aint_t flowstate;
+
+        /* flowstate */
+        flowstate = ethr_atomic_read(&lock->flowstate);
+        ASSERT(flowstate == 1);
+        ethr_atomic_dec(&lock->flowstate);
+
+        /* write state */
+        w_state = ethr_atomic_read(&lock->w_state);
+        ASSERT(w_state > 0);
+    }
 #endif
     ethr_atomic_dec(&lock->w_state);
 }
@@ -579,35 +580,34 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
 
 void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) {
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
     /* Determine lock_state via res instead of state */
     if (res != EBUSY) {
-	if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state);
-	if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state);
-	lcnt_update_stats(&(lock->stats[0]), 0, NULL);
+        if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state);
+        if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state);
+        lcnt_update_stats(&(lock->stats[0]), 0, NULL);
     } else {
         ethr_atomic_inc(&lock->stats[0].tries);
         ethr_atomic_inc(&lock->stats[0].colls);
     }
 }
 
-   
+
 void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
     /* Determine lock_state via res instead of state */
-#ifdef DEBUG
-    erts_aint_t flowstate;
-#endif 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
     if (res != EBUSY) {
-	
 #ifdef DEBUG
-	flowstate = ethr_atomic_read(&lock->flowstate);
-	ASSERT(flowstate == 0);
-    	ethr_atomic_inc( &lock->flowstate);
+        {
+            erts_aint_t flowstate;
+            flowstate = ethr_atomic_read(&lock->flowstate);
+            ASSERT(flowstate == 0);
+            ethr_atomic_inc( &lock->flowstate);
+        }
 #endif
-	ethr_atomic_inc(&lock->w_state);
-	lcnt_update_stats(&(lock->stats[0]), 0, NULL);
+        ethr_atomic_inc(&lock->w_state);
+        lcnt_update_stats(&(lock->stats[0]), 0, NULL);
     } else {
         ethr_atomic_inc(&lock->stats[0].tries);
         ethr_atomic_inc(&lock->stats[0].colls);
@@ -662,13 +662,13 @@ void erts_lcnt_clear_counters(void) {
     lcnt_lock();
 
     list = erts_lcnt_data->current_locks;
-    
+
     for (lock = list->head; lock != NULL; lock = lock->next) {
-    	for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
-	    stats = &lock->stats[i];
-	    lcnt_clear_stats(stats);
-    	}
-	lock->n_stats = 1;
+        for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
+            stats = &lock->stats[i];
+            lcnt_clear_stats(stats);
+        }
+        lock->n_stats = 1;
     }
 
     /* empty deleted locks in lock list */
@@ -681,14 +681,14 @@ void erts_lcnt_clear_counters(void) {
 
 erts_lcnt_data_t *erts_lcnt_get_data(void) {
     erts_lcnt_time_t timer_stop;
-    
+
     lcnt_lock();
-    
+
     lcnt_time(&timer_stop);
     lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start);
-    
+
     lcnt_unlock();
-    
+
     return erts_lcnt_data;
 }
 
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 09fadd7e9e..051f55271d 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -20,7 +20,7 @@
 /*
  * Description:	Statistics for locks.
  *
- * Author:	Björn-Egil Dahlberg
+ * Author:	Björn-Egil Dahlberg
  * Date:	2008-07-03
  * Abstract:
  * 	Locks statistics internal representation.
@@ -95,30 +95,35 @@
 #define ERTS_LCNT_LO_WRITE      (((Uint16) 1) << 7)
 
 #define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ  \
-				| ERTS_LCNT_LO_WRITE )
+                                | ERTS_LCNT_LO_WRITE )
 
 #define ERTS_LCNT_LT_ALL        ( ERTS_LCNT_LT_SPINLOCK   \
-				| ERTS_LCNT_LT_RWSPINLOCK \
-				| ERTS_LCNT_LT_MUTEX      \
-				| ERTS_LCNT_LT_RWMUTEX    \
-				| ERTS_LCNT_LT_PROCLOCK   )
+                                | ERTS_LCNT_LT_RWSPINLOCK \
+                                | ERTS_LCNT_LT_MUTEX      \
+                                | ERTS_LCNT_LT_RWMUTEX    \
+                                | ERTS_LCNT_LT_PROCLOCK   )
+
+#define ERTS_LCNT_LOCK_TYPE(lock)       ((lock)->flag & ERTS_LCNT_LT_ALL)
+#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL))
+#define ERTS_LCNT_CLEAR_FLAG(lock)      ((lock)->flag = 0)
+
 /* runtime options */
 
-#define ERTS_LCNT_OPT_SUSPEND   (((Uint16) 1) << 0)
-#define ERTS_LCNT_OPT_LOCATION  (((Uint16) 1) << 1)
-#define ERTS_LCNT_OPT_PROCLOCK  (((Uint16) 1) << 2)
-#define ERTS_LCNT_OPT_COPYSAVE  (((Uint16) 1) << 3)
-#define ERTS_LCNT_OPT_PORTLOCK  (((Uint16) 1) << 4)
+#define ERTS_LCNT_OPT_SUSPEND  (((Uint16) 1) << 0)
+#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1)
+#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2)
+#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3)
+#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 4)
 
 typedef struct {
     unsigned long s;
     unsigned long ns;
 } erts_lcnt_time_t;
-    
+
 extern erts_lcnt_time_t timer_start;
 
 typedef struct {
-   Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */
+    Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */
 } erts_lcnt_hist_t;
 
 typedef struct erts_lcnt_lock_stats_s {
@@ -129,10 +134,10 @@ typedef struct erts_lcnt_lock_stats_s {
 
     char         *file;       /* which file the lock was taken */
     unsigned int  line;       /* line number in file */
-    
+
     ethr_atomic_t tries;      /* n tries to get lock */
     ethr_atomic_t colls;      /* n collisions of tries to get lock */
-    
+
     unsigned long timer_n;    /* #times waited for lock */
     erts_lcnt_time_t timer;   /* total wait time for lock */
     erts_lcnt_hist_t hist;
@@ -155,7 +160,7 @@ typedef struct erts_lcnt_lock_s {
     /* statistics */
     unsigned int n_stats;
     erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/
-    
+
     /* chains for list handling  */
     /* data is hold by lcnt_lock */
     struct erts_lcnt_lock_s *prev;
@@ -167,7 +172,7 @@ typedef struct {
     erts_lcnt_lock_t *tail;
     unsigned long n;
 } erts_lcnt_lock_list_t;
-    
+
 typedef struct {
     erts_lcnt_time_t duration;			/* time since last clear */
     erts_lcnt_lock_list_t *current_locks;
@@ -205,6 +210,7 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock);
 /* lock operations (global) */
 void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag);
 void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id);
+void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock);
 void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock);
 
 void erts_lcnt_lock(erts_lcnt_lock_t *lock);
@@ -226,7 +232,5 @@ void   erts_lcnt_clear_counters(void);
 char  *erts_lcnt_lock_type(Uint16 type);
 erts_lcnt_data_t *erts_lcnt_get_data(void);
 
-#define ERTS_LCNT_LOCK_TYPE(lockp)	((lockp)->flag & ERTS_LCNT_LT_ALL)
-
 #endif /* ifdef  ERTS_ENABLE_LOCK_COUNT  */
 #endif /* ifndef ERTS_LOCK_COUNT_H__     */
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index b28bc498f6..0548c6137c 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -110,6 +110,9 @@ static struct {
 
 erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+static void lcnt_enable_proc_lock_count(Process *proc, int enable);
+#endif
 
 void
 erts_init_proc_lock(int cpus)
@@ -1083,30 +1086,32 @@ erts_proc_lock_fin(Process *p)
 /* --- Process lock counting ----------------------------------------------- */
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
-void erts_lcnt_proc_lock_init(Process *p) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) {
-    if (p->common.id != ERTS_INVALID_PID) {
-	erts_lcnt_init_lock_x(&(p->lock.lcnt_main),   "proc_main",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
-	erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq),   "proc_msgq",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
-	erts_lcnt_init_lock_x(&(p->lock.lcnt_btm),    "proc_btm",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
-	erts_lcnt_init_lock_x(&(p->lock.lcnt_link),   "proc_link",   ERTS_LCNT_LT_PROCLOCK, p->common.id);
-	erts_lcnt_init_lock_x(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK, p->common.id);
-    } else {
-	erts_lcnt_init_lock(&(p->lock.lcnt_main),   "proc_main",   ERTS_LCNT_LT_PROCLOCK);
-	erts_lcnt_init_lock(&(p->lock.lcnt_msgq),   "proc_msgq",   ERTS_LCNT_LT_PROCLOCK);
-	erts_lcnt_init_lock(&(p->lock.lcnt_btm),    "proc_btm",   ERTS_LCNT_LT_PROCLOCK);
-	erts_lcnt_init_lock(&(p->lock.lcnt_link),   "proc_link",   ERTS_LCNT_LT_PROCLOCK);
-	erts_lcnt_init_lock(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK);
-    }
-    } else {
-	sys_memzero(&(p->lock.lcnt_main), sizeof(p->lock.lcnt_main));
-	sys_memzero(&(p->lock.lcnt_msgq), sizeof(p->lock.lcnt_msgq));
-	sys_memzero(&(p->lock.lcnt_btm), sizeof(p->lock.lcnt_btm));
-	sys_memzero(&(p->lock.lcnt_link), sizeof(p->lock.lcnt_link));
-	sys_memzero(&(p->lock.lcnt_status), sizeof(p->lock.lcnt_status));
-    }
+
+void erts_lcnt_enable_proc_lock_count(int enable) {
+    int ix, max = erts_ptab_max(&erts_proc);
+    Process *proc = NULL;
+    for (ix = 0; ix < max; ++ix) {
+	if ((proc = erts_pix2proc(ix)) != NULL)
+            lcnt_enable_proc_lock_count(proc, enable);
+    } /* for all processes */
 }
-	
+
+void erts_lcnt_proc_lock_init(Process *p) {
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) {
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_main));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_link));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_status));
+    } else { /* now the common case */
+        Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL;
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_main),  "proc_main",  ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq),  "proc_msgq",  ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_btm),   "proc_btm",   ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_link),  "proc_link",  ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid);
+    } /* the lock names should really be aligned to four characters */
+} /* logic reversed */
 
 void erts_lcnt_proc_lock_destroy(Process *p) {
     erts_lcnt_destroy_lock(&(p->lock.lcnt_main));
@@ -1116,28 +1121,31 @@ void erts_lcnt_proc_lock_destroy(Process *p) {
     erts_lcnt_destroy_lock(&(p->lock.lcnt_status));
 }
 
-void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { 
-    if (locks & ERTS_PROC_LOCK_MAIN) {
-	erts_lcnt_lock(&(lock->lcnt_main));
-    }
-    if (locks & ERTS_PROC_LOCK_MSGQ) {
-        erts_lcnt_lock(&(lock->lcnt_msgq));
-    }
-    if (locks & ERTS_PROC_LOCK_BTM) {
-        erts_lcnt_lock(&(lock->lcnt_btm));
-    }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	erts_lcnt_lock(&(lock->lcnt_link));
-    }
-    if (locks & ERTS_PROC_LOCK_STATUS) {
-	erts_lcnt_lock(&(lock->lcnt_status));
+static void lcnt_enable_proc_lock_count(Process *proc, int enable) {
+    if (enable) {
+        if (!ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) {
+            erts_lcnt_proc_lock_init(proc);
+        }
     }
+    else {
+        if (ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) {
+            erts_lcnt_proc_lock_destroy(proc);
+        }
     }
 }
 
-void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { 
+void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
+    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); }
+    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); }
+    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); }
+}
+
+void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
+                                char *file, unsigned int line) {
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line);
     }
@@ -1153,90 +1161,34 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, cha
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line);
     }
-    }
 }
 
 void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { 
-    if (locks & ERTS_PROC_LOCK_MAIN) {
-	erts_lcnt_lock_unaquire(&(lock->lcnt_main));
-    }
-    if (locks & ERTS_PROC_LOCK_MSGQ) {
-        erts_lcnt_lock_unaquire(&(lock->lcnt_msgq));
-    }
-    if (locks & ERTS_PROC_LOCK_BTM) {
-        erts_lcnt_lock_unaquire(&(lock->lcnt_btm));
-    }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	erts_lcnt_lock_unaquire(&(lock->lcnt_link));
-    }
-    if (locks & ERTS_PROC_LOCK_STATUS) {
-	erts_lcnt_lock_unaquire(&(lock->lcnt_status));
-    }
-    }
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
+    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); }
+    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); }
+    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); }
 }
 
 void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { 
-    if (locks & ERTS_PROC_LOCK_MAIN) {
-	erts_lcnt_unlock(&(lock->lcnt_main));
-    }
-    if (locks & ERTS_PROC_LOCK_MSGQ) {
-        erts_lcnt_unlock(&(lock->lcnt_msgq));
-    }
-    if (locks & ERTS_PROC_LOCK_BTM) {
-        erts_lcnt_unlock(&(lock->lcnt_btm));
-    }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	erts_lcnt_unlock(&(lock->lcnt_link));
-    }
-    if (locks & ERTS_PROC_LOCK_STATUS) {
-	erts_lcnt_unlock(&(lock->lcnt_status));
-    }
-    }
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
+    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); }
+    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); }
+    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); }
 }
 void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { 
-    if (locks & ERTS_PROC_LOCK_MAIN) {
-	erts_lcnt_trylock(&(lock->lcnt_main), res);
-    }
-    if (locks & ERTS_PROC_LOCK_MSGQ) {
-        erts_lcnt_trylock(&(lock->lcnt_msgq), res);
-    }
-    if (locks & ERTS_PROC_LOCK_BTM) {
-        erts_lcnt_trylock(&(lock->lcnt_btm), res);
-    }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	erts_lcnt_trylock(&(lock->lcnt_link), res);
-    }
-    if (locks & ERTS_PROC_LOCK_STATUS) {
-	erts_lcnt_trylock(&(lock->lcnt_status), res);
-    }
-    }
-}
-
-
-void erts_lcnt_enable_proc_lock_count(int enable)
-{
-    int i, max = erts_ptab_max(&erts_proc);
-
-    for (i = 0; i < max; ++i) {
-	Process* p = erts_pix2proc(i);
-	if (p) {
-	    if (enable) {
-		if (!ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) {
-		    erts_lcnt_proc_lock_init(p);
-		}
-	    } else {
-		if (ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) {
-		    erts_lcnt_proc_lock_destroy(p);
-		}
-	    }
-	}
-    }
-}
-
-#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
+    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); }
+    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); }
+    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); }
+    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); }
+} /* reversed logic */
+#endif /* ERTS_ENABLE_LOCK_COUNT */
 
 
 /* --- Process lock checking ----------------------------------------------- */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 23f208c2eb..42f0bc88f2 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1534,12 +1534,10 @@ erts_schedule_proc2port_signal(Process *c_p,
 }
 
 static ERTS_INLINE void
-send_badsig(Port *prt)
-{
+send_badsig(Port *prt) {
     ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
     Process* rp;
     Eterm connected = ERTS_PORT_GET_CONNECTED(prt);
-
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     ERTS_LC_ASSERT(erts_get_scheduler_id());
 
@@ -1559,15 +1557,13 @@ send_badsig(Port *prt)
 					 0);
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
-    }
-}
+    } /* exit sent */
+} /* send_badsig */
 
 static void
-badsig_received(int bang_op,
-		Port *prt,
+badsig_received(int bang_op, Port *prt,
 		erts_aint32_t state,
-		int bad_output_value)
-{
+		int bad_output_value) {
     /*
      * if (bang_op)
      *   we are part of a "Prt ! Something" operation
@@ -1583,12 +1579,12 @@ badsig_received(int bang_op,
 	}
 	if (bang_op)
 	    send_badsig(prt);
-    }
-}
+    } /* not invalid */
+} /* behaved accordingly */
 
 static int
-port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
-{
+port_badsig(Port *prt, erts_aint32_t state, int op,
+            ErtsProc2PortSigData *sigdp) {
     if (op == ERTS_PROC2PORT_SIG_EXEC)
 	badsig_received(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP,
 			prt,
@@ -1597,16 +1593,14 @@ port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
 	port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
     return ERTS_PORT_REDS_BADSIG;
-}
-
-
-/*
- * bad_port_signal() will
+} /* port_badsig */
+/* bad_port_signal() will
  * - preserve signal order of signals.
  * - send a 'badsig' exit signal to connected process if 'from' is an
  *   internal pid and the port is alive when the bad signal reaches
  *   it.
  */
+
 static ErtsPortOpResult
 bad_port_signal(Process *c_p,
 		int flags,
@@ -2804,7 +2798,6 @@ void erts_init_io(int port_tab_size,
 }
 
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
-
 static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable)
 {
     if (dp->lock) {
@@ -2844,25 +2837,26 @@ static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable)
     }
 }
 
-void erts_lcnt_enable_io_lock_count(int enable)
-{
+void erts_lcnt_enable_io_lock_count(int enable) {
     erts_driver_t *dp;
-    int i, max = erts_ptab_max(&erts_port);
+    int ix, max = erts_ptab_max(&erts_port);
+    Port *prt;
 
-    for (i = 0; i < max; i++) {
-	Port *prt = erts_pix2port(i);
-	if (prt)
+    for (ix = 0; ix < max; ix++) {
+	if ((prt = erts_pix2port(ix)) != NULL) {
 	    lcnt_enable_port_lock_count(prt, enable);
-    }
+        }
+    } /* for all ports */
 
     lcnt_enable_drv_lock_count(&vanilla_driver, enable);
     lcnt_enable_drv_lock_count(&spawn_driver, enable);
     lcnt_enable_drv_lock_count(&fd_driver, enable);
-    for (dp = driver_list; dp; dp = dp->next)
+    /* enable lock counting in all drivers */
+    for (dp = driver_list; dp; dp = dp->next) {
 	lcnt_enable_drv_lock_count(dp, enable);
-}
-#endif
-
+    }
+} /* enable/disable lock counting of ports */
+#endif /* defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) */
 /*
  * Buffering of data when using line oriented I/O on ports
  */
-- 
cgit v1.2.3


From e14ca38380885e50a134b8c4297c44aec73ccb5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 16 Jun 2015 10:07:23 +0200
Subject: Revert "Add thread index to allocator enomem dump slogan"

This reverts commit 5d5f9c1857029d7e8e1de141e29d20dd3de929be.
---
 erts/emulator/beam/erl_alloc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d11f24220a..c9ac024743 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1884,8 +1884,8 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
 	size = va_arg(argp, Uint);
 	va_end(argp);
 	erl_exit(1,
-		 "%s: Cannot %s %lu bytes of memory (of type \"%s\", thread %d).\n",
-		 allctr_str, op, size, t_str, ERTS_ALC_GET_THR_IX());
+		 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
+		 allctr_str, op, size, t_str);
 	break;
     }
     case ERTS_ALC_E_NOALLCTR:
-- 
cgit v1.2.3


From 92e6fb7f31ad8977144a1cfbcee05895839dbc62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 16 Jun 2015 10:10:50 +0200
Subject: Revert "Add run queue index to process dump info"

This reverts commit 345af4a0c8d68b9369c3556fa6d911854c123d3f.
---
 erts/emulator/beam/break.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 02e65cb9c6..3cb605834f 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -242,7 +242,6 @@ print_process_info(int to, void *to_arg, Process *p)
 		   p->current[1],
 		   p->current[2]);
     }
-    erts_print(to, to_arg, "Run queue: %d\n", erts_get_runq_proc(p)->ix);
 
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
     approx_started = (time_t) p->approx_started;
-- 
cgit v1.2.3


From af376d145f2d32420590e15c99031433b8cb49f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 16 Jun 2015 10:14:41 +0200
Subject: Revert "Demote rare debug slogan of message discarding to debug
 build"

This reverts commit 38bd20f4f58e8025bd3ffc718cb7e40a4bde6396.
---
 erts/emulator/beam/bif.c | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2b782f4484..bfca861830 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1914,7 +1914,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
     } else if (is_external_pid(to)) {
 	dep = external_pid_dist_entry(to);
 	if(dep == erts_this_dist_entry) {
-#if DEBUG
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	    erts_dsprintf(dsbufp,
 			  "Discarding message %T from %T to %T in an old "
@@ -1925,7 +1924,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 			  external_pid_creation(to),
 			  erts_this_node->creation);
 	    erts_send_error_to_logger(p->group_leader, dsbufp);
-#endif
 	    return 0;
 	}
 	return remote_send(p, dep, to, to, msg, ctx);
@@ -1959,7 +1957,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
     } else if (is_external_port(to)
 	       && (external_port_dist_entry(to)
 		   == erts_this_dist_entry)) {
-#if DEBUG
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	erts_dsprintf(dsbufp,
 		      "Discarding message %T from %T to %T in an old "
@@ -1970,7 +1967,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 		      external_port_creation(to),
 		      erts_this_node->creation);
 	erts_send_error_to_logger(p->group_leader, dsbufp);
-#endif
 	return 0;
     } else if (is_internal_port(to)) {
 	int ret_val;
-- 
cgit v1.2.3


From 8e70bbb3a94b09600bf7ac9d3b18713502a64625 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 16 Jun 2015 23:11:04 +0200
Subject: Revert "lcnt: Let runq locks reflect actual call location"

This reverts commit efefd4bfda3156c6c19a61d7aa3d2f50a026d0e5.

Conflicts:
	erts/emulator/beam/erl_process.h
---
 erts/emulator/beam/erl_process.h | 37 -------------------------------------
 1 file changed, 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 1da1239609..241a82d6c2 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -2057,14 +2057,10 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
 ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
 ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
 ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
-#ifndef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
-#endif
 ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
 ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
-#ifndef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
-#endif
 ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
 ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
 ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
@@ -2142,12 +2138,6 @@ erts_smp_runq_lock(ErtsRunQueue *rq)
 #endif
 }
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-
-#define erts_smp_runq_lock(rq)	erts_smp_mtx_lock_x(&(rq)->mtx, __FILE__, __LINE__)
-
-#endif
-
 ERTS_GLB_INLINE int
 erts_smp_runq_trylock(ErtsRunQueue *rq)
 {
@@ -2166,31 +2156,6 @@ erts_smp_runq_unlock(ErtsRunQueue *rq)
 #endif
 }
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-
-#define erts_smp_xrunq_lock(rq, xrq)	erts_smp_xrunq_lock_x((rq), (xrq), __FILE__, __LINE__)
-
-ERTS_GLB_INLINE void
-erts_smp_xrunq_lock_x(ErtsRunQueue *rq, ErtsRunQueue *xrq, char* file, int line)
-{
-#ifdef ERTS_SMP
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
-    if (xrq != rq) {
-	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
-	    if (rq < xrq)
-		erts_smp_mtx_lock_x(&xrq->mtx, file, line);
-	    else {
-		erts_smp_mtx_unlock(&rq->mtx);
-		erts_smp_mtx_lock_x(&xrq->mtx, file, line);
-		erts_smp_mtx_lock_x(&rq->mtx, file, line);
-	    }
-	}
-    }
-#endif
-}
-
-#else
-
 ERTS_GLB_INLINE void
 erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
@@ -2210,8 +2175,6 @@ erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 #endif
 }
 
-#endif
-
 ERTS_GLB_INLINE void
 erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
-- 
cgit v1.2.3


From 74f0d7c8ea47b363136c86de5d7ea78a48c40570 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 17 Jun 2015 00:01:14 +0200
Subject: Save IO bytes in scheduler specific data

---
 erts/emulator/beam/atom.names      |   1 +
 erts/emulator/beam/dist.c          |   5 +-
 erts/emulator/beam/erl_alloc.types |   1 +
 erts/emulator/beam/erl_bif_info.c  |  23 ++----
 erts/emulator/beam/erl_port.h      |   3 +-
 erts/emulator/beam/erl_process.c   |   3 +
 erts/emulator/beam/erl_process.h   |   5 ++
 erts/emulator/beam/io.c            | 154 +++++++++++++++++++++++++++++++++----
 8 files changed, 159 insertions(+), 36 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 74b42c647e..0cf21b4635 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -265,6 +265,7 @@ atom get_seq_token
 atom get_tcw
 atom getenv
 atom gather_gc_info_result
+atom gather_io_bytes
 atom gather_sched_wall_time_result
 atom getting_linked
 atom getting_unlinked
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index ae46174a14..1354cee267 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2088,6 +2088,7 @@ erts_dist_command(Port *prt, int reds_limit)
     DistEntry *dep = prt->dist_entry;
     Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf);
     erts_aint32_t sched_flags;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
@@ -2142,12 +2143,12 @@ erts_dist_command(Port *prt, int reds_limit)
 	    ErtsDistOutputBuf *fob;
 
 	    size = (*send)(prt, foq.first);
+	    esdp->io.out += (Uint64) size;
 #ifdef ERTS_RAW_DIST_MSG_DBG
 	    erts_fprintf(stderr, ">> ");
 	    bw(foq.first->extp, size);
 #endif
 	    reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
-	    erts_smp_atomic_add_nob(&erts_bytes_out, size);
 	    fob = foq.first;
 	    obufsize += size_obuf(fob);
 	    foq.first = foq.first->next;
@@ -2227,12 +2228,12 @@ erts_dist_command(Port *prt, int reds_limit)
 	    ASSERT(&oq.first->data[0] <= oq.first->extp
 		   && oq.first->extp < oq.first->ext_endp);
 	    size = (*send)(prt, oq.first);
+	    esdp->io.out += (Uint64) size;
 #ifdef ERTS_RAW_DIST_MSG_DBG
 	    erts_fprintf(stderr, ">> ");
 	    bw(oq.first->extp, size);
 #endif
 	    reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
-	    erts_smp_atomic_add_nob(&erts_bytes_out, size);
 	    fob = oq.first;
 	    obufsize += size_obuf(fob);
 	    oq.first = oq.first->next;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 2721e13250..92c397ffd3 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -274,6 +274,7 @@ type    BUSY_CALLER	SHORT_LIVED	SYSTEM		busy_caller
 type	PROC_SYS_TSK	SHORT_LIVED	PROCESSES	proc_sys_task
 type	PROC_SYS_TSK_QS	SHORT_LIVED	PROCESSES	proc_sys_task_queues
 type    NEW_TIME_OFFSET	SHORT_LIVED	SYSTEM		new_time_offset
+type	IOB_REQ		SHORT_LIVED	SYSTEM		io_bytes_request
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 397c68e199..ba070b0ddf 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -60,6 +60,7 @@
 
 static Export* alloc_info_trap = NULL;
 static Export* alloc_sizes_trap = NULL;
+static Export* gather_io_bytes_trap = NULL;
 
 static Export *gather_sched_wall_time_res_trap;
 static Export *gather_gc_info_res_trap;
@@ -3220,7 +3221,6 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2)
    BIF_RET(am_true);
 }
 
-
 /* this is a general call which return some possibly useful information */
 
 BIF_RETTYPE statistics_1(BIF_ALIST_1)
@@ -3293,23 +3293,8 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = TUPLE2(hp, b1, b2);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_io) {
-	Eterm r1, r2;
-	Eterm in, out;
-	Uint hsz = 9;
-	Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in);
-	Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out);
-
-	(void) erts_bld_uint(NULL, &hsz, bytes_in);
-	(void) erts_bld_uint(NULL, &hsz, bytes_out);
-	hp = HAlloc(BIF_P, hsz);
-	in = erts_bld_uint(&hp, NULL, bytes_in);
-	out = erts_bld_uint(&hp, NULL, bytes_out);
-
-	r1 = TUPLE2(hp,  am_input, in);
-	hp += 3;
-	r2 = TUPLE2(hp, am_output, out);
-	hp += 3;
-	BIF_RET(TUPLE2(hp, r1, r2));
+	Eterm ref = erts_request_io_bytes(BIF_P);
+	BIF_TRAP2(gather_io_bytes_trap, BIF_P, ref, make_small(erts_no_schedulers));
     }
     else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
 	Eterm res, *hp, **hpp;
@@ -4388,6 +4373,8 @@ erts_bif_info_init(void)
 	= erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1);
     gather_gc_info_res_trap
 	= erts_export_put(am_erlang, am_gather_gc_info_result, 1);
+    gather_io_bytes_trap
+	= erts_export_put(am_erts_internal, am_gather_io_bytes, 2);
     process_info_init();
     os_info_init();
 }
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 3920fae2d9..66cbb67a0d 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -266,8 +266,7 @@ erts_prtsd_set(Port *prt, int ix, void *data)
 
 #endif
 
-extern erts_smp_atomic_t erts_bytes_out;	/* no bytes written out */
-extern erts_smp_atomic_t erts_bytes_in;		/* no bytes sent into the system */
+Eterm erts_request_io_bytes(Process *c_p);
 
 
 /* port status flags */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 43e84a1be1..88c1b5c121 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5542,6 +5542,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->thr_id = (Uint32) num;
     erts_sched_bif_unique_init(esdp);
 
+    esdp->io.out = (Uint64) 0;
+    esdp->io.in = (Uint64) 0;
+
     if (daww_ptr) {
 	init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr);
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 1da1239609..27b6fcef50 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -659,6 +659,11 @@ struct ErtsSchedulerData_ {
 
     ErtsSchedAllocData alloc_data;
 
+    struct {
+	Uint64 out;
+	Uint64 in;
+    } io;
+
     Uint64 reductions;
     ErtsSchedWallTime sched_wall_time;
     ErtsGCInfo gc_info;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 23f208c2eb..9099442a6f 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -65,8 +65,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key;  /* Save last DDLL error o
 							  per thread basis (for BC interfaces) */
 
 ErtsPTab erts_port erts_align_attribute(ERTS_CACHE_LINE_SIZE); /* The port table */
-erts_smp_atomic_t erts_bytes_out;	/* No bytes sent out of the system */
-erts_smp_atomic_t erts_bytes_in;	/* No bytes gotten into the system */
 
 const ErlDrvTermData driver_term_nil = (ErlDrvTermData)NIL;
 
@@ -80,6 +78,9 @@ int erts_port_synchronous_ops = 0;
 int erts_port_schedule_all_ops = 0;
 int erts_port_parallelism = 0;
 
+static erts_atomic64_t bytes_in;
+static erts_atomic64_t bytes_out;
+
 static void deliver_result(Eterm sender, Eterm pid, Eterm res);
 static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *);
 static void terminate_port(Port *p);
@@ -1681,6 +1682,7 @@ call_driver_outputv(int bang_op,
     if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt))
 	send_badsig(prt);
     else {
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
 	ErlDrvSizeT size = evp->size;
 
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)	
@@ -1698,7 +1700,10 @@ call_driver_outputv(int bang_op,
 	prt->caller = NIL;
 
 	prt->bytes_out += size;
-	erts_smp_atomic_add_nob(&erts_bytes_out, size);
+	if (esdp)
+	    esdp->io.out += (Uint64) size;
+	else
+	    erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size);
     }
 }
 
@@ -1778,7 +1783,7 @@ call_driver_output(int bang_op,
     if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt))
 	send_badsig(prt);
     else {
-
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)	
 			   || ERTS_IS_CRASH_DUMPING);
 
@@ -1794,7 +1799,10 @@ call_driver_output(int bang_op,
 	prt->caller = NIL;
 
 	prt->bytes_out += size;
-	erts_smp_atomic_add_nob(&erts_bytes_out, size);
+	if (esdp)
+	    esdp->io.out += (Uint64) size;
+	else
+	    erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size);
     }
 }
 
@@ -2741,6 +2749,9 @@ void erts_init_io(int port_tab_size,
     drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
     drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
+    erts_atomic64_init_nob(&bytes_in, 0);
+    erts_atomic64_init_nob(&bytes_out, 0);
+
     common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port));
     common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ));
     common_element_size += 10; /* name */
@@ -2782,9 +2793,6 @@ void erts_init_io(int port_tab_size,
 			 legacy_port_tab,
 			 1);
 
-    erts_smp_atomic_init_nob(&erts_bytes_out, 0);
-    erts_smp_atomic_init_nob(&erts_bytes_in, 0);
-
     sys_init_io();
 
     erts_smp_tsd_set(driver_list_lock_status_key, (void *) 1);
@@ -4573,6 +4581,102 @@ erts_port_info(Process* c_p,
 					  port_sig_info);
 }
 
+typedef struct {
+    Uint sched_id;
+    Eterm pid;
+    Uint32 refn[ERTS_REF_NUMBERS];
+    erts_smp_atomic32_t refc;
+} ErtsIOBytesReq;
+
+static void
+reply_io_bytes(void *vreq)
+{
+    ErtsIOBytesReq *req = (ErtsIOBytesReq *) vreq;
+    Process *rp;
+
+    rp = erts_proc_lookup(req->pid);
+    if (rp) {
+	ErlOffHeap *ohp = NULL;
+	ErlHeapFragment *bp = NULL;
+	ErtsProcLocks rp_locks;
+	Eterm ref, msg, ein, eout, *hp;
+	Uint64 in, out;
+	Uint hsz;
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+	Uint sched_id = esdp->no;
+	in = esdp->io.in;
+	out = esdp->io.out;
+	if (req->sched_id != sched_id)
+	    rp_locks = 0;
+	else {
+	    in += (Uint64) erts_atomic64_read_nob(&bytes_in);
+	    out += (Uint64) erts_atomic64_read_nob(&bytes_out);
+	    rp_locks = ERTS_PROC_LOCK_MAIN;
+	}
+
+	hsz = 5 /* 4-tuple */ + REF_THING_SIZE;
+
+	erts_bld_uint64(NULL, &hsz, in);
+	erts_bld_uint64(NULL, &hsz, out);
+
+	hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks);
+
+	ref = make_internal_ref(hp);
+	write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]);
+	hp += REF_THING_SIZE;
+
+	ein = erts_bld_uint64(&hp, NULL, in);
+	eout = erts_bld_uint64(&hp, NULL, out);
+
+	msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout);
+	erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+
+	if (req->sched_id == sched_id)
+	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+	if (rp_locks)
+	    erts_smp_proc_unlock(rp, rp_locks);
+    }
+
+    if (erts_smp_atomic32_dec_read_nob(&req->refc) == 0)
+	erts_free(ERTS_ALC_T_IOB_REQ, req);
+}
+
+Eterm
+erts_request_io_bytes(Process *c_p)
+{
+    Uint *hp;
+    Eterm ref;
+    Uint32 *refn;
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ,
+				     sizeof(ErtsIOBytesReq));
+
+    hp = HAlloc(c_p, REF_THING_SIZE);
+    ref = erts_sched_make_ref_in_buffer(esdp, hp);
+    refn = internal_ref_numbers(ref);
+
+    req->sched_id = esdp->no;
+    req->pid = c_p->common.id;
+    req->refn[0] = refn[0];
+    req->refn[1] = refn[1];
+    req->refn[2] = refn[2];
+    erts_smp_atomic32_init_nob(&req->refc,
+			       (erts_aint32_t) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+					  erts_no_schedulers,
+					  reply_io_bytes,
+					  (void *) req);
+#endif
+
+    reply_io_bytes((void *) req);
+
+    return ref;
+}
+
+
 typedef struct {
     int to;
     void *arg;
@@ -5111,6 +5215,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
     struct b2t_states__ b2t;
     int scheduler;
     int is_heap_need_limited = 1;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_UNDEF(mess,NIL);
     ERTS_UNDEF(scheduler,1);
@@ -5418,7 +5523,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    Uint size = ptr[1];
 	    Uint offset = ptr[2];
 
-	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
+	    if (esdp)
+		esdp->io.in += (Uint64) size;
+	    else
+		erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size);
 
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
 		ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
@@ -5452,7 +5560,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    byte *bufp = (byte *) ptr[0];
 	    Uint size = (Uint) ptr[1];
 
-	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
+	    if (esdp)
+		esdp->io.in += (Uint64) size;
+	    else
+		erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size);
 
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
 		ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory,
@@ -5489,7 +5600,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	}
 
 	case ERL_DRV_STRING: /* char*, length */
-	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]);
+	    if (esdp)
+		esdp->io.in += (Uint64) ptr[1];
+	    else
+		erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) ptr[1]);
 	    erts_reserve_heap(&factory, 2*ptr[1]);
 	    mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL);
 	    ptr += 2;
@@ -5765,6 +5879,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
 {
     erts_aint32_t state;
     Port* prt = erts_drvport2port_state(ix, &state);
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
@@ -5775,7 +5890,10 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
 	return 0;
 
     prt->bytes_in += (hlen + len);
-    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
+    if (esdp)
+	esdp->io.in += (Uint64) (hlen + len);
+    else
+	erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len));
     if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
 	return erts_net_message(prt,
 				prt->dist_entry,
@@ -5800,6 +5918,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
 {
     erts_aint32_t state;
     Port* prt = erts_drvport2port_state(ix, &state);
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
@@ -5811,7 +5930,10 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
 	return 0;
     
     prt->bytes_in += (hlen + len);
-    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
+    if (esdp)
+	esdp->io.in += (Uint64) (hlen + len);
+    else
+	erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len));
     if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
 	if (len == 0)
 	    return erts_net_message(prt,
@@ -5851,6 +5973,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     ErlDrvBinary** binv;
     Port* prt;
     erts_aint32_t state;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
@@ -5889,7 +6012,10 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
 
     /* XXX handle distribution !!! */
     prt->bytes_in += (hlen + size);
-    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size));
+    if (esdp)
+	esdp->io.in += (Uint64) (hlen + size);
+    else
+	erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + size));
     deliver_vec_message(prt, ERTS_PORT_GET_CONNECTED(prt), hbuf, hlen,
 			binv, iov, n, size);
     return 0;
-- 
cgit v1.2.3


From db193ba5dac15b1341a622307adbaf55002531b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 17 Jun 2015 11:23:11 +0200
Subject: erts: Fix erts_debug:df/1 in debug

Sentinels in select_tuple_arity instructions are not proper
tuple arities and thus cannot be checked in debug.
Print them as small integers instead.
---
 erts/emulator/beam/beam_debug.c | 40 ++++++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 0367ca8aba..78ddecafc3 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -616,24 +616,28 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_select_tuple_arity_rfI:
     case op_i_select_tuple_arity_xfI:
     case op_i_select_tuple_arity_yfI:
-	{
-	    int n = ap[-1];
-	    int ix = n;
-
-	    while (ix--) {
-		Uint arity = arityval(ap[0]);
-		erts_print(to, to_arg, "{%d} ", arity, ap[1]);
-		ap++;
-		size++;
-	    }
-	    ix = n;
-	    while (ix--) {
-		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
-		ap++;
-		size++;
-	    }
-	}
-	break;
+        {
+            int n = ap[-1];
+            int ix = n - 1; /* without sentinel */
+
+            while (ix--) {
+                Uint arity = arityval(ap[0]);
+                erts_print(to, to_arg, "{%d} ", arity, ap[1]);
+                ap++;
+                size++;
+            }
+            /* print sentinel */
+            erts_print(to, to_arg, "{%T} ", ap[0], ap[1]);
+            ap++;
+            size++;
+            ix = n;
+            while (ix--) {
+                erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+                ap++;
+                size++;
+            }
+        }
+        break;
     case op_i_jump_on_val_rfII:
     case op_i_jump_on_val_xfII:
     case op_i_jump_on_val_yfII:
-- 
cgit v1.2.3


From 738c34d4bb8f1a3811acd00af8c6c12107f8315b Mon Sep 17 00:00:00 2001
From: Bruce Yinhe <bruce@erlang.org>
Date: Thu, 18 Jun 2015 11:31:02 +0200
Subject: Change license text to APLv2

---
 erts/emulator/beam/atom.c                     | 19 ++++++++-------
 erts/emulator/beam/atom.h                     | 19 ++++++++-------
 erts/emulator/beam/atom.names                 | 19 ++++++++-------
 erts/emulator/beam/beam_bif_load.c            | 19 ++++++++-------
 erts/emulator/beam/beam_bp.c                  | 19 ++++++++-------
 erts/emulator/beam/beam_bp.h                  | 19 ++++++++-------
 erts/emulator/beam/beam_catches.c             | 19 ++++++++-------
 erts/emulator/beam/beam_catches.h             | 19 ++++++++-------
 erts/emulator/beam/beam_debug.c               | 19 ++++++++-------
 erts/emulator/beam/beam_emu.c                 | 19 ++++++++-------
 erts/emulator/beam/beam_load.c                | 19 ++++++++-------
 erts/emulator/beam/beam_load.h                | 19 ++++++++-------
 erts/emulator/beam/beam_ranges.c              | 19 ++++++++-------
 erts/emulator/beam/benchmark.c                | 21 ++++++++--------
 erts/emulator/beam/benchmark.h                | 21 ++++++++--------
 erts/emulator/beam/bif.c                      | 19 ++++++++-------
 erts/emulator/beam/bif.h                      | 19 ++++++++-------
 erts/emulator/beam/bif.tab                    | 21 ++++++++--------
 erts/emulator/beam/big.c                      | 19 ++++++++-------
 erts/emulator/beam/big.h                      | 19 ++++++++-------
 erts/emulator/beam/binary.c                   | 19 ++++++++-------
 erts/emulator/beam/break.c                    | 19 ++++++++-------
 erts/emulator/beam/code_ix.c                  | 19 ++++++++-------
 erts/emulator/beam/code_ix.h                  | 19 ++++++++-------
 erts/emulator/beam/copy.c                     | 19 ++++++++-------
 erts/emulator/beam/dist.c                     | 19 ++++++++-------
 erts/emulator/beam/dist.h                     | 19 ++++++++-------
 erts/emulator/beam/dtrace-wrapper.h           | 19 ++++++++-------
 erts/emulator/beam/elib_memmove.c             | 21 ++++++++--------
 erts/emulator/beam/erl_afit_alloc.c           | 21 ++++++++--------
 erts/emulator/beam/erl_afit_alloc.h           | 21 ++++++++--------
 erts/emulator/beam/erl_alloc.c                | 19 ++++++++-------
 erts/emulator/beam/erl_alloc.h                | 19 ++++++++-------
 erts/emulator/beam/erl_alloc.types            | 19 ++++++++-------
 erts/emulator/beam/erl_alloc_util.c           | 19 ++++++++-------
 erts/emulator/beam/erl_alloc_util.h           | 19 ++++++++-------
 erts/emulator/beam/erl_ao_firstfit_alloc.c    | 21 ++++++++--------
 erts/emulator/beam/erl_ao_firstfit_alloc.h    | 21 ++++++++--------
 erts/emulator/beam/erl_arith.c                | 19 ++++++++-------
 erts/emulator/beam/erl_async.c                | 19 ++++++++-------
 erts/emulator/beam/erl_async.h                | 19 ++++++++-------
 erts/emulator/beam/erl_bestfit_alloc.c        | 21 ++++++++--------
 erts/emulator/beam/erl_bestfit_alloc.h        | 21 ++++++++--------
 erts/emulator/beam/erl_bif_binary.c           | 19 ++++++++-------
 erts/emulator/beam/erl_bif_chksum.c           | 19 ++++++++-------
 erts/emulator/beam/erl_bif_ddll.c             | 21 ++++++++--------
 erts/emulator/beam/erl_bif_guard.c            | 19 ++++++++-------
 erts/emulator/beam/erl_bif_info.c             | 19 ++++++++-------
 erts/emulator/beam/erl_bif_lists.c            | 19 ++++++++-------
 erts/emulator/beam/erl_bif_op.c               | 19 ++++++++-------
 erts/emulator/beam/erl_bif_os.c               | 21 ++++++++--------
 erts/emulator/beam/erl_bif_port.c             | 19 ++++++++-------
 erts/emulator/beam/erl_bif_re.c               | 19 ++++++++-------
 erts/emulator/beam/erl_bif_trace.c            | 19 ++++++++-------
 erts/emulator/beam/erl_bif_unique.c           | 19 ++++++++-------
 erts/emulator/beam/erl_bif_unique.h           | 19 ++++++++-------
 erts/emulator/beam/erl_binary.h               | 19 ++++++++-------
 erts/emulator/beam/erl_bits.c                 | 19 ++++++++-------
 erts/emulator/beam/erl_bits.h                 | 19 ++++++++-------
 erts/emulator/beam/erl_cpu_topology.c         | 19 ++++++++-------
 erts/emulator/beam/erl_cpu_topology.h         | 19 ++++++++-------
 erts/emulator/beam/erl_db.c                   | 19 ++++++++-------
 erts/emulator/beam/erl_db.h                   | 21 ++++++++--------
 erts/emulator/beam/erl_db_hash.c              | 19 ++++++++-------
 erts/emulator/beam/erl_db_hash.h              | 21 ++++++++--------
 erts/emulator/beam/erl_db_tree.c              | 19 ++++++++-------
 erts/emulator/beam/erl_db_tree.h              | 21 ++++++++--------
 erts/emulator/beam/erl_db_util.c              | 19 ++++++++-------
 erts/emulator/beam/erl_db_util.h              | 19 ++++++++-------
 erts/emulator/beam/erl_debug.c                | 19 ++++++++-------
 erts/emulator/beam/erl_debug.h                | 19 ++++++++-------
 erts/emulator/beam/erl_driver.h               | 19 ++++++++-------
 erts/emulator/beam/erl_drv_nif.h              | 19 ++++++++-------
 erts/emulator/beam/erl_drv_thread.c           | 19 ++++++++-------
 erts/emulator/beam/erl_fun.c                  | 19 ++++++++-------
 erts/emulator/beam/erl_fun.h                  | 19 ++++++++-------
 erts/emulator/beam/erl_gc.c                   | 19 ++++++++-------
 erts/emulator/beam/erl_gc.h                   | 19 ++++++++-------
 erts/emulator/beam/erl_goodfit_alloc.c        | 19 ++++++++-------
 erts/emulator/beam/erl_goodfit_alloc.h        | 19 ++++++++-------
 erts/emulator/beam/erl_hl_timer.c             | 21 ++++++++--------
 erts/emulator/beam/erl_hl_timer.h             | 21 ++++++++--------
 erts/emulator/beam/erl_init.c                 | 19 ++++++++-------
 erts/emulator/beam/erl_instrument.c           | 19 ++++++++-------
 erts/emulator/beam/erl_instrument.h           | 21 ++++++++--------
 erts/emulator/beam/erl_lock_check.c           | 19 ++++++++-------
 erts/emulator/beam/erl_lock_check.h           | 21 ++++++++--------
 erts/emulator/beam/erl_lock_count.c           | 19 ++++++++-------
 erts/emulator/beam/erl_lock_count.h           | 19 ++++++++-------
 erts/emulator/beam/erl_map.c                  | 19 ++++++++-------
 erts/emulator/beam/erl_map.h                  | 21 ++++++++--------
 erts/emulator/beam/erl_math.c                 | 21 ++++++++--------
 erts/emulator/beam/erl_message.c              | 19 ++++++++-------
 erts/emulator/beam/erl_message.h              | 19 ++++++++-------
 erts/emulator/beam/erl_monitors.c             | 21 ++++++++--------
 erts/emulator/beam/erl_monitors.h             | 21 ++++++++--------
 erts/emulator/beam/erl_mtrace.c               | 19 ++++++++-------
 erts/emulator/beam/erl_mtrace.h               | 21 ++++++++--------
 erts/emulator/beam/erl_nif.c                  | 19 ++++++++-------
 erts/emulator/beam/erl_nif.h                  | 19 ++++++++-------
 erts/emulator/beam/erl_nif_api_funcs.h        | 19 ++++++++-------
 erts/emulator/beam/erl_node_container_utils.h | 19 ++++++++-------
 erts/emulator/beam/erl_node_tables.c          | 19 ++++++++-------
 erts/emulator/beam/erl_node_tables.h          | 19 ++++++++-------
 erts/emulator/beam/erl_port.h                 | 19 ++++++++-------
 erts/emulator/beam/erl_port_task.c            | 19 ++++++++-------
 erts/emulator/beam/erl_port_task.h            | 21 ++++++++--------
 erts/emulator/beam/erl_printf_term.c          | 19 ++++++++-------
 erts/emulator/beam/erl_printf_term.h          | 21 ++++++++--------
 erts/emulator/beam/erl_process.c              | 19 ++++++++-------
 erts/emulator/beam/erl_process.h              | 19 ++++++++-------
 erts/emulator/beam/erl_process_dict.c         | 21 ++++++++--------
 erts/emulator/beam/erl_process_dict.h         | 21 ++++++++--------
 erts/emulator/beam/erl_process_dump.c         | 19 ++++++++-------
 erts/emulator/beam/erl_process_lock.c         | 19 ++++++++-------
 erts/emulator/beam/erl_process_lock.h         | 21 ++++++++--------
 erts/emulator/beam/erl_ptab.c                 | 19 ++++++++-------
 erts/emulator/beam/erl_ptab.h                 | 19 ++++++++-------
 erts/emulator/beam/erl_rbtree.h               | 21 ++++++++--------
 erts/emulator/beam/erl_sched_spec_pre_alloc.c | 19 ++++++++-------
 erts/emulator/beam/erl_sched_spec_pre_alloc.h | 19 ++++++++-------
 erts/emulator/beam/erl_smp.h                  | 19 ++++++++-------
 erts/emulator/beam/erl_sock.h                 | 21 ++++++++--------
 erts/emulator/beam/erl_sys_driver.h           | 21 ++++++++--------
 erts/emulator/beam/erl_term.c                 | 19 ++++++++-------
 erts/emulator/beam/erl_term.h                 | 19 ++++++++-------
 erts/emulator/beam/erl_thr_progress.c         | 19 ++++++++-------
 erts/emulator/beam/erl_thr_progress.h         | 19 ++++++++-------
 erts/emulator/beam/erl_thr_queue.c            | 19 ++++++++-------
 erts/emulator/beam/erl_thr_queue.h            | 19 ++++++++-------
 erts/emulator/beam/erl_threads.h              | 19 ++++++++-------
 erts/emulator/beam/erl_time.h                 | 21 ++++++++--------
 erts/emulator/beam/erl_time_sup.c             | 19 ++++++++-------
 erts/emulator/beam/erl_trace.c                | 19 ++++++++-------
 erts/emulator/beam/erl_trace.h                | 19 ++++++++-------
 erts/emulator/beam/erl_unicode.c              | 19 ++++++++-------
 erts/emulator/beam/erl_unicode.h              | 21 ++++++++--------
 erts/emulator/beam/erl_unicode_normalize.h    | 35 ++++++++++++++-------------
 erts/emulator/beam/erl_utils.h                | 19 ++++++++-------
 erts/emulator/beam/erl_vm.h                   | 19 ++++++++-------
 erts/emulator/beam/erl_zlib.c                 | 21 ++++++++--------
 erts/emulator/beam/erl_zlib.h                 | 21 ++++++++--------
 erts/emulator/beam/erlang_dtrace.d            | 19 ++++++++-------
 erts/emulator/beam/error.h                    | 19 ++++++++-------
 erts/emulator/beam/export.c                   | 19 ++++++++-------
 erts/emulator/beam/export.h                   | 19 ++++++++-------
 erts/emulator/beam/external.c                 | 19 ++++++++-------
 erts/emulator/beam/external.h                 | 19 ++++++++-------
 erts/emulator/beam/global.h                   | 19 ++++++++-------
 erts/emulator/beam/hash.c                     | 21 ++++++++--------
 erts/emulator/beam/hash.h                     | 21 ++++++++--------
 erts/emulator/beam/index.c                    | 21 ++++++++--------
 erts/emulator/beam/index.h                    | 21 ++++++++--------
 erts/emulator/beam/io.c                       | 19 ++++++++-------
 erts/emulator/beam/module.c                   | 19 ++++++++-------
 erts/emulator/beam/module.h                   | 19 ++++++++-------
 erts/emulator/beam/ops.tab                    | 21 ++++++++--------
 erts/emulator/beam/packet_parser.c            | 21 ++++++++--------
 erts/emulator/beam/packet_parser.h            | 21 ++++++++--------
 erts/emulator/beam/register.c                 | 19 ++++++++-------
 erts/emulator/beam/register.h                 | 21 ++++++++--------
 erts/emulator/beam/safe_hash.c                | 21 ++++++++--------
 erts/emulator/beam/safe_hash.h                | 21 ++++++++--------
 erts/emulator/beam/sys.h                      | 19 ++++++++-------
 erts/emulator/beam/time.c                     | 21 ++++++++--------
 erts/emulator/beam/utils.c                    | 19 ++++++++-------
 erts/emulator/beam/version.h                  | 21 ++++++++--------
 167 files changed, 1726 insertions(+), 1559 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 84d2d5e3ed..fe91134ef4 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index 5904ae0f7e..ead56c83d8 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 0cf21b4635..f9a2f3e33e 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -3,16 +3,17 @@
 #
 # Copyright Ericsson AB 1996-2013. All Rights Reserved.
 #
-# The contents of this file are subject to the Erlang Public License,
-# Version 1.1, (the "License"); you may not use this file except in
-# compliance with the License. You should have received a copy of the
-# Erlang Public License along with this software. If not, it can be
-# retrieved online at http://www.erlang.org/.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# Software distributed under the License is distributed on an "AS IS"
-# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-# the License for the specific language governing rights and limitations
-# under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # %CopyrightEnd%
 #
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index c769428266..0e192b1ebd 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 4e711c89e0..016d0aaa32 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index b061401863..97d0539ac7 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c
index d374d0469e..c1fd17c65d 100644
--- a/erts/emulator/beam/beam_catches.c
+++ b/erts/emulator/beam/beam_catches.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h
index 51ef463b2f..59ee64d033 100644
--- a/erts/emulator/beam/beam_catches.h
+++ b/erts/emulator/beam/beam_catches.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 78ddecafc3..c756de8c8e 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1998-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2d4bf4240c..8b409e139b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 006ba5d0db..b70e5b9a2d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 46b0c60ab0..d5af634fad 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index cb6470638f..19079ba150 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c
index b16fe6b271..44f5c760c2 100644
--- a/erts/emulator/beam/benchmark.c
+++ b/erts/emulator/beam/benchmark.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h
index 7fc3933f3d..0fb0b93f12 100644
--- a/erts/emulator/beam/benchmark.h
+++ b/erts/emulator/beam/benchmark.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index bfca861830..4e3a1cef69 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index b877711544..c6ed60376a 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index eadba3eaff..4f0656d174 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -3,16 +3,17 @@
 #
 # Copyright Ericsson AB 1996-2013. All Rights Reserved.
 #
-# The contents of this file are subject to the Erlang Public License,
-# Version 1.1, (the "License"); you may not use this file except in
-# compliance with the License. You should have received a copy of the
-# Erlang Public License along with this software. If not, it can be
-# retrieved online at http://www.erlang.org/.
-#
-# Software distributed under the License is distributed on an "AS IS"
-# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-# the License for the specific language governing rights and limitations
-# under the License.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # %CopyrightEnd%
 #
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index a4ea9c59ca..044bf6a34e 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 5b5550da43..4aa9724ae3 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index cc0b3b9b6c..e670fbf31c 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 3cb605834f..64c8bc5e58 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index d925709bd0..209d008d18 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 16ad900228..5f00b409ef 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index cddadb346c..8849dadd00 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 1354cee267..23897a49ae 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index cd2cc0ef4a..fb777d9ac1 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h
index 6ec0c91e21..d343dc5ab0 100644
--- a/erts/emulator/beam/dtrace-wrapper.h
+++ b/erts/emulator/beam/dtrace-wrapper.h
@@ -4,16 +4,17 @@
  * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012.
  * All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/elib_memmove.c b/erts/emulator/beam/elib_memmove.c
index d2fe8649ed..d4ca30158e 100644
--- a/erts/emulator/beam/elib_memmove.c
+++ b/erts/emulator/beam/elib_memmove.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1997-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index eca4e3b3bb..47dafa53c0 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h
index b90ac8f7c5..ef050ff50e 100644
--- a/erts/emulator/beam/erl_afit_alloc.h
+++ b/erts/emulator/beam/erl_afit_alloc.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index c9ac024743..55c164bf11 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2002-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index d3109b9432..f540bae20d 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2002-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 92c397ffd3..b1d511ab78 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -3,16 +3,17 @@
 #
 # Copyright Ericsson AB 2003-2014. All Rights Reserved.
 #
-# The contents of this file are subject to the Erlang Public License,
-# Version 1.1, (the "License"); you may not use this file except in
-# compliance with the License. You should have received a copy of the
-# Erlang Public License along with this software. If not, it can be
-# retrieved online at http://www.erlang.org/.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# Software distributed under the License is distributed on an "AS IS"
-# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-# the License for the specific language governing rights and limitations
-# under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # %CopyrightEnd%
 #
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index b92533f228..236ee35d18 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2002-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index eee920e66c..df1f0aa65a 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2002-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 396aa88e0b..7c2a5c3323 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index 25b344c6a8..4200f20622 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index 47d516534f..b8c5ef9b09 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2010. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index bc06d41720..f071898046 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h
index 95374a8fc9..65538bcef0 100644
--- a/erts/emulator/beam/erl_async.h
+++ b/erts/emulator/beam/erl_async.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2011. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 59c14899a2..fb853b65ab 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h
index 870439e886..b315518b88 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.h
+++ b/erts/emulator/beam/erl_bestfit_alloc.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 934904d58e..134aa2d396 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2010-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index 4302fe8f79..e3074d6309 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2008-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 7b35edc9c4..28bec6325c 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2006-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index 6226ec2d04..4a9a6a5fcd 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 7eb31fb80e..b44382cde8 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index e006d57124..5583dcb371 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index 37dd6457db..c9192fc420 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index e07c622928..2333ca0851 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1999-2012. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 7ce950e090..3ff54c7a60 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2001-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 448c6f6f6d..86951f32b0 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2008-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 13e0160648..03f51132b1 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index 57b0bab72f..5eca09c5a6 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index cd001172a1..37d5d91c39 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 6b96787d40..ea01bf08f0 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 2e29bf8895..01734c55d7 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 388d943755..8b7807fbd9 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index f594cb9392..8395f6ecc6 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2010-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h
index b502258dae..f3fbfc6da0 100644
--- a/erts/emulator/beam/erl_cpu_topology.h
+++ b/erts/emulator/beam/erl_cpu_topology.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2010-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 844272c699..878ee32b47 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 5039b71108..a589af784c 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index dce0a3d621..81b0c4465c 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1998-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index f12cd363b0..66d8ec71d9 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1998-2014. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index c7bccc78c3..465aa566ad 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1998-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h
index 7bc235e135..6098387f5d 100644
--- a/erts/emulator/beam/erl_db_tree.h
+++ b/erts/emulator/beam/erl_db_tree.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1998-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index d47ff03a30..dab357a079 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1998-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index b2d5a306cb..1ccdc0305b 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1998-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 50bdc79506..77a1e3d7cb 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1998-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h
index af51212281..4905e64f07 100644
--- a/erts/emulator/beam/erl_debug.h
+++ b/erts/emulator/beam/erl_debug.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2004-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index e498ac70ec..6b406d069c 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 4e8c6dc68b..e2385f63f4 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2010. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 240faa823d..e0404eb5c9 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2007-2011. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 88947b5536..4268e2d40a 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2010. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index b673ef6b3c..0024b1ff71 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 71ca2713b2..d2604f1595 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2002-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index bd6dcc9078..ecd1bf4d22 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2007-2011. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index e9d8249ee1..f89f8723d9 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h
index 385de0da23..ababdbd0a1 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.h
+++ b/erts/emulator/beam/erl_goodfit_alloc.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 8eacb921fe..907491f616 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2015. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 24c57fc873..0931bb8965 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2015. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index ac1f00d2d8..d9c3b0dcf4 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1997-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index da85b86c87..12a72ad839 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h
index 37b9b67139..cb3b1920d3 100644
--- a/erts/emulator/beam/erl_instrument.h
+++ b/erts/emulator/beam/erl_instrument.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 617ce84895..3b3b247020 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2005-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 3f7f417e61..66251ef4e8 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2005-2012. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index c4956d8569..bd00480ba2 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2008-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 051f55271d..4cc6a5c695 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2008-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 95a10daa67..a91e36e3c5 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  *
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index b5941c5c9a..c391de3f11 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2014. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c
index 9b864628db..b46cc37495 100644
--- a/erts/emulator/beam/erl_math.c
+++ b/erts/emulator/beam/erl_math.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1997-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index f806d2c498..ef52823287 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1997-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 705ba5e506..fbdf3fb0e2 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1997-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 244a2b26db..7dfa01c8ac 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2004-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h
index 9972890db7..901ed8940b 100644
--- a/erts/emulator/beam/erl_monitors.h
+++ b/erts/emulator/beam/erl_monitors.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2004-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index fa1bde1c87..fdaf02f37a 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_mtrace.h b/erts/emulator/beam/erl_mtrace.h
index 204543ddb0..b34eab3c4a 100644
--- a/erts/emulator/beam/erl_mtrace.h
+++ b/erts/emulator/beam/erl_mtrace.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 27f6c6f00d..add4a66f90 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2009-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index af806736fd..7d880126f8 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2009-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index e38a016958..f93152c921 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2009-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 17f6b32bb1..211b1a0090 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2001-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 0950d7e7ef..2fb790b953 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2001-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 54c5cd1d11..694ac84232 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2001-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 66cbb67a0d..acd68ef0ad 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index c701737e26..5c38db1cbc 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2006-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 406cd3c492..335f7a77d5 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2006-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index e0dfbd31b8..267c0b3ff4 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2005-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h
index f92c99d713..87618a36d7 100644
--- a/erts/emulator/beam/erl_printf_term.h
+++ b/erts/emulator/beam/erl_printf_term.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2005-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 88c1b5c121..9ae16b1898 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index a6c3790991..20ffe7ea7c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 00761f2d0e..8606371bdf 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1999-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index 8fad2a67ab..cc53800eb5 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1999-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 36bb6b2f0e..25f0b1ed38 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2003-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 0548c6137c..0bee2c848c 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2007-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 8957e7773b..788348e613 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2007-2012. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index c688db98d8..f7997df051 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index 102d41e07f..8fd961e3ce 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h
index ea0a8976bb..5fefaea978 100644
--- a/erts/emulator/beam/erl_rbtree.h
+++ b/erts/emulator/beam/erl_rbtree.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2015. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index a490aec734..caec24bc03 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
index 9144c73acd..4d07b0f674 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 6c40edeb3e..5fc5e989a6 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2005-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_sock.h b/erts/emulator/beam/erl_sock.h
index 7ae6116dc5..7be6062115 100644
--- a/erts/emulator/beam/erl_sock.h
+++ b/erts/emulator/beam/erl_sock.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2003-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_sys_driver.h b/erts/emulator/beam/erl_sys_driver.h
index dab4a94a9b..4031eef0aa 100644
--- a/erts/emulator/beam/erl_sys_driver.h
+++ b/erts/emulator/beam/erl_sys_driver.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2001-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 565528193e..89459fb278 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 7b15b34da1..90e35151b0 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2000-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 78e0964e8b..7148b756e7 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2011-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index cf11c4e114..b89cc4c267 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2011-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index f8ca87ddcc..3a91ca9dbe 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2011-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h
index 13af758b3f..27a6d03224 100644
--- a/erts/emulator/beam/erl_thr_queue.h
+++ b/erts/emulator/beam/erl_thr_queue.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2011-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index dc20ac207f..5347979372 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2001-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index e64b86496a..36a3d52264 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index e550c999b8..7f8f560681 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2015. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index aaecc5a02e..e1b03a057f 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1999-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 4f2c70d6e7..7405490f76 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index f8e1431a53..551717139d 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2008-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_unicode.h b/erts/emulator/beam/erl_unicode.h
index 1b63b797c2..4c25d89b7c 100644
--- a/erts/emulator/beam/erl_unicode.h
+++ b/erts/emulator/beam/erl_unicode.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2008-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h
index fb0a111ca2..16c62db50e 100644
--- a/erts/emulator/beam/erl_unicode_normalize.h
+++ b/erts/emulator/beam/erl_unicode_normalize.h
@@ -1,21 +1,22 @@
 /*
-* %CopyrightBegin%
-*
-* Copyright Ericsson AB 1999-2010. All Rights Reserved.
-*
-* The contents of this file are subject to the Erlang Public License,
-* Version 1.1, (the "License"); you may not use this file except in
-* compliance with the License. You should have received a copy of the
-* Erlang Public License along with this software. If not, it can be
-* retrieved online at http://www.erlang.org/.
-*
-* Software distributed under the License is distributed on an "AS IS"
-* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-* the License for the specific language governing rights and limitations
-* under the License.
-*
-* %CopyrightEnd%
-*/
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
 /*
 * This file is automatically generated by dec.erl, do not edit manually
 */
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 6a28105cb9..6dab3bf297 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 2012-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 3a9fb1e07b..8948ca2ea9 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c
index 8e33144f96..e44e86c39c 100644
--- a/erts/emulator/beam/erl_zlib.c
+++ b/erts/emulator/beam/erl_zlib.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2009-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h
index 160166c66b..2ca1e3735e 100644
--- a/erts/emulator/beam/erl_zlib.h
+++ b/erts/emulator/beam/erl_zlib.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2009-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d
index e3ebbb84f4..c682f76e3a 100644
--- a/erts/emulator/beam/erlang_dtrace.d
+++ b/erts/emulator/beam/erlang_dtrace.d
@@ -4,16 +4,17 @@
  * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012.
  * All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index e63967adb6..cba8672c68 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2010. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index b0f08d8245..2420df36b5 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index 61a54de59f..a8bc9d2f66 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 0a69172980..c6d7e3fcc5 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 508ab8dc17..d12051c6b4 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 14d42599a1..ec9296d034 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index afaf32f8ce..e0fde337f2 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h
index 6dd66fc9b3..87fdb360e3 100644
--- a/erts/emulator/beam/hash.h
+++ b/erts/emulator/beam/hash.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index 79c3ecf1b3..06d0b5123d 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 537bc11056..14fab41026 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 75d80b1a58..900616c981 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index daa6e136c5..86dd3b5aac 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 5235528e98..c8a6351b04 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index ece038131e..1d32e72247 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -3,16 +3,17 @@
 #
 # Copyright Ericsson AB 1997-2013. All Rights Reserved.
 #
-# The contents of this file are subject to the Erlang Public License,
-# Version 1.1, (the "License"); you may not use this file except in
-# compliance with the License. You should have received a copy of the
-# Erlang Public License along with this software. If not, it can be
-# retrieved online at http://www.erlang.org/.
-#
-# Software distributed under the License is distributed on an "AS IS"
-# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-# the License for the specific language governing rights and limitations
-# under the License.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # %CopyrightEnd%
 #
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index db0e78b1a7..2dd421a9e9 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2008-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h
index 1c3a9aa3da..ff158ff8b8 100644
--- a/erts/emulator/beam/packet_parser.h
+++ b/erts/emulator/beam/packet_parser.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2008-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 4d557b3a17..7ade8bca0f 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h
index 7170463375..144536f34b 100644
--- a/erts/emulator/beam/register.h
+++ b/erts/emulator/beam/register.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 3326e5cc2a..3f039c8dfd 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h
index c691126ef9..a11370813c 100644
--- a/erts/emulator/beam/safe_hash.h
+++ b/erts/emulator/beam/safe_hash.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 2008-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 78eb0bfe2b..bb871b05ba 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index fe7c5826f5..0ab6661c9f 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2013. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index cecd88197e..342e91e983 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3,16 +3,17 @@
  *
  * Copyright Ericsson AB 1996-2014. All Rights Reserved.
  *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  *
  * %CopyrightEnd%
  */
diff --git a/erts/emulator/beam/version.h b/erts/emulator/beam/version.h
index 3952c751b7..004725eaf5 100644
--- a/erts/emulator/beam/version.h
+++ b/erts/emulator/beam/version.h
@@ -3,16 +3,17 @@
  * 
  * Copyright Ericsson AB 1996-2009. All Rights Reserved.
  * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  * 
  * %CopyrightEnd%
  */
-- 
cgit v1.2.3


From 0a54d9988e7803e7d190f504838c6fe83f79bc8f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 18 Jun 2015 19:07:12 +0200
Subject: erts: Fix timer wheel initialization bug for non smp

Init esdp->timer_wheel as NULL to please setup_aux_work_timer().
---
 erts/emulator/beam/erl_process.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 88c1b5c121..fa37d0a9c9 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5496,6 +5496,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 		    ErtsRunQueue* runq,
 		    char** daww_ptr, size_t daww_sz)
 {
+    esdp->timer_wheel = NULL;
 #ifdef ERTS_SMP
     erts_bits_init_state(&esdp->erl_bits_state);
     esdp->match_pseudo_process = NULL;
-- 
cgit v1.2.3


From 37f143e9e16e89d753b0e5e4415968dbcd5f6b65 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 22 Jun 2015 20:27:52 +0200
Subject: Fix node/dist refc count

---
 erts/emulator/beam/erl_hl_timer.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 907491f616..51a0d68247 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -2966,7 +2966,7 @@ debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct)
 	= (ErtsDebugForeachCallbackTimer *) vdfct;
 
     if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
-	&& (tmr->receiver.callback && dfct->tclbk))
+	&& (tmr->receiver.callback == dfct->tclbk))
 	(*dfct->func)(dfct->arg,
 		      tmr->timeout,
 		      tmr->head.u.arg);
@@ -2984,7 +2984,7 @@ debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct)
 			       vdfct);
 
     if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
-	&& (tmr->receiver.callback && dfct->tclbk))
+	&& (tmr->receiver.callback == dfct->tclbk))
 	(*dfct->func)(dfct->arg,
 		      tmr->timeout,
 		      tmr->head.u.arg);
@@ -3037,7 +3037,7 @@ erts_debug_callback_timer_foreach(void (*tclbk)(void *),
 	    debug_callback_timer_foreach(srv->yield.root,
 					 (void *) &dfct);
 
-	time_rbt_foreach(srv->btm_tree,
+	time_rbt_foreach(srv->time_tree,
 			 debug_callback_timer_foreach,
 			 (void *) &dfct);
     }
-- 
cgit v1.2.3


From 1c86a620d74f4f9383c4956dafd3e2486300dc0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 17 Jun 2015 17:33:29 +0200
Subject: erts: Remove HALFWORD_HEAP definition

---
 erts/emulator/beam/beam_emu.c                 |  13 +-
 erts/emulator/beam/beam_load.c                |  17 +-
 erts/emulator/beam/bif.c                      |  11 +-
 erts/emulator/beam/big.c                      |  20 +--
 erts/emulator/beam/big.h                      |   9 +-
 erts/emulator/beam/copy.c                     |  32 ----
 erts/emulator/beam/dist.c                     |  14 +-
 erts/emulator/beam/erl_alloc.c                | 142 +--------------
 erts/emulator/beam/erl_alloc_util.c           |  38 +---
 erts/emulator/beam/erl_alloc_util.h           |   3 -
 erts/emulator/beam/erl_bif_binary.c           |   3 +-
 erts/emulator/beam/erl_bif_info.c             |   6 +-
 erts/emulator/beam/erl_bif_re.c               |   4 +-
 erts/emulator/beam/erl_bif_unique.c           |   6 +-
 erts/emulator/beam/erl_bits.c                 |   4 +-
 erts/emulator/beam/erl_db.c                   |  11 +-
 erts/emulator/beam/erl_db_hash.c              |   3 -
 erts/emulator/beam/erl_db_tree.c              |   3 -
 erts/emulator/beam/erl_db_util.c              | 245 +-------------------------
 erts/emulator/beam/erl_db_util.h              |   3 -
 erts/emulator/beam/erl_gc.c                   |  29 +--
 erts/emulator/beam/erl_init.c                 |   5 -
 erts/emulator/beam/erl_lock_check.c           |   3 -
 erts/emulator/beam/erl_map.c                  |   8 -
 erts/emulator/beam/erl_map.h                  |  14 +-
 erts/emulator/beam/erl_message.h              |   3 -
 erts/emulator/beam/erl_node_container_utils.h |   2 +-
 erts/emulator/beam/erl_node_tables.c          |  16 --
 erts/emulator/beam/erl_node_tables.h          |   2 +-
 erts/emulator/beam/erl_printf_term.c          |   6 +-
 erts/emulator/beam/erl_process.c              |  27 +--
 erts/emulator/beam/erl_term.h                 |  82 ++-------
 erts/emulator/beam/erl_utils.h                |  16 +-
 erts/emulator/beam/erl_vm.h                   |   6 +-
 erts/emulator/beam/external.c                 |  50 +-----
 erts/emulator/beam/global.h                   |  25 +--
 erts/emulator/beam/io.c                       |  20 +--
 erts/emulator/beam/sys.h                      |  53 ------
 erts/emulator/beam/utils.c                    |  41 +----
 39 files changed, 94 insertions(+), 901 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8b409e139b..0c6181077c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -756,7 +756,7 @@ void** beam_ops;
 #define IsBitstring(Src, Fail) \
   if (is_not_binary(Src)) { Fail; }
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 #define BsSafeMul(A, B, Fail, Target)		\
    do { Uint64 _res = (A) * (B);		\
       if (_res / B != A) { Fail; }		\
@@ -1247,9 +1247,6 @@ void process_main(void)
 
     PROCESS_MAIN_CHK_LOCKS(c_p);
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-#if HALFWORD_HEAP
-    ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0);
-#endif
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     c_p = schedule(c_p, reds_used);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2131,8 +2128,6 @@ void process_main(void)
 		  * c_p->def_arg_reg[0].  Note that it is safe to use this
 		  * location because there are no living x registers in
 		  * a receive statement.
-		  * Note that for the halfword emulator, the two first elements
-		  * of the array are used.
 		  */
 		 BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
 		 *pi = I+3;
@@ -4535,11 +4530,11 @@ do {								\
 	     _integer = get_int32(_mb->base + _mb->offset/8);
 	 }
 	 _mb->offset += 32;
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
 	 if (IS_USMALL(0, _integer)) {
 #endif
 	     _result = make_small(_integer);
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
 	 } else {
 	     TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
 	     _result = uint_to_big((Uint) _integer, HTOP);
@@ -5129,7 +5124,7 @@ do {								\
      neg_o_reds = -c_p->def_arg_reg[4];
      FCALLS = c_p->fcalls;
      SWAPIN;
-     switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */
+     switch( c_p->def_arg_reg[3] ) {
        case HIPE_MODE_SWITCH_RES_RETURN:
 	 ASSERT(is_value(reg[0]));
 	 MoveReturn(reg[0], r(0));
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index b70e5b9a2d..10baab506a 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1894,15 +1894,14 @@ load_code(LoaderState* stp)
 			 */
 			{
 			    Eterm* hp;
-/* XXX:PaN - Halfword should use ARCH_64 variant instead */
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
 			    Uint high, low;
 # endif
 			    last_op->a[arg].val = new_literal(stp, &hp,
 							      FLOAT_SIZE_OBJECT);
 			    hp[0] = HEADER_FLONUM;
 			    last_op->a[arg].type = TAG_q;
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 			    GetInt(stp, 8, hp[1]);
 # else
 			    GetInt(stp, 4, high);
@@ -3272,14 +3271,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -3296,7 +3295,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	}
 #endif
     } else {
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
     error:
 #endif
 	op->op = genop_i_wait_error_0;
@@ -3319,14 +3318,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -3343,7 +3342,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	}
 #endif
     } else {
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
     error:
 #endif
 	op->op = genop_i_wait_error_locked_0;
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 4e3a1cef69..5ec1840c7b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4117,16 +4117,9 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3)
     if (arity < 0) {
 	goto error;
     }
-#if HALFWORD_HEAP
-    hp = HAlloc(BIF_P, 3);
-    hp[0] = HEADER_EXPORT;
-    /* Yes, May be misaligned, but X86_64 will fix it... */
-    *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity);
-#else
     hp = HAlloc(BIF_P, 2);
     hp[0] = HEADER_EXPORT;
     hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity);
-#endif
     BIF_RET(make_export(hp));
 }
 
@@ -4631,7 +4624,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2)
     if ((range = signed_val(BIF_ARG_2)) <= 0) {  /* [1..MAX_SMALL] */
 	BIF_ERROR(BIF_P, BADARG);
     }
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     if (range > ((1L << 27) - 1))
 	BIF_ERROR(BIF_P, BADARG);
 #endif
@@ -4703,7 +4696,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
     /*
      * Return either a small or a big. Use the heap for bigs if there is room.
      */
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     BIF_RET(make_small(final_hash));
 #else
     if (IS_USMALL(0, final_hash)) {
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 044bf6a34e..1e6582a7ca 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1487,20 +1487,8 @@ Eterm uint_to_big(Uint x, Eterm *y)
 
 Eterm uword_to_big(UWord x, Eterm *y)
 {
-#if HALFWORD_HEAP
-    Uint upper = x >> 32;
-    Uint lower = x & 0xFFFFFFFFUL;
-    if (upper == 0) {
-	*y = make_pos_bignum_header(1);
-    } else {
-	*y = make_pos_bignum_header(2);
-	BIG_DIGIT(y, 1) = upper;
-    }
-    BIG_DIGIT(y, 0) = lower;
-#else
     *y = make_pos_bignum_header(1);
     BIG_DIGIT(y, 0) = x;
-#endif
     return make_big(y);
 }
 
@@ -1525,7 +1513,7 @@ Eterm small_to_big(Sint x, Eterm *y)
 Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp)
 {
     Eterm *hp = *hpp;
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
     if (x >= (((Uint64) 1) << 32)) {
 	*hp = make_pos_bignum_header(2);
 	BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff));
@@ -1555,7 +1543,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	neg = 1;
 	ux = -(Uint64)x;
     }
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
     if (ux >= (((Uint64) 1) << 32)) {
 	if (neg)
 	    *hp = make_neg_bignum_header(2);
@@ -1588,7 +1576,7 @@ erts_uint64_array_to_big(Uint **hpp, int neg, int len, Uint64 *array)
 
     pot_digits = digits = 0;
     for (i = 0; i < len; i++) {
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
 	Uint low_val = array[i] & ((Uint) 0xffffffff);
 	Uint high_val = (array[i] >> 32) & ((Uint) 0xffffffff);
 	BIG_DIGIT(headerp, pot_digits) = low_val;
@@ -1651,8 +1639,6 @@ big_to_double(Wterm x, double* resp)
 /*
  * Logic has been copied from erl_bif_guard.c and slightly
  * modified to use a static instead of dynamic heap
- *
- * HALFWORD: Return relative term with 'heap' as base.
  */
 Eterm
 double_to_big(double x, Eterm *heap, Uint hsz)
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 4aa9724ae3..94f9bce10e 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -35,7 +35,7 @@
 
 typedef Uint     ErtsDigit;
 
-#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8)
+#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8)
 /* Assume 32-bit machine with long long support */
 typedef Uint64   ErtsDoubleDigit;
 typedef Uint16   ErtsHalfDigit;
@@ -90,13 +90,9 @@ typedef Uint  dsize_t;	 /* Vector size type */
 
 #define BIG_UINT_HEAP_SIZE (1 + 1)	/* always, since sizeof(Uint) <= sizeof(Eterm) */
 
-#if HALFWORD_HEAP
-#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2)
-#else
 #define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE
-#endif
 
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
 
 #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \
   ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1))
@@ -178,4 +174,3 @@ Eterm erts_sint64_to_big(Sint64, Eterm **);
 Eterm erts_chars_to_integer(Process *, char*, Uint, const int);
 
 #endif
-
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 8849dadd00..94048f4c59 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -69,11 +69,7 @@ copy_object(Eterm obj, Process* to)
  * Return the "flat" size of the object.
  */
 
-#if HALFWORD_HEAP
-Uint size_object_rel(Eterm obj, Eterm* base)
-#else
 Uint size_object(Eterm obj)
-#endif
 {
     Uint sum = 0;
     Eterm* ptr;
@@ -227,12 +223,7 @@ Uint size_object(Eterm obj)
 /*
  *  Copy a structure to a heap.
  */
-#if HALFWORD_HEAP
-Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
-                      Eterm* src_base, Eterm* dst_base)
-#else
 Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
-#endif
 {
     char* hstart;
     Uint hsize;
@@ -287,13 +278,10 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    break;
 	case TAG_PRIMARY_LIST:
 	    objp = list_val_rel(obj,src_base);
-	#if !HALFWORD_HEAP || defined(DEBUG)
 	    if (in_area(objp,hstart,hsize)) {
-		ASSERT(!HALFWORD_HEAP);
 		hp++;
 		break;
 	    }
-	#endif
 	    argp = hp++;
 	    /* Fall through */
 
@@ -309,17 +297,9 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		}
 		else {
 		    CAR(htop) = elem;
-		#if HALFWORD_HEAP
-		    CDR(htop) = CDR(objp);
-		    *tailp = make_list_rel(htop,dst_base);
-		    htop += 2;
-		    goto L_copy;
-		#else
 		    tailp = &CDR(htop);
 		    htop += 2;
-		#endif
 		}
-		ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot);
 		*tp = make_list_rel(tailp - 1, dst_base);
 		obj = CDR(objp);
 		if (!is_list(obj)) {
@@ -337,13 +317,10 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    }
 	    
 	case TAG_PRIMARY_BOXED:
-	#if !HALFWORD_HEAP || defined(DEBUG)
 	    if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) {
-		ASSERT(!HALFWORD_HEAP);
 		hp++;
 		break;
 	    }
-	#endif
 	    argp = hp++;
 
 	L_copy_boxed:
@@ -563,21 +540,12 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
  *
  * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr).
  */
-#if HALFWORD_HEAP
-Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
-		       Eterm* src_base)
-#else
 Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
-#endif
 {
     Eterm* tp = ptr;
     Eterm* hp = *hpp;
     const Eterm res = make_tuple(hp);
-#if HALFWORD_HEAP
-    const Sint offs = COMPRESS_POINTER(hp - (tp - src_base));
-#else
     const Sint offs = (hp - tp) * sizeof(Eterm);
-#endif
 
     while (sz--) {
 	Eterm val = *tp++;
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 23897a49ae..efd5109269 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -732,19 +732,11 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx)
     Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx),
 					       erts_dsend_context_dtor);
     struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin);
-    Uint ctl_size = !HALFWORD_HEAP ? 0 : (arityval(ctx->ctl_heap[0]) + 1);
-    Eterm* hp = HAlloc(p, ctl_size + PROC_BIN_SIZE);
+    Eterm* hp = HAlloc(p, PROC_BIN_SIZE);
 
     sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext));
     ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap));
-#if !HALFWORD_HEAP
     dst->ctx.dss.ctl = make_tuple(dst->ctx.ctl_heap);
-#else
-    /* Must put control tuple in low mem */
-    sys_memcpy(hp, ctx->ctl_heap,  ctl_size*sizeof(Eterm));
-    dst->ctx.dss.ctl = make_tuple(hp);
-    hp += ctl_size;
-#endif
     if (ctx->dss.acmp) {
 	sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap));
 	dst->ctx.dss.acmp = &dst->acm;
@@ -2061,9 +2053,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
 }
 
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL
-#elif defined(ARCH_32) || HALFWORD_HEAP
+#elif defined(ARCH_32)
 #define ERTS_PORT_REDS_MASK__ 0x003fffff
 #else
 #  error "Ohh come on ... !?!"
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 55c164bf11..d68f22d573 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -123,10 +123,6 @@ typedef union {
 
 static ErtsAllocatorState_t std_alloc_state;
 static ErtsAllocatorState_t ll_alloc_state;
-#if HALFWORD_HEAP
-static ErtsAllocatorState_t std_low_alloc_state;
-static ErtsAllocatorState_t ll_low_alloc_state;
-#endif
 static ErtsAllocatorState_t sl_alloc_state;
 static ErtsAllocatorState_t temp_alloc_state;
 static ErtsAllocatorState_t eheap_alloc_state;
@@ -150,24 +146,10 @@ typedef struct {
 #define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2)
 #define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC
 
-#if !HALFWORD_HEAP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
-				 ErtsAllocInfoReq,
-				 5,
-				 ERTS_ALC_T_AINFO_REQ)
-#else
-static ERTS_INLINE ErtsAllocInfoReq *
-aireq_alloc(void)
-{
-    return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq));
-}
-
-static ERTS_INLINE void
-aireq_free(ErtsAllocInfoReq *ptr)
-{
-    erts_free(ERTS_ALC_T_AINFO_REQ, ptr);
-}
-#endif
+                                 ErtsAllocInfoReq,
+                                 5,
+                                 ERTS_ALC_T_AINFO_REQ)
 
 ErtsAlcType_t erts_fix_core_allocator_ix;
 
@@ -229,10 +211,6 @@ typedef struct {
     struct au_init ets_alloc;
     struct au_init driver_alloc;
     struct au_init fix_alloc;
-#if HALFWORD_HEAP
-    struct au_init std_low_alloc;
-    struct au_init ll_low_alloc;
-#endif
 } erts_alc_hndl_args_init_t;
 
 #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
@@ -259,10 +237,6 @@ set_default_sl_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_SHORT_LIVED;
     ip->init.util.rsbcst	= 80;
-#if HALFWORD_HEAP
-    ip->init.util.force         = 1;
-    ip->init.util.low_mem       = 1;
-#endif
     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
@@ -328,10 +302,6 @@ set_default_temp_alloc_opts(struct au_init *ip)
     ip->init.util.ts 		= ERTS_ALC_MTA_TEMPORARY;
     ip->init.util.rsbcst	= 90;
     ip->init.util.rmbcmt	= 100;
-#if HALFWORD_HEAP
-    ip->init.util.force         = 1;
-    ip->init.util.low_mem       = 1;
-#endif
 }
 
 static void
@@ -350,10 +320,6 @@ set_default_eheap_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_EHEAP;
     ip->init.util.rsbcst	= 50;
-#if HALFWORD_HEAP
-    ip->init.util.force         = 1;
-    ip->init.util.low_mem       = 1;
-#endif
     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC;
 }
 
@@ -560,12 +526,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
 	= sizeof(Process);
-#if !HALFWORD_HEAP
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)]
 	= ERTS_MONITOR_SH_SIZE * sizeof(Uint);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)]
 	= ERTS_LINK_SH_SIZE * sizeof(Uint);
-#endif
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)]
 	= sizeof(ErtsDrvEventDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
@@ -745,24 +709,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     erts_allctrs[ERTS_ALC_A_SYSTEM].free		= erts_sys_free;
     erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled	= 1;
 
-#if HALFWORD_HEAP
-    /* Init low memory variants by cloning */
-    init.std_low_alloc = init.std_alloc;
-    init.std_low_alloc.init.util.name_prefix	= "std_low_";
-    init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
-    init.std_low_alloc.init.util.force = 1;
-    init.std_low_alloc.init.util.low_mem = 1;
-
-    init.ll_low_alloc = init.ll_alloc;
-    init.ll_low_alloc.init.util.name_prefix	= "ll_low_";
-    init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
-    init.ll_low_alloc.init.util.force = 1;
-    init.ll_low_alloc.init.util.low_mem = 1;
-
-    set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu);
-    set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu);
-#endif /* HALFWORD */
-
     set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
@@ -805,14 +751,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     start_au_allocator(ERTS_ALC_A_LONG_LIVED,
 		       &init.ll_alloc,
 		       &ll_alloc_state);
-#if HALFWORD_HEAP
-    start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW,
-		       &init.ll_low_alloc,
-		       &ll_low_alloc_state);
-    start_au_allocator(ERTS_ALC_A_STANDARD_LOW,
-		       &init.std_low_alloc,
-		       &std_low_alloc_state);
-#endif
     start_au_allocator(ERTS_ALC_A_EHEAP,
 		       &init.eheap_alloc,
 		       &eheap_alloc_state);
@@ -836,9 +774,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     erts_mtrace_install_wrapper_functions();
     extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
 
-#if !HALFWORD_HEAP
     init_aireq_alloc();
-#endif
 
 #ifdef DEBUG
     extra_block_size += install_debug_functions();
@@ -1238,9 +1174,6 @@ get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip)
     if (sys_strcmp(value, "de") == 0) {
 	switch (auip->init.util.alloc_no) {
 	case ERTS_ALC_A_LONG_LIVED:
-#if HALFWORD_HEAP
-	case ERTS_ALC_A_LONG_LIVED_LOW:
-#endif
 	    return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC;
 	case ERTS_ALC_A_EHEAP:
 	    return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC;
@@ -1957,48 +1890,6 @@ alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz)
     return res;
 }
 
-#if HALFWORD_HEAP
-static ERTS_INLINE int
-alcu_is_low(ErtsAlcType_t ai)
-{
-    int is_low = 0;
-    ASSERT(erts_allctrs_info[ai].enabled);
-    ASSERT(erts_allctrs_info[ai].alloc_util);
-
-    if (!erts_allctrs_info[ai].thr_spec) {
-	Allctr_t *allctr = erts_allctrs_info[ai].extra;
-	is_low = allctr->mseg_opt.low_mem;
-    }
-    else {
-	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
-	int i;
-# ifdef DEBUG
-	int found_one = 0;
-# endif
-
-	ASSERT(tspec->enabled);
-
-	for (i = tspec->size - 1; i >= 0; i--) {
-	    Allctr_t *allctr = tspec->allctr[i];
-	    if (allctr) {
-# ifdef DEBUG
-		if (!found_one) {
-		    is_low = allctr->mseg_opt.low_mem;
-		    found_one = 1;
-		}
-		else ASSERT(is_low == allctr->mseg_opt.low_mem);
-# else
-		is_low = allctr->mseg_opt.low_mem;
-		break;
-# endif
-	    }
-	}
-	ASSERT(found_one);
-    }
-    return is_low;
-}
-#endif /* HALFWORD */
-
 static ERTS_INLINE void
 add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
 {
@@ -2028,9 +1919,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	int code;
 	int ets;
 	int maximum;
-#if HALFWORD_HEAP
-	int low;
-#endif
     } want = {0};
     struct {
 	UWord total;
@@ -2043,9 +1931,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	UWord code;
 	UWord ets;
 	UWord maximum;
-#if HALFWORD_HEAP
-	UWord low;
-#endif
     } size = {0};
     Eterm atoms[sizeof(size)/sizeof(UWord)];
     UWord *uintps[sizeof(size)/sizeof(UWord)];
@@ -2104,11 +1989,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	    atoms[length] = am_maximum;
 	    uintps[length++] = &size.maximum;
 	}
-#if HALFWORD_HEAP
-	want.low = 1;
-	atoms[length] = am_low;
-	uintps[length++] = &size.low;
-#endif
     }
     else {
 	DeclareTmpHeapNoproc(tmp_heap,2);
@@ -2202,15 +2082,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		    return am_badarg;
 		}
 		break;
-#if HALFWORD_HEAP
-	    case am_low:
-		if (!want.low) {
-		    want.low = 1;
-		    atoms[length] = am_low;
-		    uintps[length++] = &size.low;
-		}
-		break;
-#endif
 	    default:
 		UnUseTmpHeapNoproc(2);
 		return am_badarg;
@@ -2288,11 +2159,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		if (save)
 		    *save = asz;
 		size.total += asz;
-#if HALFWORD_HEAP
-		if (alcu_is_low(ai)) {
-		    size.low += asz;
-		}
-#endif
 	    }
 	}
     }
@@ -2319,7 +2185,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_PROC);
-#if !HALFWORD_HEAP
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
@@ -2329,7 +2194,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_NLINK_SH);
-#endif
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 236ee35d18..db4c30b9eb 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -2074,7 +2074,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
 
     if (!blk) {
 	blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
-#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY
+#if !ERTS_SUPER_ALIGNED_MSEG_ONLY
 	if (!blk) {
 	    /* Emergency! We couldn't create the carrier as we wanted.
 	       Try to place it in a sys_alloced sbc. */
@@ -3511,8 +3511,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
     int is_mseg = 0;
 #endif
 
-    if (HALFWORD_HEAP
-	|| (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC))
+    if ((ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC))
 	|| !allow_sys_alloc_carriers) {
 	flags |= CFLG_FORCE_MSEG;
 	flags &= ~CFLG_FORCE_SYS_ALLOC;
@@ -3749,11 +3748,6 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		DEBUG_SAVE_ALIGNMENT(new_crr);
 		return new_blk;
 	    }
-#if HALFWORD_HEAP
-	    /* Old carrier unchanged; restore stat */
-	    STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz);
-	    return NULL;
-#endif	    
 	    create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc()
 						     failed */
 	}
@@ -3942,9 +3936,6 @@ static struct {
     Eterm e;
     Eterm t;
     Eterm ramv;
-#if HALFWORD_HEAP
-    Eterm low;
-#endif
     Eterm sbct;
 #if HAVE_ERTS_MSEG
     Eterm asbcst;
@@ -4035,9 +4026,6 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(e);
 	AM_INIT(t);
 	AM_INIT(ramv);
-#if HALFWORD_HEAP
-	AM_INIT(low);
-#endif
 	AM_INIT(sbct);
 #if HAVE_ERTS_MSEG
 	AM_INIT(asbcst);
@@ -4643,9 +4631,6 @@ info_options(Allctr_t *allctr,
 		   "option e: true\n"
 		   "option t: %s\n"
 		   "option ramv: %s\n"
-#if HALFWORD_HEAP
-		   "option low: %s\n"
-#endif
 		   "option sbct: %beu\n"
 #if HAVE_ERTS_MSEG
 		   "option asbcst: %bpu\n"
@@ -4664,9 +4649,6 @@ info_options(Allctr_t *allctr,
 		   "option acul: %d\n",
 		   topt,
 		   allctr->ramv ? "true" : "false",
-#if HALFWORD_HEAP
-		   allctr->mseg_opt.low_mem ? "true" : "false",
-#endif
 		   allctr->sbc_threshold,
 #if HAVE_ERTS_MSEG
 		   allctr->mseg_opt.abs_shrink_th,
@@ -4729,9 +4711,6 @@ info_options(Allctr_t *allctr,
 	add_2tup(hpp, szp, &res,
 		 am.sbct,
 		 bld_uint(hpp, szp, allctr->sbc_threshold));
-#if HALFWORD_HEAP
-	add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false);
-#endif
 	add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false);
 	add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false));
 	add_2tup(hpp, szp, &res, am.e, am_true);
@@ -5416,11 +5395,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 	Block_t *new_blk;
 	if(IS_SBC_BLK(blk)) {
 	do_carrier_resize:
-#if HALFWORD_HEAP
-	    new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG);
-#else
 	    new_blk = resize_carrier(allctr, blk, size, CFLG_SBC);
-#endif
 	    res = new_blk ? BLK2UMEM(new_blk) : NULL;
 	}
 	else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
@@ -5714,11 +5689,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 #ifdef ERTS_SMP
     if (init->tspec || init->tpref)
 	allctr->mseg_opt.sched_spec = 1;
-#endif
-# if HALFWORD_HEAP
-    allctr->mseg_opt.low_mem = init->low_mem;
-# endif
-#endif
+#endif /* ERTS_SMP */
+#endif /* HAVE_ERTS_MSEG */
 
     allctr->name_prefix			= init->name_prefix;
     if (!allctr->name_prefix)
@@ -5875,7 +5847,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 			     CFLG_MBC
 			     | CFLG_FORCE_SIZE
 			     | CFLG_NO_CPOOL
-#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY
+#if !ERTS_SUPER_ALIGNED_MSEG_ONLY
 			     | CFLG_FORCE_SYS_ALLOC
 #endif
 			     | CFLG_MAIN_CARRIER);
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index df1f0aa65a..792f8c63ac 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -45,7 +45,6 @@ typedef struct {
     int tspec;
     int tpref;
     int ramv;
-    int low_mem;   /* HALFWORD only */
     UWord sbct;
     UWord asbcst;
     UWord rsbcst;
@@ -90,7 +89,6 @@ typedef struct {
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
-    0,			/* (bool)   low_mem: HALFWORD only               */\
     512*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
@@ -125,7 +123,6 @@ typedef struct {
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
-    0,			/* (bool)   low_mem: HALFWORD only               */\
     64*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 134aa2d396..f834802764 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2550,7 +2550,6 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1)
     }
     pb = (ProcBin *) binary_val(bin);
     if (pb->thing_word == HEADER_PROC_BIN) {
-	/* XXX:PaN - Halfword - orig_size is a long, we should handle that */
 	res = erts_make_integer((Uint) pb->val->orig_size, BIF_P);
     } else { /* heap binary */
 	res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P);
@@ -2568,7 +2567,7 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1)
 #endif
 
 static int get_need(Uint u) {
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     if (u > 0xFFFFFFFFUL) {
 	if (u > 0xFFFFFFFFFFFFUL) {
 	    if (u > 0xFFFFFFFFFFFFFFUL) {
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b44382cde8..a3ff537aa1 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -73,7 +73,7 @@ static char otp_version[] = ERLANG_OTP_VERSION;
 static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     "%s"
 				     " [erts-" ERLANG_VERSION "]"
-#if !HEAP_ON_C_STACK && !HALFWORD_HEAP
+#if !HEAP_ON_C_STACK
 				     " [no-c-stack-objects]"
 #endif
 #ifndef OTP_RELEASE
@@ -84,12 +84,8 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #endif
 #endif	
 #ifdef ARCH_64
-#if HALFWORD_HEAP
-				     " [64-bit halfword]"
-#else
 				     " [64-bit]"
 #endif
-#endif
 #ifdef ERTS_SMP
 				     " [smp:%beu:%beu]"
 #endif
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 86951f32b0..7f7cd376ac 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -100,7 +100,7 @@ Sint erts_re_set_loop_limit(Sint limit)
 
 static int term_to_int(Eterm term, int *sp)
 {
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 
     if (is_small(term)) {
 	Uint x = signed_val(term);
@@ -151,7 +151,7 @@ static int term_to_int(Eterm term, int *sp)
 
 static Eterm make_signed_integer(int x, Process *p)
 {
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     return make_small(x);
 #else
     Eterm* hp;
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index 5eca09c5a6..c4a39b8897 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -338,7 +338,7 @@ static struct {
     } w;
 } raw_unique_monotonic_integer erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
 #  define ERTS_UNIQUE_MONOTONIC_OFFSET ERTS_SINT64_MIN
 #else
 #  define ERTS_UNIQUE_MONOTONIC_OFFSET MIN_SMALL
@@ -368,7 +368,7 @@ get_unique_monotonic_integer_heap_size(Uint64 raw, int positive)
 	Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET;
 	if (IS_SSMALL(value))
 	    return 0;
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
 	return ERTS_SINT64_HEAP_SIZE(value);
 #else
 	return ERTS_UINT64_HEAP_SIZE((Uint64) value);
@@ -393,7 +393,7 @@ make_unique_monotonic_integer_value(Eterm *hp, Uint hsz, Uint64 raw, int positiv
 	if (hsz == 0)
 	    res = make_small(value);
 	else {
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
 	    res = erts_sint64_to_big(value, &hp);
 #else 
 	    res = erts_uint64_to_big((Uint64) value, &hp);
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 01734c55d7..11d83686a3 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -282,7 +282,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
 	 * Simply shift whole bytes into the result.
 	 */
 	switch (BYTE_OFFSET(n)) {
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 	case 7: w = (w << 8) | *bp++;
 	case 6: w = (w << 8) | *bp++;
 	case 5: w = (w << 8) | *bp++;
@@ -387,7 +387,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
     case 3: 
 	v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); 
 	goto big_small;
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
     case 4:
 	v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24));
 	if (!IS_USMALL(sgn, v32)) {
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 878ee32b47..f3da28b65a 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1774,15 +1774,9 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	 * (it looks like an continuation pointer), but that is will crash the
 	 * emulator if this BIF is call traced.
 	 */
-#if HALFWORD_HEAP
-	Eterm *hp = HAlloc(BIF_P, 3);
-	hp[0] = make_pos_bignum_header(2);
-	*((UWord *) (UWord) (hp+1)) = (UWord) tb;
-#else
 	Eterm *hp = HAlloc(BIF_P, 2);
 	hp[0] = make_pos_bignum_header(1);
 	hp[1] = (Eterm) tb;
-#endif
 	BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp));
     }
     else {
@@ -3652,11 +3646,8 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1)
     Eterm* ptr = big_val(cont);
     DbTable *tb = *((DbTable **) (UWord) (ptr + 1));
 
-#if HALFWORD_HEAP
-    ASSERT(*ptr == make_pos_bignum_header(2));
-#else
     ASSERT(*ptr == make_pos_bignum_header(1));
-#endif
+
     db_lock(tb, LCK_WRITE);
     trap = free_table_cont(p, tb, 0, 1);
     db_unlock(tb, LCK_WRITE);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 81b0c4465c..fc1aa05ed2 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2885,9 +2885,6 @@ Ldone:
     handle->dbterm = &b->dbterm;
     handle->flags = flags;
     handle->new_size = b->dbterm.size;
-#if HALFWORD_HEAP
-    handle->abs_vec = NULL;
-#endif
     handle->lck = lck;
     return 1;
 }
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 465aa566ad..c21eac6f25 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -2589,9 +2589,6 @@ db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj,
     handle->flags = flags;
     handle->bp = (void**) pp;
     handle->new_size = (*pp)->dbterm.size;
-#if HALFWORD_HEAP
-    handle->abs_vec = NULL;
-#endif
     return 1;
 }
 
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index dab357a079..0086aa8121 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -239,11 +239,7 @@ typedef enum {
     matchCall2,
     matchCall3,
     matchPushV,
-#if HALFWORD_HEAP
-    matchPushVGuard,    /* First guard-only variable reference */
-#endif
-    matchPushVResult, /* First variable reference in result, or (if HALFWORD)
-                         in guard if also referenced in result */
+    matchPushVResult, /* First variable reference in result */
     matchPushExpr, /* Push the whole expression we're matching ('$_') */
     matchPushArrayAsList, /* Only when parameter is an Array and 
 			     not an erlang term  (DCOMP_TRACE) */
@@ -310,9 +306,6 @@ DMC_DECLARE_STACK_TYPE(unsigned);
 typedef struct DMCVariable {
     int is_bound;
     int is_in_body;
-#if HALFWORD_HEAP
-    int first_guard_label;  /* to maybe change from PushVGuard to PushVResult */
-#endif
 } DMCVariable;
 
 typedef struct DMCHeap {
@@ -415,7 +408,7 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
 	else {
 	    int i;
 	    for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) {
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef;
 #else
 		mpsp->default_heap[i] = (Eterm) 0xdeadbeef;
@@ -1755,60 +1748,6 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
     return ret;
 }
 
-
-#if HALFWORD_HEAP
-struct heap_checkpoint_t
-{
-    Process *p;
-    Eterm* htop;
-    ErlHeapFragment* mbuf;
-    unsigned used_size;
-    ErlOffHeap off_heap;
-};
-
-static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp)
-{
-    hcp->p = p;
-    hcp->htop = HEAP_TOP(p);
-    hcp->mbuf = MBUF(p);
-    hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0;
-    hcp->off_heap = MSO(p);
-}
-
-static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp)
-{
-    struct erl_off_heap_header* oh = MSO(hcp->p).first;
-
-    if (oh != hcp->off_heap.first) {
-	ASSERT(oh != NULL);
-	if (hcp->off_heap.first) {
-	    while (oh->next != hcp->off_heap.first) {
-		oh = oh->next;
-	    }
-	    oh->next = NULL;
-	}
-	erts_cleanup_offheap(&MSO(hcp->p));
-	MSO(hcp->p) = hcp->off_heap;
-    }
-    if (MBUF(hcp->p) != hcp->mbuf) {
-	ErlHeapFragment* hf = MBUF(hcp->p);
-	ASSERT(hf != NULL);
-	if (hcp->mbuf) {
-	    while (hf->next != hcp->mbuf) {
-		hf = hf->next;
-	    }
-	    hf->next = NULL;
-	}
-	free_message_buffer(MBUF(hcp->p));
-	MBUF(hcp->p) = hcp->mbuf;
-    }
-    if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) {
-	hcp->mbuf->used_size = hcp->used_size;
-    }
-    HEAP_TOP(hcp->p) = hcp->htop;
-}
-#endif /* HALFWORD_HEAP */
-
 static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base)
 {
     if (!is_immed(term)) {
@@ -1855,17 +1794,12 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
     Eterm bif_args[3];
     int fail_label;
     int atomic_trace;
-#if HALFWORD_HEAP
-    struct heap_checkpoint_t c_p_checkpoint = {};
-#endif
 #ifdef DMC_DEBUG
     Uint *heap_fence;
     Uint *stack_fence;
     Uint save_op;
 #endif /* DMC_DEBUG */
 
-    ASSERT(base==NULL || HALFWORD_HEAP);
-
     mpsp = get_match_pseudo_process(c_p, prog->heap_size);
     psp = &mpsp->process;
 
@@ -1916,11 +1850,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
 			     do_catch != 0 */
 
     *return_flags = 0U;
-
     variables = mpsp->u.variables;
-#if HALFWORD_HEAP
-    c_p_checkpoint.p = NULL;
-#endif
 
 restart:
     ep = &term;
@@ -1931,7 +1861,6 @@ restart:
     fail_label = -1;
     build_proc = psp;
     esdp->current_process = psp;
-    ASSERT_HALFWORD(!c_p_checkpoint.p);
 
 #ifdef DEBUG
     ASSERT(variables == mpsp->u.variables);
@@ -2236,31 +2165,10 @@ restart:
 	    esp -= 2;
 	    esp[-1] = t;
 	    break;
-
-	#if HALFWORD_HEAP
-	case matchPushVGuard:
-	    if (!base) goto case_matchPushV;
-	    /* Build NULL-based copy on pseudo heap for easy disposal */
-	    n = *pc++;
-	    ASSERT(is_value(variables[n].term));
-	    ASSERT(!variables[n].proc);
-	    variables[n].term = copy_object_rel(psp, variables[n].term, base);
-	    *esp++ = variables[n].term;
-	    #ifdef DEBUG
-	    variables[n].proc = psp;
-	    variables[n].base = NULL;
-	    #endif
-	    break;
-	#endif
 	case matchPushVResult:
 	    if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV;
 
 	    /* Build (NULL-based) copy on callers heap */
-	#if HALFWORD_HEAP
-	    if (!do_catch && !c_p_checkpoint.p) {
-		heap_checkpoint_init(c_p, &c_p_checkpoint);
-	    }
-        #endif
 	    n = *pc++;
 	    ASSERT(is_value(variables[n].term));
 	    ASSERT(!variables[n].proc);
@@ -2299,7 +2207,6 @@ restart:
 	    }
 	    break;
 	case matchPushArrayAsList:
-	    ASSERT_HALFWORD(base == NULL);
 	    n = arity; /* Only happens when 'term' is an array */
 	    tp = termp;
 	    ehp = HAllocX(build_proc, n*2, HEAP_XTRA);
@@ -2315,7 +2222,6 @@ restart:
 	    break;
 	case matchPushArrayAsListU:
 	    /* This instruction is NOT efficient. */
-	    ASSERT_HALFWORD(base == NULL);
 	    *esp++  = dpm_array_to_list(build_proc, termp, arity);
 	    break;
 	case matchTrue:
@@ -2619,13 +2525,6 @@ restart:
 	}
     }
 fail:
-#if HALFWORD_HEAP
-    if (c_p_checkpoint.p) {
-	/* Dispose garbage built by guards on caller heap */
-	heap_checkpoint_revert(&c_p_checkpoint);
-	c_p_checkpoint.p = NULL;
-    }
-#endif
     *return_flags = 0U;
     if (fail_label >= 0) { /* We failed during a "TryMeElse",
 			      lets restart, with the next match
@@ -2788,13 +2687,6 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position)
 {
     Eterm elem = handle->dbterm->tpl[position];
     if (!is_header(elem)) {
-#if HALFWORD_HEAP
-	if (!is_immed(elem)
-	    && !handle->tb->common.compress
-	    && !(handle->abs_vec && handle->abs_vec[position])) {
-	    return rterm2wterm(elem, handle->dbterm->tpl);
-	}
-#endif
 	return elem;
     }
 
@@ -2822,9 +2714,6 @@ void db_do_update_element(DbUpdateHandle* handle,
     Eterm* oldp;
     Uint newval_sz;
     Uint oldval_sz;
-#if HALFWORD_HEAP
-    Eterm* old_base;
-#endif
 
     if (is_both_immed(newval,oldval)) {
 	handle->dbterm->tpl[position] = newval;
@@ -2841,15 +2730,8 @@ void db_do_update_element(DbUpdateHandle* handle,
 						       handle->dbterm);
 	    handle->flags |= DB_MUST_RESIZE;
 	    oldval = handle->dbterm->tpl[position];
-        #if HALFWORD_HEAP
-	    old_base = NULL;
-	#endif
 	}
 	else {
-	#if HALFWORD_HEAP
-	    ASSERT(!handle->abs_vec);
-	    old_base = handle->dbterm->tpl;
-	#endif
 	    if (is_boxed(newval)) {
 		newp = boxed_val(newval);
 		switch (*newp & _TAG_HEADER_MASK) {
@@ -2879,13 +2761,6 @@ void db_do_update_element(DbUpdateHandle* handle,
 	    }
 	}
     }
-#if HALFWORD_HEAP
-    else {
-	old_base = (handle->tb->common.compress
-		    || (handle->abs_vec && handle->abs_vec[position])) ?
-	    NULL : handle->dbterm->tpl;
-    }
-#endif
     /* Not possible for simple memcpy or dbterm is already non-contiguous, */
     /* need to realloc... */
 
@@ -2900,19 +2775,6 @@ both_size_set:
     /* write new value in old dbterm, finalize will make a flat copy */
     handle->dbterm->tpl[position] = newval;
     handle->flags |= DB_MUST_RESIZE;
-
-#if HALFWORD_HEAP
-    if (old_base && newval_sz > 0) {
-	ASSERT(!handle->tb->common.compress);
-	if (!handle->abs_vec) {
-	    int i = header_arity(handle->dbterm->tpl[0]);
-	    handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char));
-	    sys_memset(handle->abs_vec, 0, i+1);
-	    /* abs_vec[0] not used */
-	}
-	handle->abs_vec[position] = 1;
-    }
-#endif
 }
 
 static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old,
@@ -3160,26 +3022,6 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset)
 
 	tmp_offheap.first = NULL;
 
-    #if HALFWORD_HEAP
-	if (handle->abs_vec) {
-	    int i, arity = header_arity(handle->dbterm->tpl[0]);
-
-	    top[0] = tpl[0];
-	    top += arity + 1;
-	    for (i=1; i<=arity; i++) {
-		Eterm* src_base = handle->abs_vec[i] ? NULL : tpl;
-
-		newDbTerm->tpl[i] = copy_struct_rel(tpl[i],
-						    size_object_rel(tpl[i],src_base),
-						    &top, &tmp_offheap, src_base,
-						    newDbTerm->tpl);
-	    }
-	    newDbTerm->first_oh = tmp_offheap.first;
-	    ASSERT((byte*)top <= (newp + alloc_sz));
-	    erts_free(ERTS_ALC_T_TMP, handle->abs_vec);
-	}
-	else
-    #endif /* HALFWORD_HEAP */
 	{
 	    copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top,
 			    &tmp_offheap, tpl, top);
@@ -3599,26 +3441,10 @@ static DMCRet dmc_one_term(DMCContext *context,
 	{
 	    Eterm* ref_val = internal_ref_val(c);
 	    DMC_PUSH(*text, matchEqRef);
-#if HALFWORD_HEAP
-	    {
-		union {
-		    UWord u;
-		    Uint t[2];
-		} fiddle;
-		ASSERT(thing_arityval(ref_val[0]) == 3);
-		fiddle.t[0] = ref_val[0];
-		fiddle.t[1] = ref_val[1];
-		DMC_PUSH(*text, fiddle.u);
-		fiddle.t[0] = ref_val[2];
-		fiddle.t[1] = ref_val[3];
-		DMC_PUSH(*text, fiddle.u);
-	    }
-#else
 	    n = thing_arityval(ref_val[0]);
 	    for (i = 0; i <= n; ++i) {
 		DMC_PUSH(*text, ref_val[i]);
 	    }
-#endif
 	    break;
 	}
 	case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
@@ -3627,52 +3453,18 @@ static DMCRet dmc_one_term(DMCContext *context,
 	    Eterm* bval = big_val(c);
 	    n = thing_arityval(bval[0]);
 	    DMC_PUSH(*text, matchEqBig);
-#if HALFWORD_HEAP
-	    {
-		union {
-		    UWord u;
-		    Uint t[2];
-		} fiddle;
-		ASSERT(n >= 1);
-		fiddle.t[0] = bval[0];
-		fiddle.t[1] = bval[1];
-		DMC_PUSH(*text, fiddle.u);
-		for (i = 2; i <= n; ++i) {
-		    fiddle.t[0] = bval[i];
-		    if (++i <= n) {
-			fiddle.t[1] = bval[i];
-		    } else {
-			fiddle.t[1] = (Uint) 0;
-		    }
-		    DMC_PUSH(*text, fiddle.u);
-		}
-	    }
-#else
 	    for (i = 0; i <= n; ++i) {
 		DMC_PUSH(*text, (Uint) bval[i]);
 	    }
-#endif
 	    break;
 	}
 	case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
 	    DMC_PUSH(*text,matchEqFloat);
-#if HALFWORD_HEAP
-	    {
-		union {
-		    UWord u;
-		    Uint t[2];
-		} fiddle;
-		fiddle.t[0] = float_val(c)[1];
-		fiddle.t[1] = float_val(c)[2];
-		DMC_PUSH(*text, fiddle.u);
-	    }
-#else
 	    DMC_PUSH(*text, (Uint) float_val(c)[1]);
 #ifdef ARCH_64
 	    DMC_PUSH(*text, (Uint) 0);
 #else
 	    DMC_PUSH(*text, (Uint) float_val(c)[2]);
-#endif
 #endif
 	    break;
 	default: /* BINARY, FUN, VECTOR, or EXTERNAL */
@@ -3997,24 +3789,8 @@ static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap,
     MatchOps instr = matchPushV;
 
     ASSERT(n < heap->vars_used && v->is_bound);
-    if (context->is_guard) {
-	#if HALFWORD_HEAP
-	if (!v->first_guard_label) {
-	    v->first_guard_label = DMC_STACK_NUM(*text);
-	    ASSERT(v->first_guard_label);
-	    instr = matchPushVGuard; /* may be changed to PushVResult below */
-	}
-	#endif
-    }
-    else { /* body */
-	#if HALFWORD_HEAP
-	if (v->first_guard_label) {
-	    /* Avoid double-copy, copy to result heap at first encounter in guard */
-	    DMC_POKE(*text, v->first_guard_label, matchPushVResult);
-	    v->is_in_body = 1;
-	}
-	#endif
-	if (!v->is_in_body) {
+    if (!context->is_guard) {
+        if(!v->is_in_body) {
 	    instr = matchPushVResult;
 	    v->is_in_body = 1;
 	}
@@ -5450,7 +5226,7 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
 	obj = db_alloc_tmp_uncompressed(tb, obj);
 	base = NULL;
     }
-    else base = HALFWORD_HEAP ? obj->tpl : NULL;
+    else base = NULL;
 
     res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0,
 			ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
@@ -5573,7 +5349,7 @@ void db_match_dis(Binary *bp)
 			first = 0;
 		    else
 			erts_printf(", ");
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		    erts_printf("0x%016bex", rt->data.ui[ri]);
 #else
 		    erts_printf("0x%08bex", rt->data.ui[ri]);
@@ -5597,7 +5373,7 @@ void db_match_dis(Binary *bp)
 			first = 0;
 		    else
 			erts_printf(", ");
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		    erts_printf("0x%016bex", *et);
 #else
 		    erts_printf("0x%08bex", *et);
@@ -5720,13 +5496,6 @@ void db_match_dis(Binary *bp)
 	    ++t;
 	    erts_printf("PushV\t%beu\n", n);
 	    break;
-    #if HALFWORD_HEAP
-	case matchPushVGuard:
-	    n = (Uint) *++t;
-	    ++t;
-	    erts_printf("PushVGuard\t%beu\n", n);
-	    break;
-    #endif
 	case matchPushVResult:
 	    n = (Uint) *++t;
 	    ++t;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 1ccdc0305b..c2128888c5 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -90,9 +90,6 @@ typedef struct {
     Uint new_size;
     int flags;
     void* lck;
-#if HALFWORD_HEAP
-    unsigned char* abs_vec;  /* [i] true if dbterm->tpl[i] is absolute Eterm */
-#endif
 } DbUpdateHandle;
 
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d2604f1595..6c335a9fe6 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -128,7 +128,7 @@ static void disallow_heap_frag_ref_in_old_heap(Process* p);
 static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj);
 #endif
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 # define MAX_HEAP_SIZES 154
 #else
 # define MAX_HEAP_SIZES 59
@@ -147,26 +147,10 @@ typedef struct {
     erts_smp_atomic32_t refc;
 } ErtsGCInfoReq;
 
-#if !HALFWORD_HEAP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
-				 ErtsGCInfoReq,
-				 5,
-				 ERTS_ALC_T_GC_INFO_REQ)
-#else
-static ERTS_INLINE ErtsGCInfoReq *
-gcireq_alloc(void)
-{
-    return erts_alloc(ERTS_ALC_T_GC_INFO_REQ,
-		      sizeof(ErtsGCInfoReq));
-}
-
-static ERTS_INLINE void
-gcireq_free(ErtsGCInfoReq *ptr)
-{
-    erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr);
-}
-#endif
-
+                                 ErtsGCInfoReq,
+                                 5,
+                                 ERTS_ALC_T_GC_INFO_REQ)
 /*
  * Initialize GC global data.
  */
@@ -208,7 +192,7 @@ erts_init_gc(void)
     }
 
 
-    /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] (and halfword)
+    /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words]
      * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words]
      */
 
@@ -232,10 +216,7 @@ erts_init_gc(void)
       init_gc_info(&esdp->gc_info);
     }
 
-#if !HALFWORD_HEAP
     init_gcireq_alloc();
-#endif
-
 }
 
 /*
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index d9c3b0dcf4..5c209a4af2 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -91,11 +91,6 @@ const int etp_arch_bits = 32;
 #else
 # error "Not 64-bit, nor 32-bit arch"
 #endif
-#if HALFWORD_HEAP
-const int etp_halfword = 1;
-#else
-const int etp_halfword = 0;
-#endif
 #ifdef HIPE
 const int etp_hipe = 1;
 #else
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 3b3b247020..84bee976ff 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -156,9 +156,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"instr",				NULL			},
     {	"alcu_allocator",			"index"			},
     {	"mseg",					NULL			},
-#if HALFWORD_HEAP
-    {	"pmmap",				NULL			},
-#endif
 #ifdef ERTS_SMP
     {	"port_task_pre_alloc_lock",		"address"		},
     {	"proclist_pre_alloc_lock",		"address"		},
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index a91e36e3c5..6d496ea5ee 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -170,11 +170,7 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
  */
 
 const Eterm *
-#if HALFWORD_HEAP
-erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base)
-#else
 erts_maps_get(Eterm key, Eterm map)
-#endif
 {
     Uint32 hx;
     if (is_flatmap_rel(map, map_base)) {
@@ -1993,11 +1989,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
 }
 
 const Eterm *
-#if HALFWORD_HEAP
-erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base)
-#else
 erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
-#endif
 {
     Eterm *ptr, hdr, *res;
     Uint ix, lvl = 0;
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index c391de3f11..2f7c55829f 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -105,22 +105,12 @@ Eterm  erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint
 					 Eterm k, Eterm v);
 
 const Eterm *
-#if HALFWORD_HEAP
-erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base);
-#  define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL)
-#else
 erts_maps_get(Eterm key, Eterm map);
-#  define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B)
-#endif
+#define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B)
 
 const Eterm *
-#if HALFWORD_HEAP
-erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base);
-#  define erts_hashmap_get(Hx, K, M) erts_hashmap_get_rel(Hx, K, M, NULL)
-#else
 erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
-#  define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M)
-#endif
+#define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M)
 
 /* hamt nodes v2.0
  *
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index fbdf3fb0e2..f37b430d27 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -32,9 +32,6 @@ struct external_thing_;
 struct erl_off_heap_header {
     Eterm thing_word;
     Uint size;
-#if HALFWORD_HEAP
-    void* dummy_ptr_padding__;
-#endif
     struct erl_off_heap_header* next;
 };
 
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 211b1a0090..0a4b18b748 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -255,7 +255,7 @@ extern ErtsPTab erts_port;
  * Refs                                                                    *
 \*                                                                         */
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 
 #define internal_ref_no_of_numbers(x)					\
   (internal_ref_data((x))[0])
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 2fb790b953..62a44f7129 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1207,23 +1207,11 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 			break;
 		    }
 		if (insert_bin) {
-#if HALFWORD_HEAP
-		    UWord val = (UWord) u.pb->val;
-		    DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */
-#else
 		    DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE);
-#endif
 		    Uint *hp = &id_heap[0];
 		    InsertedBin *nib;
-#if HALFWORD_HEAP
-		    int actual_need = BIG_UWORD_HEAP_SIZE(val);
-		    ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2));
-		    UseTmpHeapNoproc(actual_need);
-		    a.id = erts_bld_uword(&hp, NULL, (UWord) val);
-#else
 		    UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
 		    a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val);
-#endif
 		    erts_match_prog_foreach_offheap(u.pb->val,
 						    insert_offheap2,
 						    (void *) &a);
@@ -1231,11 +1219,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 		    nib->bin_val = u.pb->val;
 		    nib->next = inserted_bins;
 		    inserted_bins = nib;
-#if HALFWORD_HEAP
-		    UnUseTmpHeapNoproc(actual_need);
-#else
 		    UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
-#endif
 		}
 	    }		
 	    break;
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 694ac84232..64278d2ea0 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -66,7 +66,7 @@
 #define ERTS_DE_QFLGS_ALL			(ERTS_DE_QFLG_BUSY \
 						 | ERTS_DE_QFLG_EXIT)
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL)
 #else
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713)
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 267c0b3ff4..95b3572372 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -261,7 +261,7 @@ static char *format_binary(Uint16 x, char *b) {
 
 static int
 print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
-	   Eterm* obj_base) /* ignored if !HALFWORD_HEAP */
+	   Eterm* obj_base)
 {
     DECLARE_WSTACK(s);
     int res;
@@ -344,11 +344,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    PRINT_CHAR(res, fn, arg, '>');
 	    goto L_done;
 	}
-#if HALFWORD_HEAP
-	wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base);
-#else
 	wobj = (Wterm)obj;
-#endif
 	switch (tag_val_def(wobj)) {
 	case NIL_DEF:
 	    PRINT_STRING(res, fn, arg, "[]");
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7b3d12ce09..a21c1e5582 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -119,7 +119,7 @@
 #define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ))
 
 #ifdef DEBUG
-#  if defined(ARCH_64) && !HALFWORD_HEAP
+#  if defined(ARCH_64)
 #    define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
     (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4)))
 #  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
@@ -971,25 +971,10 @@ typedef struct {
     erts_smp_atomic32_t refc;
 } ErtsSchedWallTimeReq;
 
-#if !HALFWORD_HEAP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq,
-				 ErtsSchedWallTimeReq,
-				 5,
-				 ERTS_ALC_T_SCHED_WTIME_REQ)
-#else
-static ERTS_INLINE ErtsSchedWallTimeReq *
-swtreq_alloc(void)
-{
-    return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ,
-		      sizeof(ErtsSchedWallTimeReq));
-}
-
-static ERTS_INLINE void
-swtreq_free(ErtsSchedWallTimeReq *ptr)
-{
-    erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr);
-}
-#endif
+                                 ErtsSchedWallTimeReq,
+                                 5,
+                                 ERTS_ALC_T_SCHED_WTIME_REQ)
 
 static void
 reply_sched_wall_time(void *vswtrp)
@@ -5797,9 +5782,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 #endif
 
     init_misc_aux_work();
-#if !HALFWORD_HEAP
     init_swtreq_alloc();
-#endif
 
     erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
     debug_wait_completed_flags = 0;
@@ -8286,7 +8269,7 @@ add_pend_suspend(Process *suspendee,
 					 sizeof(ErtsPendingSuspend));
     psp->next = NULL;
 #ifdef DEBUG
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead;
 #else
     psp->end = (ErtsPendingSuspend *) 0xdeaddead;
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 90e35151b0..46900edfc7 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -21,33 +21,12 @@
 #ifndef __ERL_TERM_H
 #define __ERL_TERM_H
 
-#include "sys.h" /* defines HALFWORD_HEAP */
-
 typedef UWord Wterm;  /* Full word terms */
 
-#if HALFWORD_HEAP
-#  define HEAP_ON_C_STACK 0
-#  if HALFWORD_ASSERT
-#    ifdef ET_DEBUG
-#      undef ET_DEBUG
-#    endif
-#    define ET_DEBUG 1
-#  endif
-#  if 1
-#    define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL
-#    define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer))
-#    define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm))
-#  else
-#    define CHECK_POINTER_MASK 0x0UL
-#    define COMPRESS_POINTER(AnUint) (AnUint)
-#    define EXPAND_POINTER(APointer) (APointer)
-#  endif
-#else
-#  define HEAP_ON_C_STACK 1
-#  define CHECK_POINTER_MASK 0x0UL
-#  define COMPRESS_POINTER(AnUint) (AnUint)
-#  define EXPAND_POINTER(APointer) (APointer)
-#endif
+#define HEAP_ON_C_STACK 1
+#define CHECK_POINTER_MASK 0x0UL
+#define COMPRESS_POINTER(AnUint) (AnUint)
+#define EXPAND_POINTER(APointer) (APointer)
 
 struct erl_node_; /* Declared in erl_node_tables.h */
 
@@ -190,13 +169,10 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 
 
 /* boxed object access methods */
-#if HALFWORD_HEAP
-#define _is_taggable_pointer(x)	 (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0)
-#define _boxed_precond(x)        (is_boxed(x))
-#else
+
 #define _is_taggable_pointer(x)	 (((Uint)(x) & 0x3) == 0)
 #define  _boxed_precond(x)       (is_boxed(x))
-#endif
+
 #define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
 #define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED)
 _ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*)
@@ -226,11 +202,7 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
 #define is_list(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST)
 #define is_not_list(x)		(!is_list((x)))
 #endif
-#if HALFWORD_HEAP
 #define _list_precond(x)        (is_list(x))
-#else
-#define _list_precond(x)       (is_list(x))
-#endif
 #define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST))
 _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define list_val(x)		_ET_APPLY(list_val,(x))
@@ -250,7 +222,7 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs)  /*XXX*/
 
 /* fixnum ("small") access methods */
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 #define SMALL_BITS	(64-4)
 #define SMALL_DIGITS	(17)
 #else
@@ -396,11 +368,7 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm)
 _ET_DECLARE_CHECKED(Eterm*,export_val,Wterm)
 #define export_val(x)	_ET_APPLY(export_val,(x))
 #define is_export_header(x)	((x) == HEADER_EXPORT)
-#if HALFWORD_HEAP
-#define HEADER_EXPORT   _make_header(2,_TAG_HEADER_EXPORT)
-#else
 #define HEADER_EXPORT   _make_header(1,_TAG_HEADER_EXPORT)
-#endif
 
 /* bignum access methods */
 #define make_pos_bignum_header(sz)	_make_header((sz),_TAG_HEADER_POS_BIG)
@@ -424,7 +392,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Wterm)
 #define big_val(x)		_ET_APPLY(big_val,(x))
 
 /* flonum ("float") access methods */
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 #define HEADER_FLONUM   _make_header(1,_TAG_HEADER_FLOAT)
 #else
 #define HEADER_FLONUM	_make_header(2,_TAG_HEADER_FLOAT)
@@ -445,12 +413,12 @@ typedef union float_def
     byte   fb[sizeof(ieee754_8)];
     Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)];
     Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)];
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     Uint   fdw;
 #endif
 } FloatDef;
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 
 #define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1)
 
@@ -727,7 +695,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm)
 #define ERTS_MAX_REF_NUMBERS	3
 #define ERTS_REF_NUMBERS	ERTS_MAX_REF_NUMBERS
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 #  define ERTS_REF_WORDS	(ERTS_REF_NUMBERS/2 + 1)
 #  define ERTS_REF_32BIT_WORDS  (ERTS_REF_NUMBERS+1)
 #else
@@ -749,7 +717,7 @@ typedef struct {
 #define make_ref_thing_header(DW) \
   _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF)
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 
 /*
  * Ref layout on a 64-bit little endian machine:
@@ -1159,17 +1127,9 @@ extern unsigned tag_val_def(Wterm);
 #define FLOAT_BIG 	_NUMBER_CODE(FLOAT_DEF,BIG_DEF)
 #define FLOAT_FLOAT	_NUMBER_CODE(FLOAT_DEF,FLOAT_DEF)
 
-#if HALFWORD_HEAP
-#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE)))
-#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE))
-
-#else /* HALFWORD_HEAP */
-
 #define ptr2rel(PTR,BASE) (PTR)
 #define rterm2wterm(REL,BASE) (REL)
 
-#endif /* !HALFWORD_HEAP */
-
 #define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE))
 #define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE))
 #define make_fun_rel make_boxed_rel
@@ -1216,25 +1176,7 @@ extern unsigned tag_val_def(Wterm);
 
 #define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE))
 
-
-#if HALFWORD_HEAP
-ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
-{
-    /* If bases differ, assume a and b are on different "heaps",
-       ie can only be same if immed */
-    ASSERT(a_base == b_base || is_immed(a) || is_immed(b)
-	   || rterm2wterm(a,a_base) != rterm2wterm(b,b_base));
-
-    return a == b && (a_base == b_base || is_immed(a));
-}
-#endif
-
-#else /* !HALFWORD_HEAP */
 #define is_same(A,A_BASE,B,B_BASE) ((A)==(B))
-#endif
 
 #endif	/* __ERL_TERM_H */
 
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 6dab3bf297..1732579d04 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -157,24 +157,11 @@ void erts_init_utils_mem(void);
 erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint);
 void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *);
 
-#if HALFWORD_HEAP
-int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
-#  define eq(A,B) eq_rel(A,NULL,B,NULL)
-#else
 int eq(Eterm, Eterm);
-#  define eq_rel(A,A_BASE,B,B_BASE) eq(A,B)
-#endif
+#define eq_rel(A,A_BASE,B,B_BASE) eq(A,B)
 
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
-#if HALFWORD_HEAP
-Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int, int);
-#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0,0)
-#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1,0)
-#define CMP(A,B)                         erts_cmp_rel_opt(A,NULL,B,NULL,0,0)
-#define CMP_TERM(A,B)                    erts_cmp_rel_opt(A,NULL,B,NULL,1,0)
-#define CMP_EQ_ONLY(A,B)                 erts_cmp_rel_opt(A,NULL,B,NULL,0,1)
-#else
 Sint erts_cmp(Eterm, Eterm, int, int);
 Sint cmp(Eterm a, Eterm b);
 #define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp(A,B,0,0)
@@ -182,7 +169,6 @@ Sint cmp(Eterm a, Eterm b);
 #define CMP(A,B)                         erts_cmp(A,B,0,0)
 #define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
 #define CMP_EQ_ONLY(A,B)                 erts_cmp(A,B,0,1)
-#endif
 
 #define cmp_lt(a,b)          (CMP((a),(b)) <  0)
 #define cmp_le(a,b)          (CMP((a),(b)) <= 0)
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 8948ca2ea9..2d4141ac8d 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -117,12 +117,8 @@
 #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p))
 
 #if defined(DEBUG) || defined(CHECK_FOR_HOLES)
-#if HALFWORD_HEAP
-# define ERTS_HOLE_MARKER (0xdeadbeef)
-#else
 # define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef)
-#endif
-#endif
+#endif /* egil: 32-bit ? */
 
 /*
  * Allocate heap memory on the ordinary heap, NEVER in a heap
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c6d7e3fcc5..f7b372d294 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2339,10 +2339,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
     Eterm val;
     FloatDef f;
     Sint r = 0;
-#if HALFWORD_HEAP
-    UWord wobj;
-#endif
-
 
     if (ctx) {
 	WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
@@ -2362,11 +2358,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 
  outer_loop:
     while (!WSTACK_ISEMPTY(s)) {
-#if HALFWORD_HEAP
-	obj = (Eterm) (wobj = WSTACK_POP(s));
-#else
 	obj = WSTACK_POP(s);
-#endif
+
 	switch (val = WSTACK_POP(s)) {
 	case ENC_TERM:
 	    break;
@@ -2384,11 +2377,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    break;
 	case ENC_PATCH_FUN_SIZE:
 	    {
-#if HALFWORD_HEAP
-		byte* size_p = (byte *) wobj;
-#else
 		byte* size_p = (byte *) obj;
-#endif
 		put_int32(ep - size_p, size_p);
 	    }
 	    goto outer_loop;
@@ -2435,21 +2424,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	case ENC_LAST_ARRAY_ELEMENT:
 	    /* obj is the tuple */
 	    {
-#if HALFWORD_HEAP
-		Eterm* ptr = (Eterm *) wobj;
-#else
 		Eterm* ptr = (Eterm *) obj;
-#endif
 		obj = *ptr;
 	    }
 	    break;
 	default:		/* ENC_LAST_ARRAY_ELEMENT+1 and upwards */
 	    {
-#if HALFWORD_HEAP
-		Eterm* ptr = (Eterm *) wobj;
-#else
 		Eterm* ptr = (Eterm *) obj;
-#endif
 		obj = *ptr++;
 		WSTACK_PUSH2(s, val-1, (UWord)ptr);
 	    }
@@ -3049,7 +3030,7 @@ dec_term(ErtsDistExternal *edep,
 		Sint sn = get_int32(ep);
 
 		ep += 4;
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		*objp = make_small(sn);
 #else
 		if (MY_IS_SSMALL(sn)) {
@@ -3371,7 +3352,7 @@ dec_term_atom_common:
 		    RefThing *rtp = (RefThing *) hp;
 		    ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE);
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		    hp += REF_THING_HEAD_SIZE + ref_words/2 + 1;
 		    rtp->header = make_ref_thing_header(ref_words/2 + 1);
 #else
@@ -3382,13 +3363,13 @@ dec_term_atom_common:
 		}
 		else {
 		    ExternalThing *etp = (ExternalThing *) hp;
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		    hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1;
 #else
 		    hp += EXTERNAL_THING_HEAD_SIZE + ref_words;
 #endif
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		    etp->header = make_external_ref_header(ref_words/2 + 1);
 #else
 		    etp->header = make_external_ref_header(ref_words);
@@ -3401,7 +3382,7 @@ dec_term_atom_common:
 		    ref_num = &(etp->data.ui32[0]);
 		}
 
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		*(ref_num++) = ref_words /* 32-bit arity */;
 #endif
 		ref_num[0] = r0;
@@ -3409,7 +3390,7 @@ dec_term_atom_common:
 		    ref_num[i] = get_int32(ep);
 		    ep += 4;
 		}
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		if ((1 + ref_words) % 2)
 		    ref_num[ref_words] = 0;
 #endif
@@ -3561,12 +3542,7 @@ dec_term_atom_common:
                 }
 		*objp = make_export(hp);
 		*hp++ = HEADER_EXPORT;
-#if HALFWORD_HEAP
-		*((UWord *) (UWord) hp) =  (UWord) erts_export_get_or_make_stub(mod, name, arity);
-		hp += 2;
-#else
 		*hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity);
-#endif
 		break;
 	    }
 	    break;
@@ -4192,11 +4168,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	case EXPORT_DEF:
 	    {
 		Export* ep = *((Export **) (export_val(obj) + 1));
-#if HALFWORD_HEAP
-		result += 2;
-#else
 		result += 1;
-#endif
 		result += encode_size_struct2(acmp, ep->code[0], dflags);
 		result += encode_size_struct2(acmp, ep->code[1], dflags);
 		result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags);
@@ -4311,7 +4283,7 @@ init_done:
 	switch (tag) {
 	case INTEGER_EXT:
 	    SKIP(4);
-#if !defined(ARCH_64) || HALFWORD_HEAP
+#if !defined(ARCH_64)
 	    heap_size += BIG_UINT_HEAP_SIZE;
 #endif
 	    break;
@@ -4400,7 +4372,7 @@ init_done:
 		ep += 2;
 		atom_extra_skip = 1 + 4*id_words;
 		/* In case it is an external ref */
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 		heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1;
 #else
 		heap_size += EXTERNAL_THING_HEAD_SIZE + id_words;
@@ -4486,11 +4458,7 @@ init_done:
 	    break;
 	case EXPORT_EXT:
 	    terms += 3;
-#if HALFWORD_HEAP
-	    heap_size += 3;
-#else
 	    heap_size += 2;
-#endif
 	    break;
 	case NEW_FUN_EXT:
 	    {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ec9296d034..1d2d75bc1e 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -310,9 +310,6 @@ typedef union {
 typedef struct proc_bin {
     Eterm thing_word;		/* Subtag REFC_BINARY_SUBTAG. */
     Uint size;			/* Binary size in bytes. */
-#if HALFWORD_HEAP
-    void* dummy_ptr_padding__;
-#endif
     struct erl_off_heap_header *next;
     Binary *val;		/* Pointer to Binary structure. */
     byte *bytes;		/* Pointer to the actual data bytes. */
@@ -959,28 +956,14 @@ void erl_error(char*, va_list);
 /* copy.c */
 Eterm copy_object(Eterm, Process*);
 
-#if HALFWORD_HEAP
-Uint size_object_rel(Eterm, Eterm*);
-#  define size_object(A) size_object_rel(A,NULL)
-
-Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base);
-#  define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL)
-
-Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base);
-#  define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL)
-
-#else /* !HALFWORD_HEAP */
-
 Uint size_object(Eterm);
-#  define size_object_rel(A,B) size_object(A)
+#define size_object_rel(A,B) size_object(A)
 
 Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
-#  define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH)
+#define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH)
 
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
-#  define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D)
-
-#endif
+#define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D)
 
 
 void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
@@ -1180,7 +1163,7 @@ void bin_write(int, void*, byte*, size_t);
 int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */
 
 struct Sint_buf {
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     char s[22];
 #else
     char s[12];
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 900616c981..c365b8859b 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1105,7 +1105,7 @@ io_list_vec_len(Eterm obj, int* vsize, Uint* csize,
     Uint p_v_size = 0;
     Uint p_c_size = 0;
     Uint p_in_clist = 0;
-    Uint total; /* Uint due to halfword emulator */
+    Uint total;
 
     goto L_jump_start;  /* avoid a push */
 
@@ -5248,25 +5248,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    break;
 	case ERL_DRV_INT:  /* signed int argument */
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
-#if HALFWORD_HEAP
-	    erts_bld_sint64(NULL, &need, (Sint64)ptr[0]);
-#else
 	    /* check for bignum */
 	    if (!IS_SSMALL((Sint)ptr[0]))
 		need += BIG_UINT_HEAP_SIZE;  /* use small_to_big */
-#endif
 	    ptr++;
 	    depth++;
 	    break;
 	case ERL_DRV_UINT:  /* unsigned int argument */
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
-#if HALFWORD_HEAP
-	    erts_bld_uint64(NULL, &need, (Uint64)ptr[0]);
-#else
 	    /* check for bignum */
 	    if (!IS_USMALL(0, (Uint)ptr[0]))
 		need += BIG_UINT_HEAP_SIZE;  /* use small_to_big */
-#endif
 	    ptr++;
 	    depth++;
 	    break;
@@ -5465,10 +5457,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    break;
 
 	case ERL_DRV_INT:  /* signed int argument */
-#if HALFWORD_HEAP
-	    erts_reserve_heap(&factory, BIG_NEED_SIZE(2));
-	    mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]);
-#else
 	    erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
 	    if (IS_SSMALL((Sint)ptr[0]))
 		mess = make_small((Sint)ptr[0]);
@@ -5476,15 +5464,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		mess = small_to_big((Sint)ptr[0], factory.hp);
 		factory.hp += BIG_UINT_HEAP_SIZE;
 	    }
-#endif
 	    ptr++;
 	    break;
 
 	case ERL_DRV_UINT:  /* unsigned int argument */
-#if HALFWORD_HEAP
-	    erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64));
-	    mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]);
-#else
 	    erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE);
 	    if (IS_USMALL(0, (Uint)ptr[0]))
 		mess = make_small((Uint)ptr[0]);
@@ -5492,7 +5475,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		mess = uint_to_big((Uint)ptr[0], factory.hp);
 		factory.hp += BIG_UINT_HEAP_SIZE;
 	    }
-#endif
 	    ptr++;
 	    break;
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index bb871b05ba..14eeb0ee8f 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -285,62 +285,11 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 #else
 #error Neither 32 nor 64 bit architecture
 #endif
-#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
-#    define HALFWORD_HEAP 1
-#    define HALFWORD_ASSERT 0
-#    define ASSERT_HALFWORD(COND) ASSERT(COND)
-#    undef ERTS_SIZEOF_TERM
-#    define ERTS_SIZEOF_TERM 4
-#else
-#    define HALFWORD_HEAP 0
-#    define HALFWORD_ASSERT 0
-#    define ASSERT_HALFWORD(COND)
-#endif
 
 #if SIZEOF_VOID_P != SIZEOF_SIZE_T
 #error sizeof(void*) != sizeof(size_t)
 #endif
 
-#if HALFWORD_HEAP
-
-#if SIZEOF_INT == 4
-typedef unsigned int Eterm;
-typedef unsigned int Uint;
-typedef int          Sint;
-#define ERTS_UINT_MAX UINT_MAX
-#define ERTS_SIZEOF_ETERM SIZEOF_INT
-#define ErtsStrToSint strtol
-#else
-#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
-#endif
-
-#if SIZEOF_VOID_P == SIZEOF_LONG
-typedef unsigned long UWord;
-typedef long          SWord;
-#define SWORD_CONSTANT(Const) Const##L
-#define UWORD_CONSTANT(Const) Const##UL
-#define ERTS_UWORD_MAX ULONG_MAX
-#define ERTS_SWORD_MAX LONG_MAX
-#elif SIZEOF_VOID_P == SIZEOF_INT
-typedef unsigned int UWord;
-typedef int          SWord;
-#define SWORD_CONSTANT(Const) Const
-#define UWORD_CONSTANT(Const) Const##U
-#define ERTS_UWORD_MAX UINT_MAX
-#define ERTS_SWORD_MAX INT_MAX
-#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
-typedef unsigned long long UWord;
-typedef long long          SWord;
-#define SWORD_CONSTANT(Const) Const##LL
-#define UWORD_CONSTANT(Const) Const##ULL
-#define ERTS_UWORD_MAX ULLONG_MAX
-#define ERTS_SWORD_MAX LLONG_MAX
-#else
-#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
-#endif
-
-#else /* !HALFWORD_HEAP */
-
 #if SIZEOF_VOID_P == SIZEOF_LONG
 typedef unsigned long Eterm;
 typedef unsigned long Uint;
@@ -383,8 +332,6 @@ typedef Uint UWord;
 typedef Sint SWord;
 #define ERTS_UINT_MAX ERTS_UWORD_MAX
 
-#endif /* HALFWORD_HEAP */
-
 typedef UWord BeamInstr;
 
 #ifndef HAVE_INT64
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 342e91e983..28d1b38f75 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -898,7 +898,7 @@ tail_recur:
 	    Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
 
 	    UINT32_HASH_STEP(y2, FUNNY_NUMBER2);
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 	    if (y2 >> 32)
 		UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2);
 #endif
@@ -1019,7 +1019,7 @@ tail_recur:
 	    }
 	    d = BIG_DIGIT(ptr, k);
 	    k = sizeof(ErtsDigit);
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
 	    if (!(d >> 32))
 		k /= 2;
 #endif
@@ -1989,7 +1989,7 @@ tail_recur:
 	    (atom_tab(atom_val(term))->slot.bucket.hvalue);
 	break;
     case SMALL_DEF:
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
     {
 	Sint y1 = signed_val(term);
 	Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
@@ -2598,11 +2598,7 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp)
  * Test for equality of two terms.
  * Returns 0 if not equal, or a non-zero value otherwise.
  */
-#if HALFWORD_HEAP
-int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
-#else
 int eq(Eterm a, Eterm b)
-#endif
 {
     DECLARE_WSTACK(stack);
     Sint sz;
@@ -2992,7 +2988,6 @@ static int cmp_atoms(Eterm a, Eterm b)
 		    bb->name+3, bb->len-3);
 }
 
-#if !HALFWORD_HEAP
 /* cmp(Eterm a, Eterm b)
  *  For compatibility with HiPE - arith-based compare.
  */
@@ -3000,23 +2995,10 @@ Sint cmp(Eterm a, Eterm b)
 {
     return erts_cmp(a, b, 0, 0);
 }
-#endif
 
-#if HALFWORD_HEAP
-static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base,
-                                      Eterm b, Eterm* b_base,
-                                      int exact, int eq_only);
-#else
 static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
-#endif
 
-#if HALFWORD_HEAP
-Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base,
-                      Eterm b, Eterm* b_base,
-                      int exact, int eq_only)
-#else
 Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
-#endif
 {
     if (is_atom(a) && is_atom(b)) {
         return cmp_atoms(a, b);
@@ -3028,11 +3010,7 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
         GET_DOUBLE_REL(b, bf, b_base);
         return float_comp(af.fd, bf.fd);
     }
-#if HALFWORD_HEAP
-    return erts_cmp_compound_rel_opt(a,a_base,b,b_base,exact,eq_only);
-#else
     return erts_cmp_compound(a,b,exact,eq_only);
-#endif
 }
 
 
@@ -3040,13 +3018,7 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
  * exact = 1 -> term-based compare
  * exact = 0 -> arith-based compare
  */
-#if HALFWORD_HEAP
-static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base,
-                                      Eterm b, Eterm* b_base,
-                                      int exact, int eq_only)
-#else
 static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
-#endif
 {
 #define PSTACK_TYPE struct erts_cmp_hashmap_state
     struct erts_cmp_hashmap_state {
@@ -3551,13 +3523,8 @@ tailrecur_ne:
     {
 	FloatDef f1, f2;
 	Eterm big;
-#if HALFWORD_HEAP
-	Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base);
-	Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base);
-#else
 	Eterm aw = a;
 	Eterm bw = b;
-#endif
 #define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2))
 #define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1))
 #define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */
@@ -4369,7 +4336,7 @@ iolist_size(const int yield_support, ErtsIOListState *state, Eterm obj, ErlDrvSi
 {
     int res, init_yield_count, yield_count;
     Eterm* objp;
-    Uint size = (Uint) *sizep; /* Intentionally Uint due to halfword heap */
+    Uint size = (Uint) *sizep;
     DECLARE_ESTACK(s);
 
     if (!yield_support)
-- 
cgit v1.2.3


From 49e744df7a31e329c03567f3bd7cfb72e4a555d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 17 Jun 2015 21:03:53 +0200
Subject: erts: Remove halfword in erl_driver.h

---
 erts/emulator/beam/erl_driver.h | 9 ---------
 1 file changed, 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 6b406d069c..ef2d41e7a2 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -57,10 +57,6 @@
 #  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
 #  undef SIZEOF_LONG_LONG
 #endif
-#ifdef HALFWORD_HEAP_EMULATOR
-#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
-#  undef HALFWORD_HEAP_EMULATOR
-#endif
 #include "erl_int_sizes_config.h"
 #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR
 #  error SIZEOF_CHAR mismatch
@@ -78,11 +74,6 @@
 #  error SIZEOF_LONG_LONG mismatch
 #endif
 
-/* This is OK to override by the NIF/driver implementor */
-#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR)
-#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__
-#endif
-
 #include "erl_drv_nif.h"
 
 #include <stdlib.h>
-- 
cgit v1.2.3


From 16b83562b0d2e9f9892744dcbf7894ef7d18a82a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 17 Jun 2015 21:08:52 +0200
Subject: erts: Remove halfword in erl_nif.h

---
 erts/emulator/beam/erl_nif.h | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 7d880126f8..5da6bb877a 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -86,10 +86,6 @@
 #  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
 #  undef SIZEOF_LONG_LONG
 #endif
-#ifdef HALFWORD_HEAP_EMULATOR
-#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
-#  undef HALFWORD_HEAP_EMULATOR
-#endif
 #include "erl_int_sizes_config.h"
 
 #ifdef __cplusplus
@@ -109,16 +105,11 @@ typedef long long ErlNifSInt64;
 #error No 64-bit integer type
 #endif
 
-#ifdef HALFWORD_HEAP_EMULATOR
-#  define ERL_NIF_VM_VARIANT "beam.halfword" 
-typedef unsigned int ERL_NIF_TERM;
-#else
-#  define ERL_NIF_VM_VARIANT "beam.vanilla" 
-#  if SIZEOF_LONG == SIZEOF_VOID_P
+#define ERL_NIF_VM_VARIANT "beam.vanilla" 
+#if SIZEOF_LONG == SIZEOF_VOID_P
 typedef unsigned long ERL_NIF_TERM;
-#  elif SIZEOF_LONG_LONG == SIZEOF_VOID_P
+#elif SIZEOF_LONG_LONG == SIZEOF_VOID_P
 typedef unsigned long long ERL_NIF_TERM;
-#  endif
 #endif
 
 typedef ERL_NIF_TERM ERL_NIF_UINT;
-- 
cgit v1.2.3


From 4f9c6bd8fc7090dcbc4b5b3cf595e1689fdaff7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 14:30:28 +0200
Subject: erts: Remove halfword basic relative heap operations

---
 erts/emulator/beam/big.c                      |   2 +-
 erts/emulator/beam/copy.c                     |  48 +++---
 erts/emulator/beam/erl_binary.h               |   7 +-
 erts/emulator/beam/erl_db_hash.c              |   2 +-
 erts/emulator/beam/erl_db_tree.c              |  10 +-
 erts/emulator/beam/erl_db_util.c              |  60 ++++----
 erts/emulator/beam/erl_db_util.h              |   2 +-
 erts/emulator/beam/erl_map.c                  |  16 +-
 erts/emulator/beam/erl_map.h                  |   9 +-
 erts/emulator/beam/erl_node_container_utils.h |   2 -
 erts/emulator/beam/erl_printf_term.c          |  10 +-
 erts/emulator/beam/erl_term.h                 |  56 -------
 erts/emulator/beam/io.c                       |   4 +-
 erts/emulator/beam/utils.c                    | 213 +++++++++++++-------------
 14 files changed, 188 insertions(+), 253 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 1e6582a7ca..8662398dcf 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1669,7 +1669,7 @@ double_to_big(double x, Eterm *heap, Uint hsz)
     sz = BIG_NEED_SIZE(ds); /* number of words including arity */
 
     hp = heap;
-    res = make_big_rel(hp, heap);
+    res = make_big(hp);
     xp = (ErtsDigit*) (hp + 1);
 
     ASSERT(ds < hsz);
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 94048f4c59..38c40f4e7d 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -80,7 +80,7 @@ Uint size_object(Eterm obj)
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST:
 	    sum += 2;
-	    ptr = list_val_rel(obj,base);
+	    ptr = list_val(obj);
 	    obj = *ptr++;
 	    if (!IS_CONST(obj)) {
 		ESTACK_PUSH(s, obj);
@@ -89,11 +89,11 @@ Uint size_object(Eterm obj)
 	    break;
 	case TAG_PRIMARY_BOXED:
 	    {
-		Eterm hdr = *boxed_val_rel(obj,base);
+		Eterm hdr = *boxed_val(obj);
 		ASSERT(is_header(hdr));
 		switch (hdr & _TAG_HEADER_MASK) {
 		case ARITYVAL_SUBTAG:
-		    ptr = tuple_val_rel(obj,base);
+		    ptr = tuple_val(obj);
 		    arity = header_arity(hdr);
 		    sum += arity + 1;
 		    if (arity == 0) { /* Empty tuple -- unusual. */
@@ -109,7 +109,7 @@ Uint size_object(Eterm obj)
 		    break;
 		case FUN_SUBTAG:
 		    {
-			Eterm* bptr = fun_val_rel(obj,base);
+			Eterm* bptr = fun_val(obj);
 			ErlFunThing* funp = (ErlFunThing *) bptr;
 			unsigned eterms = 1 /* creator */ + funp->num_free;
 			unsigned sz = thing_arityval(hdr);
@@ -130,7 +130,7 @@ Uint size_object(Eterm obj)
                             {
                                 Uint n;
                                 flatmap_t *mp;
-                                mp  = (flatmap_t*)flatmap_val_rel(obj,base);
+                                mp  = (flatmap_t*)flatmap_val(obj);
                                 ptr = (Eterm *)mp;
                                 n   = flatmap_get_size(mp) + 1;
                                 sum += n + 2;
@@ -149,7 +149,7 @@ Uint size_object(Eterm obj)
 			    {
 				Eterm *head;
 				Uint sz;
-				head  = hashmap_val_rel(obj, base);
+				head  = hashmap_val(obj);
 				sz    = hashmap_bitcount(MAP_HEADER_VAL(hdr));
 				sum  += 1 + sz + header_arity(hdr);
 				head += 1 + header_arity(hdr);
@@ -188,11 +188,11 @@ Uint size_object(Eterm obj)
 			} else {
 			    extra_bytes = 0;
 			}
-			hdr = *binary_val_rel(real_bin,base);
+			hdr = *binary_val(real_bin);
 			if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) {
 			    sum += PROC_BIN_SIZE;
 			} else {
-			    sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes);
+			    sum += heap_bin_size(binary_size(obj)+extra_bytes);
 			}
 			goto pop_next;
 		    }
@@ -259,7 +259,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     switch (primary_tag(obj)) {
     case TAG_PRIMARY_LIST:
 	argp = &res;
-	objp = list_val_rel(obj,src_base);
+	objp = list_val(obj);
 	goto L_copy_list;
     case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed;
     default:
@@ -277,7 +277,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    hp++;
 	    break;
 	case TAG_PRIMARY_LIST:
-	    objp = list_val_rel(obj,src_base);
+	    objp = list_val(obj);
 	    if (in_area(objp,hstart,hsize)) {
 		hp++;
 		break;
@@ -300,12 +300,12 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    tailp = &CDR(htop);
 		    htop += 2;
 		}
-		*tp = make_list_rel(tailp - 1, dst_base);
+		*tp = make_list(tailp - 1);
 		obj = CDR(objp);
 		if (!is_list(obj)) {
 		    break;
 		}
-		objp = list_val_rel(obj,src_base);
+		objp = list_val(obj);
 	    }
 	    switch (primary_tag(obj)) {
 	    case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
@@ -317,21 +317,21 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    }
 	    
 	case TAG_PRIMARY_BOXED:
-	    if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) {
+	    if (in_area(boxed_val(obj),hstart,hsize)) {
 		hp++;
 		break;
 	    }
 	    argp = hp++;
 
 	L_copy_boxed:
-	    objp = boxed_val_rel(obj, src_base);
+	    objp = boxed_val(obj);
 	    hdr = *objp;
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
 		{
 		    int const_flag = 1; /* assume constant tuple */
 		    i = arityval(hdr);
-		    *argp = make_tuple_rel(htop, dst_base);
+		    *argp = make_tuple(htop);
 		    tp = htop;	/* tp is pointer to new arity value */
 		    *htop++ = *objp++; /* copy arity value */
 		    while (i--) {
@@ -360,7 +360,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    while (i--)  {
 			*tp++ = *objp++;
 		    }
-		    *argp = make_binary_rel(hbot, dst_base);
+		    *argp = make_binary(hbot);
 		    pb = (ProcBin*) hbot;
 		    erts_refc_inc(&pb->val->refc, 2);
 		    pb->next = off_heap->first;
@@ -387,7 +387,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			extra_bytes = 0;
 		    } 
 		    real_size = size+extra_bytes;
-		    objp = binary_val_rel(real_bin,src_base);
+		    objp = binary_val(real_bin);
 		    if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) {
 			ErlHeapBin* from = (ErlHeapBin *) objp;
 			ErlHeapBin* to;
@@ -417,7 +417,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			off_heap->first = (struct erl_off_heap_header*) to;
 			OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
 		    }
-		    *argp = make_binary_rel(hbot, dst_base);
+		    *argp = make_binary(hbot);
 		    if (extra_bytes != 0) {
 			ErlSubBin* res;
 			hbot -= ERL_SUB_BIN_SIZE;
@@ -429,7 +429,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			res->offs = 0;
 			res->is_writable = 0;
 			res->orig = *argp;
-			*argp = make_binary_rel(hbot, dst_base);
+			*argp = make_binary(hbot);
 		    }
 		    break;
 		}
@@ -447,7 +447,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    funp->next = off_heap->first;
 		    off_heap->first = (struct erl_off_heap_header*) funp;
 		    erts_refc_inc(&funp->fe->refc, 2);
-		    *argp = make_fun_rel(tp, dst_base);
+		    *argp = make_fun(tp);
 		}
 		break;
 	    case EXTERNAL_PID_SUBTAG:
@@ -467,7 +467,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		  off_heap->first = (struct erl_off_heap_header*)etp;
 		  erts_refc_inc(&etp->node->refc, 2);
 
-		  *argp = make_external_rel(tp, dst_base);
+		  *argp = make_external(tp);
 		}
 		break;
 	    case MAP_SUBTAG:
@@ -475,7 +475,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		switch (MAP_HEADER_TYPE(hdr)) {
 		    case MAP_HEADER_TAG_FLATMAP_HEAD :
                         i = flatmap_get_size(objp) + 3;
-                        *argp = make_flatmap_rel(htop, dst_base);
+                        *argp = make_flatmap(htop);
                         while (i--) {
                             *htop++ = *objp++;
                         }
@@ -486,7 +486,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
 			i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr));
 			while (i--)  { *htop++ = *objp++; }
-			*argp = make_hashmap_rel(tp, dst_base);
+			*argp = make_hashmap(tp);
 			break;
 		    default:
 			erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
@@ -499,7 +499,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		i = thing_arityval(hdr)+1;
 		hbot -= i;
 		tp = hbot;
-		*argp = make_boxed_rel(hbot, dst_base);
+		*argp = make_boxed(hbot);
 		while (i--) {
 		    *tp++ = *objp++;
 		}
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index ea01bf08f0..d24d041530 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -72,7 +72,6 @@ typedef struct erl_heap_bin {
  */
 
 #define binary_size(Bin) (binary_val(Bin)[1])
-#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1])
 
 #define binary_bitsize(Bin)			\
   ((*binary_val(Bin) == HEADER_SUB_BIN) ?	\
@@ -100,7 +99,7 @@ typedef struct erl_heap_bin {
 
 #define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr)    \
 do {									\
-    Eterm* _real_bin = binary_val_rel(Bin,BasePtr);			\
+    Eterm* _real_bin = binary_val(Bin);		                	\
     Uint _offs = 0;							\
     Bitoffs = Bitsize = 0;						\
     if (*_real_bin == HEADER_SUB_BIN) {					\
@@ -108,7 +107,7 @@ do {									\
 	_offs = _sb->offs;						\
         Bitoffs = _sb->bitoffs;						\
         Bitsize = _sb->bitsize;						\
-	_real_bin = binary_val_rel(_sb->orig,BasePtr);			\
+	_real_bin = binary_val(_sb->orig);	        		\
     }									\
     if (*_real_bin == HEADER_PROC_BIN) {				\
 	Bytep = ((ProcBin *) _real_bin)->bytes + _offs;			\
@@ -135,7 +134,7 @@ do {									\
 
 #define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \
   do {									\
-    ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr);	        \
+    ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin);	                \
     if (_sb->thing_word == HEADER_SUB_BIN) {				\
       RealBin = _sb->orig;						\
       ByteOffset = _sb->offs;						\
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index fc1aa05ed2..80563e1f02 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2193,7 +2193,7 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl)
 		    erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl);
 		}
 		else {
-		    Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl);
+		    Eterm obj = make_tuple(list->dbterm.tpl);
 		    erts_print(to, to_arg, "%R", obj, list->dbterm.tpl);
 		}
 		if (list->next != 0)
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index c21eac6f25..3b698a6adf 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -2745,7 +2745,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	    return cmp_rel(a,NULL,b,b_base);
 	}
 	aa = list_val(a);
-	bb = list_val_rel(b,b_base);
+	bb = list_val(b);
 	while (1) {
 	    if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done)
 		return j;
@@ -2754,20 +2754,20 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	    if (is_not_list(*aa) || is_not_list(*bb))
 		return do_cmp_partly_bound(*aa, *bb, b_base, done);
 	    aa = list_val(*aa);
-	    bb = list_val_rel(*bb,b_base);
+	    bb = list_val(*bb);
 	}
     case TAG_PRIMARY_BOXED:
 	if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) {
 	    return cmp_rel(a,NULL,b,b_base);
 	}
 	a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
-	b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
+	b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
 	if (a_hdr != b_hdr) {
 	    return cmp_rel(a, NULL, b, b_base);
 	}
 	if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) {
 	    aa = tuple_val(a);
-	    bb = tuple_val_rel(b, b_base);
+	    bb = tuple_val(b);
 	    /* compare the arities */
 	    i = arityval(*aa);	/* get the arity*/
 	    if (i < arityval(*bb)) return(-1);
@@ -3154,7 +3154,7 @@ static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show,
 	}
 	else {
 	    prefix = "";
-	    term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl);
+	    term = make_tuple(t->dbterm.tpl);
 	}
 	erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n",
 		   offset, "", prefix, term, t->dbterm.tpl,
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 0086aa8121..432ca297e9 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1903,9 +1903,9 @@ restart:
 	    variables[n].term = dpm_array_to_list(psp, termp, arity);
 	    break;
 	case matchTuple: /* *ep is a tuple of arity n */
-	    if (!is_tuple_rel(*ep,base))
+	    if (!is_tuple(*ep))
 		FAIL();
-	    ep = tuple_val_rel(*ep,base);
+	    ep = tuple_val(*ep);
 	    n = *pc++;
 	    if (arityval(*ep) != n)
 		FAIL();
@@ -1913,9 +1913,9 @@ restart:
 	    break;
 	case matchPushT: /* *ep is a tuple of arity n, 
 			    push ptr to first element */
-	    if (!is_tuple_rel(*ep,base))
+	    if (!is_tuple(*ep))
 		FAIL();
-	    tp = tuple_val_rel(*ep,base);
+	    tp = tuple_val(*ep);
 	    n = *pc++;
 	    if (arityval(*tp) != n)
 		FAIL();
@@ -1925,51 +1925,51 @@ restart:
 	case matchList:
 	    if (!is_list(*ep))
 		FAIL();
-	    ep = list_val_rel(*ep,base);
+	    ep = list_val(*ep);
 	    break;
 	case matchPushL:
 	    if (!is_list(*ep))
 		FAIL();
-	    *sp++ = list_val_rel(*ep,base);
+	    *sp++ = list_val(*ep);
 	    ++ep;
 	    break;
         case matchMap:
-            if (!is_map_rel(*ep, base)) {
+            if (!is_map(*ep)) {
                 FAIL();
             }
             n = *pc++;
-            if (is_flatmap_rel(*ep,base)) {
-		if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) {
+            if (is_flatmap(*ep)) {
+		if (flatmap_get_size(flatmap_val(*ep)) < n) {
 		    FAIL();
 		}
             } else {
-		ASSERT(is_hashmap_rel(*ep,base));
-		if (hashmap_size_rel(*ep, base) < n) {
+		ASSERT(is_hashmap(*ep));
+		if (hashmap_size(*ep) < n) {
 		    FAIL();
 		}
 	    }
-            ep = flatmap_val_rel(*ep, base);
+            ep = flatmap_val(*ep);
             break;
         case matchPushM:
-            if (!is_map_rel(*ep, base)) {
+            if (!is_map(*ep)) {
                 FAIL();
             }
             n = *pc++;
-            if (is_flatmap_rel(*ep,base)) {
-		if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) {
+            if (is_flatmap(*ep)) {
+		if (flatmap_get_size(flatmap_val(*ep)) < n) {
 		    FAIL();
 		}
 	    } else {
-		ASSERT(is_hashmap_rel(*ep,base));
-		if (hashmap_size_rel(*ep, base) < n) {
+		ASSERT(is_hashmap(*ep));
+		if (hashmap_size(*ep) < n) {
 		    FAIL();
 		}
 	    }
-            *sp++ = flatmap_val_rel(*ep++, base);
+            *sp++ = flatmap_val(*ep++);
             break;
         case matchKey:
             t = (Eterm) *pc++;
-            tp = erts_maps_get_rel(t, make_boxed_rel(ep, base), base);
+            tp = erts_maps_get(t, make_boxed(ep));
             if (!tp) {
                 FAIL();
             }
@@ -2001,18 +2001,18 @@ restart:
 	    ++ep;
 	    break;
 	case matchEqFloat:
-	    if (!is_float_rel(*ep,base))
+	    if (!is_float(*ep))
 		FAIL();
-	    if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double)))
+	    if (memcmp(float_val(*ep) + 1, pc, sizeof(double)))
 		FAIL();
 	    pc += TermWords(2);
 	    ++ep;
 	    break;
 	case matchEqRef: {
 	    Eterm* epc = (Eterm*)pc;
-	    if (!is_ref_rel(*ep,base))
+	    if (!is_ref(*ep))
 		FAIL();
-	    if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) {
+	    if (!eq_rel(make_internal_ref(epc), epc, *ep, base)) {
 		FAIL();
 	    }
 	    i = thing_arityval(*epc);
@@ -2021,9 +2021,9 @@ restart:
 	    break;
 	}
 	case matchEqBig:
-	    if (!is_big_rel(*ep,base))
+	    if (!is_big(*ep))
 		FAIL();
-	    tp = big_val_rel(*ep,base);
+	    tp = big_val(*ep);
 	    {
 		Eterm *epc = (Eterm *) pc;
 		if (*tp != *epc)
@@ -2193,8 +2193,8 @@ restart:
 		sz = size_object_rel(term, base);
 		top = HAllocX(build_proc, sz, HEAP_XTRA);
 		if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) {
-		    ASSERT(is_tuple_rel(term,base));
-		    *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz,
+		    ASSERT(is_tuple(term));
+		    *esp++ = copy_shallow_rel(tuple_val(term), sz,
 					      &top, &MSO(build_proc), base);
 		}
 		else {
@@ -2741,7 +2741,7 @@ void db_do_update_element(DbUpdateHandle* handle,
 		case _TAG_HEADER_HEAP_BIN:
 		    newval_sz = header_arity(*newp) + 1;
 		    if (is_boxed(oldval)) {
-			oldp = boxed_val_rel(oldval,old_base);
+			oldp = boxed_val(oldval);
 			switch (*oldp & _TAG_HEADER_MASK) {
 			case _TAG_HEADER_POS_BIG:
 			case _TAG_HEADER_NEG_BIG:
@@ -3023,7 +3023,7 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset)
 	tmp_offheap.first = NULL;
 
 	{
-	    copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top,
+	    copy_struct_rel(make_tuple(tpl), handle->new_size, &top,
 			    &tmp_offheap, tpl, top);
 	    newDbTerm->first_oh = tmp_offheap.first;
 	    ASSERT((byte*)top == (newp + alloc_sz));
@@ -5228,7 +5228,7 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
     }
     else base = NULL;
 
-    res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0,
+    res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), base, NULL, 0,
 			ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
 
     if (is_value(res) && hpp!=NULL) {
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index c2128888c5..d07e61e599 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -306,7 +306,7 @@ ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
 ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b)
 {
     if (!tb->compress) {
-	return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl);
+	return eq_rel(a, NULL, make_tuple(b->tpl), b->tpl);
     }
     else {
 	return db_eq_comp(tb, a, b);
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 6d496ea5ee..500234c436 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -173,19 +173,19 @@ const Eterm *
 erts_maps_get(Eterm key, Eterm map)
 {
     Uint32 hx;
-    if (is_flatmap_rel(map, map_base)) {
+    if (is_flatmap(map)) {
 	Eterm *ks, *vs;
 	flatmap_t *mp;
 	Uint n, i;
 
-	mp  = (flatmap_t *)flatmap_val_rel(map, map_base);
+	mp  = (flatmap_t *)flatmap_val(map);
 	n   = flatmap_get_size(mp);
 
 	if (n == 0) {
 	    return NULL;
 	}
 
-	ks  = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1;
+	ks  = (Eterm *)tuple_val(mp->keys) + 1;
 	vs  = flatmap_get_values(mp);
 
 	if (is_immed(key)) {
@@ -203,10 +203,10 @@ erts_maps_get(Eterm key, Eterm map)
 	}
 	return NULL;
     }
-    ASSERT(is_hashmap_rel(map, map_base));
+    ASSERT(is_hashmap(map));
     hx = hashmap_make_hash(key);
 
-    return erts_hashmap_get_rel(hx, key, map, map_base);
+    return erts_hashmap_get(hx, key, map);
 }
 
 BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
@@ -1998,7 +1998,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
     UseTmpHeapNoproc(2);
 
     ASSERT(is_boxed(node));
-    ptr = boxed_val_rel(node, map_base);
+    ptr = boxed_val(node);
     hdr = *ptr;
     ASSERT(is_header(hdr));
     ASSERT(is_hashmap_header_head(hdr));
@@ -2019,7 +2019,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
         node  = ptr[ix+1];
 
         if (is_list(node)) { /* LEAF NODE [K|V] */
-            ptr = list_val_rel(node,map_base);
+            ptr = list_val(node);
             res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL;
             break;
         }
@@ -2027,7 +2027,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
         hx = hashmap_shift_hash(th,hx,lvl,key);
 
         ASSERT(is_boxed(node));
-        ptr = boxed_val_rel(node, map_base);
+        ptr = boxed_val(node);
         hdr = *ptr;
         ASSERT(is_header(hdr));
         ASSERT(!is_hashmap_header_head(hdr));
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 2f7c55829f..be6f791a4e 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -57,7 +57,6 @@ typedef struct flatmap_s {
 
 
 #define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size)
-#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
 #define hashmap_make_hash(Key)        make_internal_hash(Key)
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
@@ -104,13 +103,9 @@ Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int rejec
 Eterm  erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n,
 					 Eterm k, Eterm v);
 
-const Eterm *
-erts_maps_get(Eterm key, Eterm map);
-#define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B)
+const Eterm *erts_maps_get(Eterm key, Eterm map);
 
-const Eterm *
-erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
-#define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M)
+const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
 
 /* hamt nodes v2.0
  *
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 0a4b18b748..b6a3531e28 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -328,8 +328,6 @@ extern ErtsPTab erts_port;
 					 : external_ref_channel_no((x)))
 #define is_ref(x)			(is_internal_ref((x))		\
 					 || is_external_ref((x)))
-#define is_ref_rel(x,Base)		(is_internal_ref_rel((x),Base)	\
-					 || is_external_ref_rel((x),Base))
 #define is_not_ref(x)			(!is_ref(x))
 
 #endif
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 95b3572372..7a40f775f2 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -123,7 +123,7 @@ is_printable_string(Eterm list, Eterm* base)
     int c;
 
     while(is_list(list)) {
-	Eterm* consp = list_val_rel(list, base);
+	Eterm* consp = list_val(list);
 	Eterm hd = CAR(consp);
 
 	if (!is_byte(hd))
@@ -308,7 +308,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		obj = (Eterm) popped.word;
 	    L_print_one_cons:
 		{
-		    Eterm* cons = list_val_rel(obj, obj_base);
+		    Eterm* cons = list_val(obj);
 		    Eterm tl;
 		    
 		    obj = CAR(cons);
@@ -423,7 +423,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    if (is_printable_string(obj, obj_base)) {
 		int c;
 		PRINT_CHAR(res, fn, arg, '"');
-		nobj = list_val_rel(obj, obj_base);
+		nobj = list_val(obj);
 		while (1) {
 		    if ((*dcount)-- <= 0)
 			goto L_done;
@@ -437,7 +437,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		    }
 		    if (is_not_list(*nobj))
 			break;
-		    nobj = list_val_rel(*nobj, obj_base);
+		    nobj = list_val(*nobj);
 		}
 		PRINT_CHAR(res, fn, arg, '"');
 	    } else {
@@ -468,7 +468,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    }
 	    else {
 		byte* bytep;
-		Uint bytesize = binary_size_rel(obj,obj_base);
+		Uint bytesize = binary_size(obj);
 		Uint bitoffs;
 		Uint bitsize;
 		byte octet;
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 46900edfc7..ad4ef3c01a 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -977,27 +977,20 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #define MAP_HEADER_VAL(Hdr)   (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff))
 
 #define make_hashmap(x)		      make_boxed((Eterm*)(x))
-#define make_hashmap_rel 	      make_boxed_rel
 #define is_hashmap(x)		      (is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
 #define is_not_hashmap(x)             (!is_hashmap(x))
-#define is_hashmap_rel(RTERM,BASE)    is_hashmap(rterm2wterm(RTERM,BASE))
 #define is_hashmap_header(x)	      (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY)
 #define hashmap_val(x)		      _unchecked_boxed_val((x))
-#define hashmap_val_rel(RTERM, BASE)  hashmap_val(rterm2wterm(RTERM, BASE))
 
 #define make_flatmap(x)               make_boxed((Eterm*)(x))
-#define make_flatmap_rel(x, BASE)     make_boxed_rel((Eterm*)(x),(BASE))
 #define is_flatmap(x)                 (is_boxed((x)) && is_flatmap_header(*boxed_val((x))))
-#define is_flatmap_rel(RTERM,BASE)    is_flatmap(rterm2wterm(RTERM,BASE))
 #define is_not_flatmap(x)             (!is_flatmap((x)))
 #define is_flatmap_header(x)          (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP)
 #define flatmap_val(x)                (_unchecked_boxed_val((x)))
-#define flatmap_val_rel(RTERM, BASE)  flatmap_val(rterm2wterm(RTERM, BASE))
 
 #define is_map_header(x)       (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
 #define is_map(x)              (is_boxed((x)) && is_map_header(*boxed_val(x)))
 #define is_not_map(x)          (!is_map(x))
-#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE))
 
 /* number tests */
 
@@ -1127,55 +1120,6 @@ extern unsigned tag_val_def(Wterm);
 #define FLOAT_BIG 	_NUMBER_CODE(FLOAT_DEF,BIG_DEF)
 #define FLOAT_FLOAT	_NUMBER_CODE(FLOAT_DEF,FLOAT_DEF)
 
-#define ptr2rel(PTR,BASE) (PTR)
-#define rterm2wterm(REL,BASE) (REL)
-
-#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE))
-#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE))
-#define make_fun_rel make_boxed_rel
-#define make_binary_rel make_boxed_rel
-#define make_tuple_rel make_boxed_rel
-#define make_external_rel make_boxed_rel
-#define make_internal_ref_rel make_boxed_rel
-#define make_big_rel make_boxed_rel
-
-#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE))
-#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE))
-#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE))
-#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE))
-#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE))
-#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE))
-#define big_val_rel(RTERM,BASE)	big_val(rterm2wterm(RTERM,BASE))
-#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE))
-#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE))
-
-#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE))
-#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE))
-
-#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE))
-#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE))
-
-#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE))
-#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE))
-#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE))
-
-#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE))
-#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE))
-#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE))
-#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE))
-#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE))
-#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE))
-
-#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f)
-
-#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE))
-#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE))
-#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE))
-#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE))
-#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE))
-
-#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE))
-
 #define is_same(A,A_BASE,B,B_BASE) ((A)==(B))
 
 #endif	/* __ERL_TERM_H */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c365b8859b..6b4b90cb06 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4091,10 +4091,10 @@ erts_port_control(Process* c_p,
     binp = NULL;
 
     if (is_binary(data) && binary_bitoffset(data) == 0) {
-	Eterm *ebinp = binary_val_rel(data, NULL);
+	Eterm *ebinp = binary_val(data);
 	ASSERT(!tmp_alloced);
 	if (*ebinp == HEADER_SUB_BIN)
-	    ebinp = binary_val_rel(((ErlSubBin *) ebinp)->orig, NULL);
+	    ebinp = binary_val(((ErlSubBin *) ebinp)->orig);
 	if (*ebinp != HEADER_PROC_BIN)
 	    copy = 1;
 	else {
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 28d1b38f75..9f919fa2ab 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2612,8 +2612,8 @@ tailrecur_ne:
     switch (primary_tag(a)) {
     case TAG_PRIMARY_LIST:
 	if (is_list(b)) {
-	    Eterm* aval = list_val_rel(a, a_base);
-	    Eterm* bval = list_val_rel(b, b_base);
+	    Eterm* aval = list_val(a);
+	    Eterm* bval = list_val(b);
 	    while (1) {
 		Eterm atmp = CAR(aval);
 		Eterm btmp = CAR(bval);
@@ -2633,22 +2633,22 @@ tailrecur_ne:
 		    b = btmp;
 		    goto tailrecur_ne;
 		}
-		aval = list_val_rel(atmp, a_base);
-		bval = list_val_rel(btmp, b_base);
+		aval = list_val(atmp);
+		bval = list_val(btmp);
 	    }
 	}
 	break; /* not equal */
 
     case TAG_PRIMARY_BOXED:
 	{	
-	    Eterm hdr = *boxed_val_rel(a,a_base);
+	    Eterm hdr = *boxed_val(a);
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
 		{
-		    aa = tuple_val_rel(a, a_base);
-		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
+		    aa = tuple_val(a);
+		    if (!is_boxed(b) || *boxed_val(b) != *aa)
 			goto not_equal;
-		    bb = tuple_val_rel(b,b_base);
+		    bb = tuple_val(b);
 		    if ((sz = arityval(*aa)) == 0) goto pop_next;
 		    ++aa;
 		    ++bb;
@@ -2667,11 +2667,11 @@ tailrecur_ne:
 		    Uint a_bitoffs;
 		    Uint b_bitoffs;
 		    
-		    if (!is_binary_rel(b,b_base)) {
+		    if (!is_binary(b)) {
 			goto not_equal;
 		    }
-		    a_size = binary_size_rel(a,a_base);
-		    b_size = binary_size_rel(b,b_base);
+		    a_size = binary_size(a);
+		    b_size = binary_size(b);
 		    if (a_size != b_size) {
 			goto not_equal;
 		    }
@@ -2687,9 +2687,9 @@ tailrecur_ne:
 		}
 	    case EXPORT_SUBTAG:
 		{
-		    if (is_export_rel(b,b_base)) {
-			Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
-			Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
+		    if (is_export(b)) {
+			Export* a_exp = *((Export **) (export_val(a) + 1));
+			Export* b_exp = *((Export **) (export_val(b) + 1));
 			if (a_exp == b_exp) goto pop_next;
 		    }
 		    break; /* not equal */
@@ -2699,10 +2699,10 @@ tailrecur_ne:
 		    ErlFunThing* f1;
 		    ErlFunThing* f2;
   
-		    if (!is_fun_rel(b,b_base))
+		    if (!is_fun(b))
 			goto not_equal;
-		    f1 = (ErlFunThing *) fun_val_rel(a,a_base);
-		    f2 = (ErlFunThing *) fun_val_rel(b,b_base);
+		    f1 = (ErlFunThing *) fun_val(a);
+		    f2 = (ErlFunThing *) fun_val(b);
 		    if (f1->fe->module != f2->fe->module ||
 			f1->fe->old_index != f2->fe->old_index ||
 			f1->fe->old_uniq != f2->fe->old_uniq ||
@@ -2720,15 +2720,15 @@ tailrecur_ne:
 		ExternalThing *ap;
 		ExternalThing *bp;
 
-		if(!is_external_rel(b,b_base))
+		if(!is_external(b))
 		    goto not_equal;
 
-		ap = external_thing_ptr_rel(a,a_base);
-		bp = external_thing_ptr_rel(b,b_base);
+		ap = external_thing_ptr(a);
+		bp = external_thing_ptr(b);
 
 		if(ap->header == bp->header && ap->node == bp->node) {
-		    ASSERT(1 == external_data_words_rel(a,a_base));
-		    ASSERT(1 == external_data_words_rel(b,b_base));
+		    ASSERT(1 == external_data_words(a));
+		    ASSERT(1 == external_data_words(b));
 		    
 		    if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next;
 		}
@@ -2749,11 +2749,11 @@ tailrecur_ne:
 		ExternalThing* athing;
 		ExternalThing* bthing;
 
-		if(!is_external_ref_rel(b,b_base))
+		if(!is_external_ref(b))
 		    goto not_equal;
 
-		athing = external_thing_ptr_rel(a,a_base);
-		bthing = external_thing_ptr_rel(b,b_base);
+		athing = external_thing_ptr(a);
+		bthing = external_thing_ptr(b);
 
 		if(athing->node != bthing->node)
 		    goto not_equal;
@@ -2765,12 +2765,12 @@ tailrecur_ne:
 
 		goto ref_common;
 	    case REF_SUBTAG:
-		    if (!is_internal_ref_rel(b,b_base))
+		    if (!is_internal_ref(b))
 			goto not_equal;
 
 		    {
-			RefThing* athing = ref_thing_ptr_rel(a,a_base);
-			RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+			RefThing* athing = ref_thing_ptr(a);
+			RefThing* bthing = ref_thing_ptr(b);
 			alen = internal_thing_ref_no_of_numbers(athing);
 			blen = internal_thing_ref_no_of_numbers(bthing);
 			anum = internal_thing_ref_numbers(athing);
@@ -2820,10 +2820,10 @@ tailrecur_ne:
 		{
 		    int i;
   
-		    if (!is_big_rel(b,b_base))
+		    if (!is_big(b))
 			goto not_equal;
-		    aa = big_val_rel(a,a_base);
-		    bb = big_val_rel(b,b_base);
+		    aa = big_val(a);
+		    bb = big_val(b);
 		    if (*aa != *bb)
 			goto not_equal;
 		    i = BIG_ARITY(aa);
@@ -2838,19 +2838,19 @@ tailrecur_ne:
 		    FloatDef af;
 		    FloatDef bf;
   
-		    if (is_float_rel(b,b_base)) {
-			GET_DOUBLE_REL(a, af, a_base);
-			GET_DOUBLE_REL(b, bf, b_base);
+		    if (is_float(b)) {
+			GET_DOUBLE(a, af);
+			GET_DOUBLE(b, bf);
 			if (af.fd == bf.fd) goto pop_next;
 		    }
 		    break; /* not equal */
 		}
 	    case MAP_SUBTAG:
-                if (is_flatmap_rel(a, a_base)) {
-		    aa = flatmap_val_rel(a, a_base);
-		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
+                if (is_flatmap(a)) {
+		    aa = flatmap_val(a);
+		    if (!is_boxed(b) || *boxed_val(b) != *aa)
 			goto not_equal;
-		    bb = flatmap_val_rel(b,b_base);
+		    bb = flatmap_val(b);
 		    sz = flatmap_get_size((flatmap_t*)aa);
 
 		    if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal;
@@ -2862,11 +2862,11 @@ tailrecur_ne:
 		    goto term_array;
 
                 } else {
-		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != hdr)
+		    if (!is_boxed(b) || *boxed_val(b) != hdr)
 			goto not_equal;
 
-		    aa = hashmap_val_rel(a, a_base) + 1;
-		    bb = hashmap_val_rel(b, b_base) + 1;
+		    aa = hashmap_val(a) + 1;
+		    bb = hashmap_val(b) + 1;
 		    switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
 		    case HAMT_SUBTAG_HEAD_ARRAY:
 			aa++; bb++;
@@ -3004,10 +3004,10 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
         return cmp_atoms(a, b);
     } else if (is_both_small(a, b)) {
         return (signed_val(a) - signed_val(b));
-    } else if (is_float_rel(a, a_base) && is_float_rel(b, b_base)) {
+    } else if (is_float(a) && is_float(b)) {
         FloatDef af, bf;
-        GET_DOUBLE_REL(a, af, a_base);
-        GET_DOUBLE_REL(b, bf, b_base);
+        GET_DOUBLE(a, af);
+        GET_DOUBLE(b, bf);
         return float_comp(af.fd, bf.fd);
     }
     return erts_cmp_compound(a,b,exact,eq_only);
@@ -3111,9 +3111,9 @@ tailrecur_ne:
 	    if (is_internal_port(b)) {
 		bnode = erts_this_node;
 		bdata = internal_port_data(b);
-	    } else if (is_external_port_rel(b,b_base)) {
-		bnode = external_port_node_rel(b,b_base);
-		bdata = external_port_data_rel(b,b_base);
+	    } else if (is_external_port(b)) {
+		bnode = external_port_node(b);
+		bdata = external_port_data(b);
 	    } else {
 		a_tag = PORT_DEF;
 		goto mixed_types;
@@ -3129,9 +3129,9 @@ tailrecur_ne:
 	    if (is_internal_pid(b)) {
 		bnode = erts_this_node;
 		bdata = internal_pid_data(b);
-	    } else if (is_external_pid_rel(b,b_base)) {
-		bnode = external_pid_node_rel(b,b_base);
-		bdata = external_pid_data_rel(b,b_base);
+	    } else if (is_external_pid(b)) {
+		bnode = external_pid_node(b);
+		bdata = external_pid_data(b);
 	    } else {
 		a_tag = PID_DEF;
 		goto mixed_types;
@@ -3164,8 +3164,8 @@ tailrecur_ne:
 	    a_tag = LIST_DEF;
 	    goto mixed_types;
 	}
-	aa = list_val_rel(a,a_base);
-	bb = list_val_rel(b,b_base);
+	aa = list_val(a);
+	bb = list_val(b);
 	while (1) {
 	    Eterm atmp = CAR(aa);
 	    Eterm btmp = CAR(bb);
@@ -3185,20 +3185,20 @@ tailrecur_ne:
 		b = btmp;
 		goto tailrecur_ne;
 	    }
-	    aa = list_val_rel(atmp,a_base);
-	    bb = list_val_rel(btmp,b_base);
+	    aa = list_val(atmp);
+	    bb = list_val(btmp);
 	}
     case TAG_PRIMARY_BOXED:
 	{
-	    Eterm ahdr = *boxed_val_rel(a,a_base);
+	    Eterm ahdr = *boxed_val(a);
 	    switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
 	    case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):
-		if (!is_tuple_rel(b,b_base)) {
+		if (!is_tuple(b)) {
 		    a_tag = TUPLE_DEF;
 		    goto mixed_types;
 		}
-		aa = tuple_val_rel(a,a_base);
-		bb = tuple_val_rel(b,b_base);
+		aa = tuple_val(a);
+		bb = tuple_val(b);
 		/* compare the arities */
 		i = arityval(ahdr);	/* get the arity*/
 		if (i != arityval(*bb)) {
@@ -3214,18 +3214,18 @@ tailrecur_ne:
 		{
                     struct erts_cmp_hashmap_state* sp;
                     if (is_flatmap_header(ahdr)) {
-                        if (!is_flatmap_rel(b,b_base)) {
-                            if (is_hashmap_rel(b,b_base)) {
-                                aa = (Eterm *)flatmap_val_rel(a,a_base);
-                                i = flatmap_get_size((flatmap_t*)aa) - hashmap_size_rel(b,b_base);
+                        if (!is_flatmap(b)) {
+                            if (is_hashmap(b)) {
+                                aa = (Eterm *)flatmap_val(a);
+                                i = flatmap_get_size((flatmap_t*)aa) - hashmap_size(b);
                                 ASSERT(i != 0);
                                 RETURN_NEQ(i);
                             }
                             a_tag = MAP_DEF;
                             goto mixed_types;
                         }
-                        aa = (Eterm *)flatmap_val_rel(a,a_base);
-                        bb = (Eterm *)flatmap_val_rel(b,b_base);
+                        aa = (Eterm *)flatmap_val(a);
+                        bb = (Eterm *)flatmap_val(b);
 
                         i = flatmap_get_size((flatmap_t*)aa);
                         if (i != flatmap_get_size((flatmap_t*)bb)) {
@@ -3252,21 +3252,21 @@ tailrecur_ne:
                             goto bodyrecur;
                         }
                     }
-		    if (!is_hashmap_rel(b,b_base)) {
-                        if (is_flatmap_rel(b,b_base)) {
-                            bb = (Eterm *)flatmap_val_rel(b,b_base);
-                            i = hashmap_size_rel(a,a_base) - flatmap_get_size((flatmap_t*)bb);
+		    if (!is_hashmap(b)) {
+                        if (is_flatmap(b)) {
+                            bb = (Eterm *)flatmap_val(b);
+                            i = hashmap_size(a) - flatmap_get_size((flatmap_t*)bb);
                             ASSERT(i != 0);
                             RETURN_NEQ(i);
                         }
 			a_tag = MAP_DEF;
 			goto mixed_types;
 		    }
-		    i = hashmap_size_rel(a,a_base) - hashmap_size_rel(b,b_base);
+		    i = hashmap_size(a) - hashmap_size(b);
 		    if (i) {
 			RETURN_NEQ(i);
 		    }
-                    if (hashmap_size_rel(a,a_base) == 0) {
+                    if (hashmap_size(a) == 0) {
                         goto pop_next;
                     }
 
@@ -3301,31 +3301,31 @@ tailrecur_ne:
                     goto bodyrecur;
 		}
 	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
-		if (!is_float_rel(b,b_base)) {
+		if (!is_float(b)) {
 		    a_tag = FLOAT_DEF;
 		    goto mixed_types;
 		} else {
 		    FloatDef af;
 		    FloatDef bf; 
 
-		    GET_DOUBLE_REL(a, af, a_base);
-		    GET_DOUBLE_REL(b, bf, b_base);
+		    GET_DOUBLE(a, af);
+		    GET_DOUBLE(b, bf);
 		    ON_CMP_GOTO(float_comp(af.fd, bf.fd));
 		}
 	    case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
 	    case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
-		if (!is_big_rel(b,b_base)) {
+		if (!is_big(b)) {
 		    a_tag = BIG_DEF;
 		    goto mixed_types;
 		}
-		ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base)));
+		ON_CMP_GOTO(big_comp(a, b));
 	    case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE):
-		if (!is_export_rel(b,b_base)) {
+		if (!is_export(b)) {
 		    a_tag = EXPORT_DEF;
 		    goto mixed_types;
 		} else {
-		    Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
-		    Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
+		    Export* a_exp = *((Export **) (export_val(a) + 1));
+		    Export* b_exp = *((Export **) (export_val(b) + 1));
 
 		    if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
 			RETURN_NEQ(j);
@@ -3337,12 +3337,12 @@ tailrecur_ne:
 		}
 		break;
 	    case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):
-		if (!is_fun_rel(b,b_base)) {
+		if (!is_fun(b)) {
 		    a_tag = FUN_DEF;
 		    goto mixed_types;
 		} else {
-		    ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base);
-		    ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base);
+		    ErlFunThing* f1 = (ErlFunThing *) fun_val(a);
+		    ErlFunThing* f2 = (ErlFunThing *) fun_val(b);
 		    Sint diff;
 
 		    diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name,
@@ -3374,29 +3374,29 @@ tailrecur_ne:
 		if (is_internal_pid(b)) {
 		    bnode = erts_this_node;
 		    bdata = internal_pid_data(b);
-		} else if (is_external_pid_rel(b,b_base)) {
-		    bnode = external_pid_node_rel(b,b_base);
-		    bdata = external_pid_data_rel(b,b_base);
+		} else if (is_external_pid(b)) {
+		    bnode = external_pid_node(b);
+		    bdata = external_pid_data(b);
 		} else {
 		    a_tag = EXTERNAL_PID_DEF;
 		    goto mixed_types;
 		}
-		anode = external_pid_node_rel(a,a_base);
-		adata = external_pid_data_rel(a,a_base);
+		anode = external_pid_node(a);
+		adata = external_pid_data(a);
 		goto pid_common;
 	    case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):
 		if (is_internal_port(b)) {
 		    bnode = erts_this_node;
 		    bdata = internal_port_data(b);
-		} else if (is_external_port_rel(b,b_base)) {
-		    bnode = external_port_node_rel(b,b_base);
-		    bdata = external_port_data_rel(b,b_base);
+		} else if (is_external_port(b)) {
+		    bnode = external_port_node(b);
+		    bdata = external_port_data(b);
 		} else {
 		    a_tag = EXTERNAL_PORT_DEF;
 		    goto mixed_types;
 		}
-		anode = external_port_node_rel(a,a_base);
-		adata = external_port_data_rel(a,a_base);
+		anode = external_port_node(a);
+		adata = external_port_data(a);
 		goto port_common;
 	    case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
 		/*
@@ -3404,14 +3404,13 @@ tailrecur_ne:
 		 * (32-bit words), *not* ref data words.
 		 */
 
-		
-		if (is_internal_ref_rel(b,b_base)) {
-		    RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+		if (is_internal_ref(b)) {
+		    RefThing* bthing = ref_thing_ptr(b);
 		    bnode = erts_this_node;
 		    bnum = internal_thing_ref_numbers(bthing);
 		    blen = internal_thing_ref_no_of_numbers(bthing);
-		} else if(is_external_ref_rel(b,b_base)) {
-		    ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+		} else if(is_external_ref(b)) {
+		    ExternalThing* bthing = external_thing_ptr(b);
 		    bnode = bthing->node;
 		    bnum = external_thing_ref_numbers(bthing);
 		    blen = external_thing_ref_no_of_numbers(bthing);
@@ -3420,7 +3419,7 @@ tailrecur_ne:
 		    goto mixed_types;
 		}
 		{
-		    RefThing* athing = ref_thing_ptr_rel(a,a_base);
+		    RefThing* athing = ref_thing_ptr(a);
 		    anode = erts_this_node;
 		    anum = internal_thing_ref_numbers(athing);
 		    alen = internal_thing_ref_no_of_numbers(athing);
@@ -3453,13 +3452,13 @@ tailrecur_ne:
 			RETURN_NEQ((Sint32) (anum[i] - bnum[i]));
 		goto pop_next;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
-		if (is_internal_ref_rel(b,b_base)) {
-		    RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+		if (is_internal_ref(b)) {
+		    RefThing* bthing = ref_thing_ptr(b);
 		    bnode = erts_this_node;
 		    bnum = internal_thing_ref_numbers(bthing);
 		    blen = internal_thing_ref_no_of_numbers(bthing);
-		} else if (is_external_ref_rel(b,b_base)) {
-		    ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+		} else if (is_external_ref(b)) {
+		    ExternalThing* bthing = external_thing_ptr(b);
 		    bnode = bthing->node;
 		    bnum = external_thing_ref_numbers(bthing);
 		    blen = external_thing_ref_no_of_numbers(bthing);
@@ -3468,7 +3467,7 @@ tailrecur_ne:
 		    goto mixed_types;
 		}
 		{
-		    ExternalThing* athing = external_thing_ptr_rel(a,a_base);
+		    ExternalThing* athing = external_thing_ptr(a);
 		    anode = athing->node;
 		    anum = external_thing_ref_numbers(athing);
 		    alen = external_thing_ref_no_of_numbers(athing);
@@ -3476,13 +3475,13 @@ tailrecur_ne:
 		goto ref_common;
 	    default:
 		/* Must be a binary */
-		ASSERT(is_binary_rel(a,a_base));
-		if (!is_binary_rel(b,b_base)) {
+		ASSERT(is_binary(a));
+		if (!is_binary(b)) {
 		    a_tag = BINARY_DEF;
 		    goto mixed_types;
 		} else {
-		    Uint a_size = binary_size_rel(a,a_base);
-		    Uint b_size = binary_size_rel(b,b_base);
+		    Uint a_size = binary_size(a);
+		    Uint b_size = binary_size(b);
 		    Uint a_bitsize;
 		    Uint b_bitsize;
 		    Uint a_bitoffs;
@@ -3596,7 +3595,7 @@ tailrecur_ne:
 		}
 	    } else {
 		big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm));
-		j = big_comp(aw, rterm2wterm(big,big_buf));
+		j = big_comp(aw, big);
 	    }
 	    if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) {
 		j = -j;
-- 
cgit v1.2.3


From 17bcc73e511eee06ca64d51edb401f8340fe9abc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 14:59:28 +0200
Subject: erts: Remove halfword pointer compression

 * Removed COMPRESS_POINTER and EXPAND_POINTER
---
 erts/emulator/beam/beam_bif_load.c    |  6 ++---
 erts/emulator/beam/beam_emu.c         | 41 ++++++++++++++++-------------------
 erts/emulator/beam/erl_bif_op.c       |  2 +-
 erts/emulator/beam/erl_gc.c           |  4 ++--
 erts/emulator/beam/erl_process.c      |  2 +-
 erts/emulator/beam/erl_process_dump.c |  6 ++---
 erts/emulator/beam/erl_term.h         | 16 ++++++--------
 erts/emulator/beam/external.c         | 36 +++++++++++++++---------------
 8 files changed, 54 insertions(+), 59 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 0e192b1ebd..11508a1b39 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -913,7 +913,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) {
+	    if (in_area(val, mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -933,7 +933,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) {
+	    if (in_area(val, mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -943,7 +943,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
                 if (header_is_bin_matchstate(val)) {
                     ErlBinMatchState *ms = (ErlBinMatchState*) p;
                     ErlBinMatchBuffer *mb = &(ms->mb);
-                    if (in_area(EXPAND_POINTER(mb->orig), mod_start, mod_size)) {
+                    if (in_area(mb->orig, mod_start, mod_size)) {
                         return 1;
                     }
                 }
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0c6181077c..f4111c19f1 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -620,46 +620,45 @@ void** beam_ops;
 
 #define GetTupleElement(Src, Element, Dest)				\
   do {									\
-    tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + 	\
-				(Element));				\
-    (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1));			\
+    tmp_arg1 = (Eterm) (((unsigned char *) tuple_val(Src)) + (Element));\
+    (Dest) = (*(Eterm *) tmp_arg1);	                		\
   } while (0)
 
 #define ExtractNextElement(Dest)					  \
     tmp_arg1 += sizeof(Eterm);						  \
-    (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1))))
+    (Dest) = (* (Eterm *) (((unsigned char *) tmp_arg1)))
 
 #define ExtractNextElement2(Dest)				\
   do {								\
     Eterm* ene_dstp = &(Dest);					\
-    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
-    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
+    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
+    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
     tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);			\
   } while (0)
 
 #define ExtractNextElement3(Dest)		\
   do {						\
     Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
-    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
-    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
+    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
+    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
+    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
     tmp_arg1 += 3*sizeof(Eterm);		\
   } while (0)
 
 #define ExtractNextElement4(Dest)		\
   do {						\
     Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
-    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
-    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
-    ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4];	\
+    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
+    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
+    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
+    ene_dstp[3] = ((Eterm *) tmp_arg1)[4];	\
     tmp_arg1 += 4*sizeof(Eterm);		\
   } while (0)
 
 #define ExtractElement(Element, Dest)		\
   do {						\
      tmp_arg1 += (Element);			\
-     (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1));		\
+     (Dest) = (* (Eterm *) tmp_arg1);		\
   } while (0)
 
 #define EqualImmed(X, Y, Action) if (X != Y) { Action; }
@@ -698,8 +697,7 @@ void** beam_ops;
 
 #define IsArity(Pointer, Arity, Fail)					  \
     if (*(Eterm *)							  \
-	EXPAND_POINTER(tmp_arg1 = (Eterm) 				  \
-		       COMPRESS_POINTER(tuple_val(Pointer))) != (Arity))  \
+	(tmp_arg1 = (Eterm) (tuple_val(Pointer))) != (Arity))             \
     { 									  \
         Fail; 								  \
     }
@@ -742,8 +740,7 @@ void** beam_ops;
   do {									      \
     if (is_not_tuple(Src) || 						      \
 	*(Eterm *)							      \
-	EXPAND_POINTER(tmp_arg1 = 					      \
-		       (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \
+	(tmp_arg1 = (Eterm) (tuple_val(Src))) != Arity) {                     \
         Fail;								      \
     }									      \
   } while (0)
@@ -3218,7 +3215,7 @@ do {								\
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
 	 Dispatch();
@@ -3267,7 +3264,7 @@ do {								\
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(1));
 	 SET_I(next);
 	 Dispatch();
@@ -3299,7 +3296,7 @@ do {								\
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
 	 Dispatchfun();
@@ -3347,7 +3344,7 @@ do {								\
      SWAPIN;
      if (next != NULL) {
 	r(0) = reg[0];
-	SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
+	SET_CP(c_p, (BeamInstr *) E[0]);
 	E = ADD_BYTE_OFFSET(E, Arg(1));
 	SET_I(next);
 	Dispatchfun();
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index c9192fc420..d53a9e11ca 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -258,7 +258,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2)
 	    BIF_RET(am_true);
 	}
     } else if (is_export(arg1)) {
-	Export* exp = (Export *) EXPAND_POINTER((export_val(arg1))[1]);
+	Export* exp = (Export *) (export_val(arg1)[1]);
 
 	if (exp->code[2] == (Uint) arity) {
 	    BIF_RET(am_true);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6c335a9fe6..734f120e09 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1662,7 +1662,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	val = *hp++;
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
-	    ptr = (Eterm *) EXPAND_POINTER(val);
+	    ptr = (Eterm *) val;
 	    if (!in_area(ptr, old_heap, old_heap_size)) {
 		if (in_area(ptr, new_heap, new_heap_size)) {
 		    abort();
@@ -1675,7 +1675,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	    }
 	    break;
 	case TAG_PRIMARY_LIST:
-	    ptr = (Eterm *) EXPAND_POINTER(val);
+	    ptr = (Eterm *) val;
 	    if (!in_area(ptr, old_heap, old_heap_size)) {
 		if (in_area(ptr, new_heap, new_heap_size)) {
 		    abort();
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a21c1e5582..135f09c04d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12501,7 +12501,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
     }
 
     if (is_CP(x)) {
-        erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x));
+        erts_print(to, to_arg, "Return addr %p (", (Eterm *) x);
         print_function_from_pc(to, to_arg, cp_val(x));
         erts_print(to, to_arg, ")\n");
         yreg = 0;
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 25f0b1ed38..3b8ae11e94 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -365,7 +365,7 @@ heap_dump(int to, void *to_arg, Eterm x)
 
     while (x != OUR_NIL) {
 	if (is_CP(x)) {
-	    next = (Eterm *) EXPAND_POINTER(x);
+	    next = (Eterm *) x;
 	} else if (is_list(x)) {
 	    ptr = list_val(x);
 	    if (ptr[0] != OUR_NIL) {
@@ -378,7 +378,7 @@ heap_dump(int to, void *to_arg, Eterm x)
 		    ptr[1] = make_small(0);
 		}
 		x = ptr[0];
-		ptr[0] = (Eterm) COMPRESS_POINTER(next);
+		ptr[0] = (Eterm) next;
 		next = ptr + 1;
 		continue;
 	    }
@@ -408,7 +408,7 @@ heap_dump(int to, void *to_arg, Eterm x)
 			ptr[0] = OUR_NIL;
 		    } else {
 			x = ptr[arity];
-			ptr[0] = (Eterm) COMPRESS_POINTER(next);
+			ptr[0] = (Eterm) next;
 			next = ptr + arity - 1;
 			continue;
 		    }
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index ad4ef3c01a..cc0b187240 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -25,8 +25,6 @@ typedef UWord Wterm;  /* Full word terms */
 
 #define HEAP_ON_C_STACK 1
 #define CHECK_POINTER_MASK 0x0UL
-#define COMPRESS_POINTER(AnUint) (AnUint)
-#define EXPAND_POINTER(APointer) (APointer)
 
 struct erl_node_; /* Declared in erl_node_tables.h */
 
@@ -174,7 +172,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define  _boxed_precond(x)       (is_boxed(x))
 
 #define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
-#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED)
+#define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED)
 _ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*)
 #define make_boxed(x)		_ET_APPLY(make_boxed,(x))
 #if 1
@@ -185,12 +183,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm)
 #else
 #define is_boxed(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
 #endif
-#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED)))
+#define _unchecked_boxed_val(x) ((Eterm*) ((x) - TAG_PRIMARY_BOXED))
 _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
 #define boxed_val(x)		_ET_APPLY(boxed_val,(x))
 
 /* cons cell ("list") access methods */
-#define _unchecked_make_list(x)	((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST)
+#define _unchecked_make_list(x)	((Uint)(x) + TAG_PRIMARY_LIST)
 _ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*)
 #define make_list(x)		_ET_APPLY(make_list,(x))
 #if 1
@@ -203,7 +201,7 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
 #define is_not_list(x)		(!is_list((x)))
 #endif
 #define _list_precond(x)        (is_list(x))
-#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST))
+#define _unchecked_list_val(x) ((Eterm*) ((x) - TAG_PRIMARY_LIST))
 _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define list_val(x)		_ET_APPLY(list_val,(x))
 
@@ -214,7 +212,7 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define CDR(x)  ((x)[1])
 
 /* generic tagged pointer (boxed or list) access methods */
-#define _unchecked_ptr_val(x)	((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3)))
+#define _unchecked_ptr_val(x)	((Eterm*) ((x) & ~((Uint) 0x3)))
 #define ptr_val(x)		_unchecked_ptr_val((x))	/*XXX*/
 #define _unchecked_offset_ptr(x,offs)	((x)+((offs)*sizeof(Eterm)))
 #define offset_ptr(x,offs)	_unchecked_offset_ptr(x,offs)	/*XXX*/
@@ -1009,14 +1007,14 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #error "fix yer arch, like"
 #endif
 
-#define _unchecked_make_cp(x)	((Eterm) COMPRESS_POINTER(x))
+#define _unchecked_make_cp(x)	((Eterm)(x))
 _ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*)
 #define make_cp(x)	_ET_APPLY(make_cp,(x))
 
 #define is_not_CP(x)	((x) & _CPMASK)
 #define is_CP(x)	(!is_not_CP(x))
 
-#define _unchecked_cp_val(x)	((BeamInstr*) EXPAND_POINTER(x))
+#define _unchecked_cp_val(x)	((BeamInstr*) (x))
 _ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm)
 #define cp_val(x)	_ET_APPLY(cp_val,(x))
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index f7b372d294..c3e93d1ad2 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2958,7 +2958,7 @@ dec_term(ErtsDistExternal *edep,
             case B2TDecodeList:
                 objp = next - 2;
                 while (n > 0) {
-                    objp[0] = (Eterm) COMPRESS_POINTER(next);
+                    objp[0] = (Eterm) next;
                     objp[1] = make_list(next);
                     next = objp;
                     objp -= 2;
@@ -2969,7 +2969,7 @@ dec_term(ErtsDistExternal *edep,
             case B2TDecodeTuple:
                 objp = next - 1;
                 while (n-- > 0) {
-                    objp[0] = (Eterm) COMPRESS_POINTER(next);
+                    objp[0] = (Eterm) next;
                     next = objp;
                     objp--;
                 }
@@ -3022,7 +3022,7 @@ dec_term(ErtsDistExternal *edep,
     while (next != NULL) {
 
 	objp = next;
-	next = (Eterm *) EXPAND_POINTER(*objp);
+	next = (Eterm *) *objp;
 
 	switch (*ep++) {
 	case INTEGER_EXT:
@@ -3153,7 +3153,7 @@ dec_term_atom_common:
 		reds -= n;
 	    }
 	    while (n-- > 0) {
-		objp[0] = (Eterm) COMPRESS_POINTER(next);
+		objp[0] = (Eterm) next;
 		next = objp;
 		objp--;
 	    }
@@ -3171,8 +3171,8 @@ dec_term_atom_common:
 	    *objp = make_list(hp);
             hp += 2 * n;
 	    objp = hp - 2;
-	    objp[0] = (Eterm) COMPRESS_POINTER((objp+1));
-	    objp[1] = (Eterm) COMPRESS_POINTER(next);
+	    objp[0] = (Eterm) (objp+1);
+	    objp[1] = (Eterm) next;
 	    next = objp;
 	    objp -= 2;
             n--;
@@ -3185,7 +3185,7 @@ dec_term_atom_common:
 		reds -= n;
 	    }
             while (n > 0) {
-		objp[0] = (Eterm) COMPRESS_POINTER(next);
+		objp[0] = (Eterm) next;
 		objp[1] = make_list(next);
 		next = objp;
 		objp -= 2;
@@ -3576,7 +3576,7 @@ dec_term_atom_common:
                      * The list of maps is for later validation.
                      */
 
-                    mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list);
+                    mp->thing_word = (Eterm) maps_list;
                     maps_list      = (Eterm *) mp;
 
                     mp->size       = size;
@@ -3584,8 +3584,8 @@ dec_term_atom_common:
                     *objp          = make_flatmap(mp);
 
                     for (n = size; n; n--) {
-                        *vptr = (Eterm) COMPRESS_POINTER(next);
-                        *kptr = (Eterm) COMPRESS_POINTER(vptr);
+                        *vptr = (Eterm) next;
+                        *kptr = (Eterm) vptr;
                         next  = kptr;
                         vptr--;
                         kptr--;
@@ -3599,8 +3599,8 @@ dec_term_atom_common:
                     hamt->leaf_array = hp;
 
                     for (n = size; n; n--) {
-                        CDR(hp) = (Eterm) COMPRESS_POINTER(next);
-                        CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp));
+                        CDR(hp) = (Eterm) next;
+                        CAR(hp) = (Eterm) &CDR(hp);
                         next = &CAR(hp);
                         hp += 2;
                     }
@@ -3677,11 +3677,11 @@ dec_term_atom_common:
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
-		    funp->env[i] = (Eterm) COMPRESS_POINTER(next);
+		    funp->env[i] = (Eterm) next;
 		    next = funp->env + i;
 		}
 		/* Creator */
-		funp->creator = (Eterm) COMPRESS_POINTER(next);
+		funp->creator = (Eterm) next;
 		next = &(funp->creator);
 		break;
 	    }
@@ -3750,7 +3750,7 @@ dec_term_atom_common:
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
-		    funp->env[i] = (Eterm) COMPRESS_POINTER(next);
+		    funp->env[i] = (Eterm) next;
 		    next = funp->env + i;
 		}
 		break;
@@ -3846,11 +3846,11 @@ dec_term_atom_common:
      */
 
     while (maps_list) {
-	next  = (Eterm *)(EXPAND_POINTER(*maps_list));
+	next = (Eterm *) *maps_list;
 	*maps_list = MAP_HEADER_FLATMAP;
 	if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list))
 	    goto error;
-	maps_list  = next;
+	maps_list = next;
     }
 
     ASSERT(hp <= factory->hp_end
@@ -3876,7 +3876,7 @@ dec_term_atom_common:
 	PSTACK_DESTROY(hamt_array);
     }
 
-    ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL);
+    ASSERT((Eterm*)*dbg_resultp != NULL);
 
     if (ctx) {
         ctx->state = B2TDone;
-- 
cgit v1.2.3


From 287db3cf7ecb1bb23664cf872508675461a4be56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 15:19:20 +0200
Subject: erts: Remove halfword heap relative comparisions

 * Removed cmp_rel, cmp_rel_term and eq_rel
---
 erts/emulator/beam/erl_db_hash.c |  7 ++-----
 erts/emulator/beam/erl_db_tree.c | 13 ++++++-------
 erts/emulator/beam/erl_db_util.c | 10 +++++-----
 erts/emulator/beam/erl_db_util.h |  4 ++--
 erts/emulator/beam/erl_map.c     |  4 ++--
 erts/emulator/beam/erl_utils.h   |  3 ---
 6 files changed, 17 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 80563e1f02..e4f4a7beb0 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -460,9 +460,6 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb)
     }
 }	
 
-#define EQ_REL(x,y,y_base) \
-    (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base)))
-
 /* Is this a live object (not pseodo-deleted) with the specified key? 
 */
 static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
@@ -472,7 +469,7 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
 	ASSERT(!is_header(itemKey));
-	return EQ_REL(key, itemKey, b->dbterm.tpl);
+	return EQ(key, itemKey);
     }
 }
 
@@ -485,7 +482,7 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b,
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
 	ASSERT(!is_header(itemKey));
-	return EQ_REL(key, itemKey, b->dbterm.tpl);
+	return EQ(key, itemKey);
     }
 }
 
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 3b698a6adf..d9eac550af 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -576,8 +576,7 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base,
 			       TreeDbTerm* obj)
 {
-    return cmp_rel(key, key_base,
-		   GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl);
+    return CMP(key, GETKEY(tb,obj->dbterm.tpl));
 }
 
 static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base,
@@ -585,7 +584,7 @@ static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base,
 {
     Eterm obj_key = GETKEY(tb,obj->dbterm.tpl);
     return is_same(key, key_base, obj_key, obj->dbterm.tpl)
-	|| cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0;
+	|| CMP(key, obj_key) == 0;
 }
 
 static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
@@ -2742,7 +2741,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
     switch (a & _TAG_PRIMARY_MASK) {
     case TAG_PRIMARY_LIST:
 	if (!is_list(b)) {
-	    return cmp_rel(a,NULL,b,b_base);
+	    return CMP(a,b);
 	}
 	aa = list_val(a);
 	bb = list_val(b);
@@ -2758,12 +2757,12 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	}
     case TAG_PRIMARY_BOXED:
 	if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) {
-	    return cmp_rel(a,NULL,b,b_base);
+	    return CMP(a,b);
 	}
 	a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
 	b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
 	if (a_hdr != b_hdr) {
-	    return cmp_rel(a, NULL, b, b_base);
+	    return CMP(a,b);
 	}
 	if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) {
 	    aa = tuple_val(a);
@@ -2781,7 +2780,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	}
 	/* Drop through */
       default:
-	  return cmp_rel(a, NULL, b, b_base);
+	  return CMP(a,b);
     }
 }
 
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 432ca297e9..7bf45bd586 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1990,13 +1990,13 @@ restart:
 	    break;
 	case matchCmp:
 	    n = *pc++;
-	    if (!eq_rel(variables[n].term, base, *ep, base))
+	    if (!EQ(variables[n].term, *ep))
 		FAIL();
 	    ++ep;
 	    break;
 	case matchEqBin:
 	    t = (Eterm) *pc++;
-	    if (!eq_rel(t,NULL,*ep,base))
+	    if (!EQ(t,*ep))
 		FAIL();
 	    ++ep;
 	    break;
@@ -2012,7 +2012,7 @@ restart:
 	    Eterm* epc = (Eterm*)pc;
 	    if (!is_ref(*ep))
 		FAIL();
-	    if (!eq_rel(make_internal_ref(epc), epc, *ep, base)) {
+	    if (!EQ(make_internal_ref(epc), *ep)) {
 		FAIL();
 	    }
 	    i = thing_arityval(*epc);
@@ -3063,7 +3063,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
 
     ASSERT((*hpp - hp) <= bp->size);
 #ifdef DEBUG_CLONE
-    ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone));
+    ASSERT(EQ(make_tuple(hp),make_tuple(bp->debug_clone)));
 #endif
     return make_tuple(hp);
 }
@@ -3087,7 +3087,7 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p,
 	*hpp = erts_produce_heap(&factory, extra, 0);
         erts_factory_close(&factory);
 #ifdef DEBUG_CLONE
-	ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone));
+	ASSERT(EQ(copy, obj->debug_clone[pos]));
 #endif
 	return copy;
     }
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index d07e61e599..36c15496b9 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -287,7 +287,7 @@ ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj)
 	Uint size = size_object_rel(key, obj->tpl);
 	Eterm* hp = HAlloc(p, size);
 	Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL);
-	ASSERT(eq_rel(res,NULL,key,obj->tpl));
+	ASSERT(EQ(res,key));
 	return res;
     }
 }
@@ -306,7 +306,7 @@ ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
 ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b)
 {
     if (!tb->compress) {
-	return eq_rel(a, NULL, make_tuple(b->tpl), b->tpl);
+	return EQ(a, make_tuple(b->tpl));
     }
     else {
 	return db_eq_comp(tb, a, b);
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 500234c436..8f376fe595 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -197,7 +197,7 @@ erts_maps_get(Eterm key, Eterm map)
 	}
 
 	for (i = 0; i < n; i++) {
-	    if (eq_rel(ks[i], map_base, key, NULL)) {
+	    if (EQ(ks[i], key)) {
 		return &vs[i];
 	    }
 	}
@@ -2020,7 +2020,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
 
         if (is_list(node)) { /* LEAF NODE [K|V] */
             ptr = list_val(node);
-            res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL;
+            res = EQ(CAR(ptr), key) ? &(CDR(ptr)) : NULL;
             break;
         }
 
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 1732579d04..4058d63eaf 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -158,14 +158,11 @@ erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint);
 void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *);
 
 int eq(Eterm, Eterm);
-#define eq_rel(A,A_BASE,B,B_BASE) eq(A,B)
 
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
 Sint erts_cmp(Eterm, Eterm, int, int);
 Sint cmp(Eterm a, Eterm b);
-#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp(A,B,0,0)
-#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp(A,B,1,0)
 #define CMP(A,B)                         erts_cmp(A,B,0,0)
 #define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
 #define CMP_EQ_ONLY(A,B)                 erts_cmp(A,B,0,1)
-- 
cgit v1.2.3


From 256c60d885b6fe0e8a715fa9bfdd371ad14c8857 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 16:01:10 +0200
Subject: erts: Remove halfword object manipulation

 * Remove macros size_object_rel, copy_struct_rel and copy_shallow_rel
---
 erts/emulator/beam/erl_db_tree.c | 36 ++++++++++++++++++------------------
 erts/emulator/beam/erl_db_util.c | 33 +++++++++++++++------------------
 erts/emulator/beam/erl_db_util.h |  6 +++---
 erts/emulator/beam/global.h      |  7 -------
 4 files changed, 36 insertions(+), 46 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index d9eac550af..0c0218f5f9 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1002,9 +1002,9 @@ static int db_select_continue_tree(Process *p,
 	    }
 
 	    key = GETKEY(tb, sc.lastobj);
-	    sz = size_object_rel(key,sc.lastobj);
+	    sz = size_object(key);
 	    hp = HAlloc(p, 9 + sz);
-	    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
+	    key = copy_struct(key, sz, &hp, &MSO(p));
 	    continuation = TUPLE8
 		(hp,
 		 tptr[1],
@@ -1045,9 +1045,9 @@ static int db_select_continue_tree(Process *p,
 	}
     }
     /* Not done yet, let's trap. */
-    sz = size_object_rel(key,sc.lastobj);
+    sz = size_object(key);
     hp = HAlloc(p, 9 + sz);
-    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
+    key = copy_struct(key, sz, &hp, &MSO(p));
     continuation = TUPLE8
 	(hp,
 	 tptr[1],
@@ -1152,9 +1152,9 @@ static int db_select_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object_rel(key, sc.lastobj);
+    sz = size_object(key);
     hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
+    key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
     mpb=db_make_mp_binary(p,mpi.mp,&hp);
@@ -1250,7 +1250,7 @@ static int db_select_count_continue_tree(Process *p,
 	RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE);
     }
     /* Not done yet, let's trap. */
-    sz = size_object_rel(key, sc.lastobj);
+    sz = size_object(key);
     if (IS_USMALL(0, sc.got)) {
 	hp = HAlloc(p, sz + 6);
 	egot = make_small(sc.got);
@@ -1260,7 +1260,7 @@ static int db_select_count_continue_tree(Process *p,
 	egot = uint_to_big(sc.got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
+    key = copy_struct(key, sz, &hp, &MSO(p));
     continuation = TUPLE5
 	(hp,
 	 tptr[1],
@@ -1346,7 +1346,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object_rel(key, sc.lastobj);
+    sz = size_object(key);
     if (IS_USMALL(0, sc.got)) {
 	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
 	egot = make_small(sc.got);
@@ -1356,7 +1356,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
 	egot = uint_to_big(sc.got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
+    key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1482,9 +1482,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 	}
 
 	key = GETKEY(tb, sc.lastobj);
-	sz = size_object_rel(key, sc.lastobj);
+	sz = size_object(key);
 	hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-	key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
+	key = copy_struct(key, sz, &hp, &MSO(p));
 	if (mpi.all_objects)
 	    (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
 	mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1507,9 +1507,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object_rel(key, sc.lastobj);
+    sz = size_object(key);
     hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
+    key = copy_struct(key, sz, &hp, &MSO(p));
 
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
@@ -1598,7 +1598,7 @@ static int db_select_delete_continue_tree(Process *p,
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
     }
     /* Not done yet, let's trap. */
-    sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
+    sz = size_object(key);
     if (IS_USMALL(0, sc.accum)) {
 	hp = HAlloc(p, sz + 6);
 	eaccsum = make_small(sc.accum);
@@ -1608,7 +1608,7 @@ static int db_select_delete_continue_tree(Process *p,
 	eaccsum = uint_to_big(sc.accum, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
+    key = copy_struct(key, sz, &hp, &MSO(p));
     continuation = TUPLE5
 	(hp,
 	 tptr[1],
@@ -1695,7 +1695,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
-    sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
+    sz = size_object(key);
     if (IS_USMALL(0, sc.accum)) {
 	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
 	eaccsum = make_small(sc.accum);
@@ -1705,7 +1705,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 	eaccsum = uint_to_big(sc.accum, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
+    key = copy_struct(key, sz, &hp, &MSO(p));
     mpb = db_make_mp_binary(p,mpi.mp,&hp);
     
     continuation = TUPLE5
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 7bf45bd586..41b32fc0e7 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1751,9 +1751,9 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
 static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base)
 {
     if (!is_immed(term)) {
-	Uint sz = size_object_rel(term, base);
+	Uint sz = size_object(term);
 	Eterm* top = HAllocX(p, sz, HEAP_XTRA);
-	return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL);
+	return copy_struct(term, sz, &top, &MSO(p));
     }
     return term;
 }
@@ -2190,16 +2190,14 @@ restart:
 	    if (in_flags & ERTS_PAM_COPY_RESULT) {
 		Uint sz;
 		Eterm* top;
-		sz = size_object_rel(term, base);
+		sz = size_object(term);
 		top = HAllocX(build_proc, sz, HEAP_XTRA);
 		if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) {
 		    ASSERT(is_tuple(term));
-		    *esp++ = copy_shallow_rel(tuple_val(term), sz,
-					      &top, &MSO(build_proc), base);
+		    *esp++ = copy_shallow(tuple_val(term), sz, &top, &MSO(build_proc));
 		}
 		else {
-		    *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc),
-					     base, NULL);
+		    *esp++ = copy_struct(term, sz, &top, &MSO(build_proc));
 		}
 	    }
 	    else {
@@ -2767,7 +2765,7 @@ void db_do_update_element(DbUpdateHandle* handle,
     newval_sz = is_immed(newval) ? 0 : size_object(newval);
 new_size_set:
 
-    oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base);
+    oldval_sz = is_immed(oldval) ? 0 : size_object(oldval);
 both_size_set:
 
     handle->new_size = handle->new_size - oldval_sz + newval_sz;
@@ -2873,7 +2871,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest,
     tpl[arity + 1] = alloc_size;
 
     tmp_offheap.first = NULL;
-    tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl);
+    tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap);
     dest->first_oh = tmp_offheap.first;
     for (i=1; i<=arity; i++) {
 	if (i != tb->keypos) {
@@ -2892,7 +2890,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest,
 	Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm));
 	dest->debug_clone = dbg_top;
 	tmp_offheap.first = dest->first_oh;
-	copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top);
+	copy_struct(obj, dest->size, &dbg_top, &tmp_offheap);
 	dest->first_oh = tmp_offheap.first;
 	ASSERT(dbg_top == dest->debug_clone + dest->size);
     }
@@ -2939,7 +2937,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
     newp->size = size;
     top = newp->tpl;
     tmp_offheap.first  = NULL;
-    copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top);
+    copy_struct(obj, size, &top, &tmp_offheap);
     newp->first_oh = tmp_offheap.first;
 #ifdef DEBUG_CLONE
     newp->debug_clone = NULL;
@@ -3023,8 +3021,7 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset)
 	tmp_offheap.first = NULL;
 
 	{
-	    copy_struct_rel(make_tuple(tpl), handle->new_size, &top,
-			    &tmp_offheap, tpl, top);
+	    copy_struct(make_tuple(tpl), handle->new_size, &top, &tmp_offheap);
 	    newDbTerm->first_oh = tmp_offheap.first;
 	    ASSERT((byte*)top == (newp + alloc_sz));
 	}
@@ -3041,9 +3038,9 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
     hp[0] = bp->tpl[0];
     *hpp += arity + 1;
 
-    hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos],
-				     size_object_rel(bp->tpl[tb->keypos], bp->tpl),
-				     hpp, off_heap, bp->tpl, NULL);
+    hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos],
+                                 size_object(bp->tpl[tb->keypos]),
+                                 hpp, off_heap);
 
     erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap);
 
@@ -3092,9 +3089,9 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p,
 	return copy;
     }
     else {
-	Uint sz = size_object_rel(obj->tpl[pos], obj->tpl);
+	Uint sz = size_object(obj->tpl[pos]);
 	*hpp = HAlloc(p, sz + extra);
-	return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL);
+	return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p));
     }
 }
 
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 36c15496b9..10899bb3e7 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -284,9 +284,9 @@ ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj)
     Eterm key = GETKEY(tb, obj->tpl);
     if IS_CONST(key) return key;
     else {
-	Uint size = size_object_rel(key, obj->tpl);
+	Uint size = size_object(key);
 	Eterm* hp = HAlloc(p, size);
-	Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL);
+	Eterm res = copy_struct(key, size, &hp, &MSO(p));
 	ASSERT(EQ(res,key));
 	return res;
     }
@@ -299,7 +299,7 @@ ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
 	return db_copy_from_comp(tb, bp, hpp, off_heap);
     }
     else {
-	return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl);
+	return copy_shallow(bp->tpl, bp->size, hpp, off_heap);
     }
 }
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 1d2d75bc1e..3559f0cd13 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -955,16 +955,9 @@ void erl_error(char*, va_list);
 
 /* copy.c */
 Eterm copy_object(Eterm, Process*);
-
 Uint size_object(Eterm);
-#define size_object_rel(A,B) size_object(A)
-
 Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
-#define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH)
-
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
-#define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D)
-
 
 void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
 		      Eterm* refs, unsigned nrefs);
-- 
cgit v1.2.3


From ba020bd06c34eaf5b450495a852f31357ef042b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 16:53:49 +0200
Subject: erts: Remove halfword copy_object_rel

Near duplication of copy_object but with base ptr that is no longer used.
---
 erts/emulator/beam/erl_db_util.c | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 41b32fc0e7..ae9a853411 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1748,17 +1748,6 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
     return ret;
 }
 
-static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base)
-{
-    if (!is_immed(term)) {
-	Uint sz = size_object(term);
-	Eterm* top = HAllocX(p, sz, HEAP_XTRA);
-	return copy_struct(term, sz, &top, &MSO(p));
-    }
-    return term;
-}
-
-
 /*
 ** Execution of the match program, this is Pam.
 ** May return THE_NON_VALUE, which is a bailout.
@@ -2167,12 +2156,11 @@ restart:
 	    break;
 	case matchPushVResult:
 	    if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV;
-
 	    /* Build (NULL-based) copy on callers heap */
 	    n = *pc++;
 	    ASSERT(is_value(variables[n].term));
 	    ASSERT(!variables[n].proc);
-	    variables[n].term = copy_object_rel(c_p, variables[n].term, base);
+	    variables[n].term = copy_object(variables[n].term, c_p);
 	    *esp++ = variables[n].term;
 	    #ifdef DEBUG
 	    variables[n].proc = c_p;
-- 
cgit v1.2.3


From d6712b1c54de471beb1784bb329ea217767f70ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 17:03:48 +0200
Subject: erts: Reinstate copy_object over-allocation optimization

---
 erts/emulator/beam/copy.c        | 35 ++++++++++++++++++-----------------
 erts/emulator/beam/erl_db_util.c |  2 +-
 erts/emulator/beam/global.h      |  4 +++-
 3 files changed, 22 insertions(+), 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 38c40f4e7d..3de2c10203 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -40,29 +40,30 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*);
 /*
  *  Copy object "obj" to process p.
  */
-Eterm
-copy_object(Eterm obj, Process* to)
-{
-    Uint size = size_object(obj);
-    Eterm* hp = HAlloc(to, size);
-    Eterm res;
+Eterm copy_object_x(Eterm obj, Process* to, Uint extra) {
+    if (!is_immed(obj)) {
+        Uint size = size_object(obj);
+        Eterm* hp = HAllocX(to, size, extra);
+        Eterm res;
 
 #ifdef USE_VM_PROBES
-    if (DTRACE_ENABLED(copy_object)) {
-        DTRACE_CHARBUF(proc_name, 64);
+        if (DTRACE_ENABLED(copy_object)) {
+            DTRACE_CHARBUF(proc_name, 64);
 
-        erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)),
-                      "%T", to->common.id);
-        DTRACE2(copy_object, proc_name, size);
-    }
+            erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)),
+                    "%T", to->common.id);
+            DTRACE2(copy_object, proc_name, size);
+        }
 #endif
-    res = copy_struct(obj, size, &hp, &to->off_heap);
+        res = copy_struct(obj, size, &hp, &to->off_heap);
 #ifdef DEBUG
-    if (eq(obj, res) == 0) {
-	erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n");
-    }
+        if (eq(obj, res) == 0) {
+            erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n");
+        }
 #endif
-    return res;
+        return res;
+    }
+    return obj;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index ae9a853411..8047711e6f 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2160,7 +2160,7 @@ restart:
 	    n = *pc++;
 	    ASSERT(is_value(variables[n].term));
 	    ASSERT(!variables[n].proc);
-	    variables[n].term = copy_object(variables[n].term, c_p);
+	    variables[n].term = copy_object_x(variables[n].term, c_p, HEAP_XTRA);
 	    *esp++ = variables[n].term;
 	    #ifdef DEBUG
 	    variables[n].proc = c_p;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 3559f0cd13..870afb414f 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -954,7 +954,9 @@ __decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...);
 void erl_error(char*, va_list);
 
 /* copy.c */
-Eterm copy_object(Eterm, Process*);
+Eterm copy_object_x(Eterm, Process*, Uint);
+#define copy_object(Term, Proc) copy_object_x(Term,Proc,0)
+
 Uint size_object(Eterm);
 Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
-- 
cgit v1.2.3


From 64aa348700cc380f3525be01d3c815f6ecb398cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 17:33:14 +0200
Subject: erts: Remove halfword is_same bases macro

Keep is_same macro for readability but remove base pointers.
---
 erts/emulator/beam/erl_db_tree.c |  6 +++---
 erts/emulator/beam/erl_term.h    |  2 +-
 erts/emulator/beam/utils.c       | 16 ++++++++--------
 3 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 0c0218f5f9..d7c4e7e5ba 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -583,7 +583,7 @@ static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base,
 				  TreeDbTerm* obj)
 {
     Eterm obj_key = GETKEY(tb,obj->dbterm.tpl);
-    return is_same(key, key_base, obj_key, obj->dbterm.tpl)
+    return is_same(key, obj_key)
 	|| CMP(key, obj_key) == 0;
 }
 
@@ -2735,7 +2735,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	*done = 1;
 	return 0;
     }
-    if (is_same(a,NULL,b,b_base))
+    if (is_same(a,b))
 	return 0;
     
     switch (a & _TAG_PRIMARY_MASK) {
@@ -2748,7 +2748,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	while (1) {
 	    if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done)
 		return j;
-	    if (is_same(*aa, NULL, *bb, b_base))
+	    if (is_same(*aa, *bb))
 		return 0;
 	    if (is_not_list(*aa) || is_not_list(*bb))
 		return do_cmp_partly_bound(*aa, *bb, b_base, done);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index cc0b187240..95ced40af7 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1118,7 +1118,7 @@ extern unsigned tag_val_def(Wterm);
 #define FLOAT_BIG 	_NUMBER_CODE(FLOAT_DEF,BIG_DEF)
 #define FLOAT_FLOAT	_NUMBER_CODE(FLOAT_DEF,FLOAT_DEF)
 
-#define is_same(A,A_BASE,B,B_BASE) ((A)==(B))
+#define is_same(A,B) ((A)==(B))
 
 #endif	/* __ERL_TERM_H */
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 9f919fa2ab..acd6cf1072 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2606,7 +2606,7 @@ int eq(Eterm a, Eterm b)
     Eterm* bb;
 
 tailrecur:
-    if (is_same(a, a_base, b, b_base)) goto pop_next;
+    if (is_same(a, b)) goto pop_next;
 tailrecur_ne:
 
     switch (primary_tag(a)) {
@@ -2617,7 +2617,7 @@ tailrecur_ne:
 	    while (1) {
 		Eterm atmp = CAR(aval);
 		Eterm btmp = CAR(bval);
-		if (!is_same(atmp,a_base,btmp,b_base)) {
+		if (!is_same(atmp,btmp)) {
 		    WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval));
 		    a = atmp;
 		    b = btmp;
@@ -2625,7 +2625,7 @@ tailrecur_ne:
 		}
 		atmp = CDR(aval);
 		btmp = CDR(bval);
-		if (is_same(atmp,a_base,btmp,b_base)) {
+		if (is_same(atmp,btmp)) {
 		    goto pop_next;
 		}
 		if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -2899,7 +2899,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */
 	Eterm* bp = bb;
 	Sint i = sz;
 	for (;;) {
-	    if (!is_same(*ap,a_base,*bp,b_base)) break;
+	    if (!is_same(*ap,*bp)) break;
 	    if (--i == 0) goto pop_next;
 	    ++ap;
 	    ++bp;
@@ -3085,7 +3085,7 @@ static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
 bodyrecur:
     j = 0;
 tailrecur:
-    if (is_same(a,a_base,b,b_base)) {	/* Equal values or pointers. */
+    if (is_same(a,b)) {	/* Equal values or pointers. */
 	goto pop_next;
     }
 tailrecur_ne:
@@ -3169,7 +3169,7 @@ tailrecur_ne:
 	while (1) {
 	    Eterm atmp = CAR(aa);
 	    Eterm btmp = CAR(bb);
-	    if (!is_same(atmp,a_base,btmp,b_base)) {
+	    if (!is_same(atmp,btmp)) {
 		WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa));
 		a = atmp;
 		b = btmp;
@@ -3177,7 +3177,7 @@ tailrecur_ne:
 	    }
 	    atmp = CDR(aa);
 	    btmp = CDR(bb);
-	    if (is_same(atmp,a_base,btmp,b_base)) {
+	    if (is_same(atmp,btmp)) {
 		goto pop_next;
 	    }
 	    if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -3643,7 +3643,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
     while (--i) {
 	a = *aa++;
 	b = *bb++;
-	if (!is_same(a,a_base, b,b_base)) {
+	if (!is_same(a, b)) {
 	    if (is_atom(a) && is_atom(b)) {
 		if ((j = cmp_atoms(a, b)) != 0) {
 		    goto not_equal;
-- 
cgit v1.2.3


From af8bb5e30b021b7b8e80ed96f3f3e16c18b865e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 17:43:59 +0200
Subject: erts: Remove halfword BINARY RELs

 * ERTS_GET_BINARY_BYTES_REL
 * ERTS_GET_REAL_BIN_REL
---
 erts/emulator/beam/copy.c            | 2 +-
 erts/emulator/beam/erl_binary.h      | 8 +-------
 erts/emulator/beam/erl_printf_term.c | 2 +-
 erts/emulator/beam/utils.c           | 8 ++++----
 4 files changed, 7 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 3de2c10203..ec769c3b49 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -179,7 +179,7 @@ Uint size_object(Eterm obj)
 			Uint bitoffs;
 			Uint extra_bytes;
 			Eterm hdr;
-			ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base);
+			ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
 			if ((bitsize + bitoffs) > 8) {
 			    sum += ERL_SUB_BIN_SIZE;
 			    extra_bytes = 2;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index d24d041530..e181b5555d 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -94,10 +94,7 @@ typedef struct erl_heap_bin {
  * Bitsize: output variable (Uint)
  */
 
-#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \
-     ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL)
-
-#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr)    \
+#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize)                \
 do {									\
     Eterm* _real_bin = binary_val(Bin);		                	\
     Uint _offs = 0;							\
@@ -130,9 +127,6 @@ do {									\
  */
 
 #define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \
-     ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL)
-
-#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \
   do {									\
     ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin);	                \
     if (_sb->thing_word == HEADER_SUB_BIN) {				\
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 7a40f775f2..5aef2c76a4 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -472,7 +472,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		Uint bitoffs;
 		Uint bitsize;
 		byte octet;
-		ERTS_GET_BINARY_BYTES_REL(obj, bytep, bitoffs, bitsize, obj_base);
+		ERTS_GET_BINARY_BYTES(obj, bytep, bitoffs, bitsize);
 
 		if (bitsize || !bytesize
 		    || !is_printable_ascii(bytep, bytesize, bitoffs)) {
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index acd6cf1072..a741e2e2e6 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2675,8 +2675,8 @@ tailrecur_ne:
 		    if (a_size != b_size) {
 			goto not_equal;
 		    }
-		    ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
-		    ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
+		    ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
+		    ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
 		    if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
 			if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next;
 		    } else if (a_bitsize == b_bitsize) {
@@ -3490,8 +3490,8 @@ tailrecur_ne:
 		    int cmp;
 		    byte* a_ptr;
 		    byte* b_ptr;
-		    ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
-		    ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
+		    ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
+		    ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
 		    if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
 			min_size = (a_size < b_size) ? a_size : b_size;
 			if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) {
-- 
cgit v1.2.3


From 0cec9781bc01c7e51ecfcb2fc4f356f2cc59b03c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Jun 2015 17:50:53 +0200
Subject: erts: Remove halfword specific allocator types

---
 erts/emulator/beam/erl_alloc.types | 36 ------------------------------------
 1 file changed, 36 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index b1d511ab78..1d5dec4807 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -87,11 +87,6 @@ allocator	EHEAP		true		eheap_alloc
 allocator	ETS		true		ets_alloc
 allocator	FIXED_SIZE	true		fix_alloc
 
-+if halfword
-allocator	LONG_LIVED_LOW	true		ll_low_alloc
-allocator	STANDARD_LOW	true		std_low_alloc
-+endif
-
 +else # Non smp build
 
 allocator	TEMPORARY	false		temp_alloc
@@ -102,11 +97,6 @@ allocator	EHEAP		false		eheap_alloc
 allocator	ETS		false		ets_alloc
 allocator	FIXED_SIZE	false		fix_alloc
 
-+if halfword
-allocator	LONG_LIVED_LOW	false		ll_low_alloc
-allocator	STANDARD_LOW	false		std_low_alloc
-+endif
-
 +endif
 
 allocator	BINARY		true		binary_alloc
@@ -349,29 +339,6 @@ type	SSB		SHORT_LIVED	PROCESSES	ssb
 
 +endif
 
-
-+if halfword
-
-type	DDLL_PROCESS	STANDARD_LOW	SYSTEM		ddll_processes
-type	MONITOR_LH	STANDARD_LOW	PROCESSES	monitor_lh
-type	NLINK_LH	STANDARD_LOW	PROCESSES	nlink_lh
-type	CODE		LONG_LIVED_LOW	CODE		code
-type	DB_HEIR_DATA	STANDARD_LOW	ETS		db_heir_data
-type	DB_MS_PSDO_PROC	LONG_LIVED_LOW	ETS		db_match_pseudo_proc
-type	SCHDLR_DATA	LONG_LIVED_LOW	SYSTEM		scheduler_data
-type  	LL_TEMP_TERM 	LONG_LIVED_LOW	SYSTEM		ll_temp_term
-
-type	NIF_TRAP_EXPORT	STANDARD_LOW	CODE		nif_trap_export_entry
-type	EXPORT		LONG_LIVED_LOW	CODE		export_entry
-type	MONITOR_SH	STANDARD_LOW	PROCESSES	monitor_sh
-type	NLINK_SH	STANDARD_LOW	PROCESSES	nlink_sh
-type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
-type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
-type	GC_INFO_REQ	STANDARD_LOW	SYSTEM		gc_info_request
-type	PORT_DATA_HEAP	STANDARD_LOW	SYSTEM		port_data_heap
-
-+else # "fullword"
-
 type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
 type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
 type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
@@ -390,9 +357,6 @@ type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
 
-+endif
-
-
 #
 # Types used by system specific code
 #
-- 
cgit v1.2.3


From 9c2f061abe085733824d6e2a659d500813c296ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 22 Jun 2015 09:43:56 +0200
Subject: erts: Remove halfword relative printf

---
 erts/emulator/beam/erl_db_hash.c     |  4 ++--
 erts/emulator/beam/erl_db_tree.c     |  7 +++----
 erts/emulator/beam/erl_printf_term.c | 15 +++++----------
 erts/emulator/beam/erl_printf_term.h |  3 +--
 4 files changed, 11 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index e4f4a7beb0..616d9557c2 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2187,11 +2187,11 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl)
 		    erts_print(to, to_arg, "*");
 		if (tb->common.compress) {
 		    Eterm key = GETKEY(tb, list->dbterm.tpl);
-		    erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl);
+		    erts_print(to, to_arg, "key=%T", key);
 		}
 		else {
 		    Eterm obj = make_tuple(list->dbterm.tpl);
-		    erts_print(to, to_arg, "%R", obj, list->dbterm.tpl);
+		    erts_print(to, to_arg, "%T", obj);
 		}
 		if (list->next != 0)
 		    erts_print(to, to_arg, ",");
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index d7c4e7e5ba..b8791d9f8e 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -2796,7 +2796,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_
 	erts_fprintf(stderr," > ");
     else
 	erts_fprintf(stderr," == ");
-    erts_fprintf(stderr,"%R\n", bound_key, bk_base);
+    erts_fprintf(stderr,"%T\n", bound_key);
 #endif
     return ret;
 }
@@ -3155,9 +3155,8 @@ static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show,
 	    prefix = "";
 	    term = make_tuple(t->dbterm.tpl);
 	}
-	erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n",
-		   offset, "", prefix, term, t->dbterm.tpl,
-		   t, t->balance);
+	erts_print(to, to_arg, "%*s%s%T (addr = %p, bal = %d)\n",
+		   offset, "", prefix, term, t, t->balance);
     }
     do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); 
 }
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 5aef2c76a4..bbb868b76a 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -117,8 +117,7 @@ do {									\
 /* return 0 if list is not a non-empty flat list of printable characters */
 
 static int
-is_printable_string(Eterm list, Eterm* base)
-{
+is_printable_string(Eterm list) {
     int len = 0;
     int c;
 
@@ -260,9 +259,7 @@ static char *format_binary(Uint16 x, char *b) {
 #endif
 
 static int
-print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
-	   Eterm* obj_base)
-{
+print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) {
     DECLARE_WSTACK(s);
     int res;
     int i;
@@ -420,7 +417,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    PRINT_CHAR(res, fn, arg, '>');
 	    break;
 	case LIST_DEF:
-	    if (is_printable_string(obj, obj_base)) {
+	    if (is_printable_string(obj)) {
 		int c;
 		PRINT_CHAR(res, fn, arg, '"');
 		nobj = list_val(obj);
@@ -644,13 +641,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 
 
 int
-erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision,
-		 ErlPfEterm* term_base)
-{
+erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision) {
     int res;
     ERTS_CT_ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm));
 
-    res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base);
+    res = print_term(fn, arg, (Eterm)term, &precision);
     if (res < 0)
 	return res;
     if (precision <= 0)
diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h
index 87618a36d7..4618457f27 100644
--- a/erts/emulator/beam/erl_printf_term.h
+++ b/erts/emulator/beam/erl_printf_term.h
@@ -22,6 +22,5 @@
 #define ERL_PRINTF_TERM_H__
 
 #include "erl_printf_format.h"
-int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision,
-		     ErlPfEterm* term_base);
+int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision);
 #endif
-- 
cgit v1.2.3


From 109f5a0e23de7174f34fddcfaac855d35125e70b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 22 Jun 2015 11:52:48 +0200
Subject: erts: Remove halfword CHECK_POINTER_MASK

---
 erts/emulator/beam/erl_term.h | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 95ced40af7..0d8e981da4 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -24,8 +24,6 @@
 typedef UWord Wterm;  /* Full word terms */
 
 #define HEAP_ON_C_STACK 1
-#define CHECK_POINTER_MASK 0x0UL
-
 struct erl_node_; /* Declared in erl_node_tables.h */
 
 /*
-- 
cgit v1.2.3


From 437dd11b324e17e3c7cf83c8e4f70fd19b51d05f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 22 Jun 2015 14:51:04 +0200
Subject: erts: Remove halfword bases in ETS

---
 erts/emulator/beam/erl_db.c      |   2 +-
 erts/emulator/beam/erl_db_tree.c | 167 ++++++++++++++++-----------------------
 erts/emulator/beam/erl_db_util.c |  17 ++--
 erts/emulator/beam/erl_db_util.h |   2 +-
 4 files changed, 75 insertions(+), 113 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index f3da28b65a..9ec14ab5ae 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2834,7 +2834,7 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
 	    BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3],
 		      BIF_P,lst,BIF_ARG_2,ret);
 	}
-	res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0,
+	res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0,
 			    ERTS_PAM_COPY_RESULT, &dummy);
 	if (is_value(res)) {
 	    hp = HAlloc(BIF_P, 2);
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index b8791d9f8e..311b69d114 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -280,7 +280,7 @@ struct select_delete_context {
 /*
 ** Forward declarations 
 */
-static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base);
+static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key);
 static TreeDbTerm *linkout_object_tree(DbTableTree *tb, 
 				       Eterm object);
 static int do_free_tree_cont(DbTableTree *tb, int num_left);
@@ -291,15 +291,15 @@ static int delsub(TreeDbTerm **this);
 static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot);
 static TreeDbTerm *find_node(DbTableTree *tb, Eterm key);
 static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key);
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key);
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key);
 static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*,
 					 Eterm key);
 static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*,
 					 Eterm key);
 static void traverse_backwards(DbTableTree *tb,
 			       DbTreeStack*,
-			       Eterm lastkey, Eterm* lk_base,
+			       Eterm lastkey,
 			       int (*doit)(DbTableTree *tb,
 					   TreeDbTerm *,
 					   void *,
@@ -307,7 +307,7 @@ static void traverse_backwards(DbTableTree *tb,
 			       void *context); 
 static void traverse_forward(DbTableTree *tb,
 			     DbTreeStack*,
-			     Eterm lastkey, Eterm* lk_base,
+			     Eterm lastkey,
 			     int (*doit)(DbTableTree *tb,
 					 TreeDbTerm *,
 					 void *,
@@ -315,8 +315,8 @@ static void traverse_forward(DbTableTree *tb,
 			     void *context); 
 static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 		     Eterm *partly_bound_key);
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base);
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done);
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key);
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done);
 
 static int analyze_pattern(DbTableTree *tb, Eterm pattern, 
 			   struct mp_info *mpi);
@@ -517,7 +517,7 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     if (is_atom(key) && key == am_EOT)
 	return DB_ERROR_BADKEY;
     stack = get_any_stack(tb);
-    this = find_next(tb, stack, key, NULL);
+    this = find_next(tb, stack, key);
     release_stack(tb,stack);
     if (this == NULL) {
 	*ret = am_EOT;
@@ -563,7 +563,7 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     if (is_atom(key) && key == am_EOT)
 	return DB_ERROR_BADKEY;
     stack = get_any_stack(tb);
-    this = find_prev(tb, stack, key, NULL);
+    this = find_prev(tb, stack, key);
     release_stack(tb,stack);
     if (this == NULL) {
 	*ret = am_EOT;
@@ -573,18 +573,13 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     return DB_ERROR_NONE;
 }
 
-static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base,
-			       TreeDbTerm* obj)
-{
+static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, TreeDbTerm* obj) {
     return CMP(key, GETKEY(tb,obj->dbterm.tpl));
 }
 
-static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base,
-				  TreeDbTerm* obj)
-{
+static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, TreeDbTerm* obj) {
     Eterm obj_key = GETKEY(tb,obj->dbterm.tpl);
-    return is_same(key, obj_key)
-	|| CMP(key, obj_key) == 0;
+    return is_same(key, obj_key) || CMP(key, obj_key) == 0;
 }
 
 static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
@@ -618,7 +613,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
 	    (*this)->balance = 0;
 	    (*this)->left = (*this)->right = NULL;
 	    break;
-	} else if ((c = cmp_key(tb, key, NULL, *this)) < 0) {
+	} else if ((c = cmp_key(tb, key, *this)) < 0) {
 	    /* go lefts */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
@@ -773,7 +768,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret)
 
     *ret = am_true;
 
-    if ((res = linkout_tree(tb, key, NULL)) != NULL) {
+    if ((res = linkout_tree(tb, key)) != NULL) {
 	free_term(tb, res);
     }
     return DB_ERROR_NONE;
@@ -969,15 +964,15 @@ static int db_select_continue_tree(Process *p,
     stack = get_any_stack(tb);
     if (chunk_size) {
 	if (reverse) {
-	    traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
+	    traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc);
 	} else {
-	    traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
+	    traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc);
 	}
     } else {
 	if (reverse) {
-	    traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc);
+	    traverse_forward(tb, stack, lastkey, &doit_select, &sc);
 	} else {
-	    traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc);
+	    traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
 	}
     }
     release_stack(tb,stack);
@@ -1025,8 +1020,8 @@ static int db_select_continue_tree(Process *p,
     key = GETKEY(tb, sc.lastobj);
     if (chunk_size) {
 	if (end_condition != NIL && 
-	    ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) ||
-	     (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) {
+	    ((!reverse && cmp_partly_bound(end_condition,key) < 0) ||
+	     (reverse && cmp_partly_bound(end_condition,key) > 0))) {
 	    /* done anyway */
 	    if (!sc.got) {
 		RET_TO_BIF(am_EOT, DB_ERROR_NONE);
@@ -1038,8 +1033,8 @@ static int db_select_continue_tree(Process *p,
 	}
     } else {
 	if (end_condition != NIL && 
-	    ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) ||
-	     (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) {
+	    ((!reverse && cmp_partly_bound(end_condition,key) > 0) ||
+	     (reverse && cmp_partly_bound(end_condition,key) < 0))) {
 	    /* done anyway */
 	    RET_TO_BIF(sc.accum,DB_ERROR_NONE);
 	}
@@ -1074,7 +1069,6 @@ static int db_select_tree(Process *p, DbTable *tbl,
     struct select_context sc;
     struct mp_info mpi;
     Eterm lastkey = THE_NON_VALUE;
-    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1126,20 +1120,18 @@ static int db_select_tree(Process *p, DbTable *tbl,
 	if (mpi.some_limitation) {
 	    if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
-		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.most;
 	}
-	traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc);
+	traverse_forward(tb, stack, lastkey, &doit_select, &sc);
     } else {
 	if (mpi.some_limitation) {
 	    if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
-		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.least;
 	}
-	traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc);
+	traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
     }
     release_stack(tb,stack);
 #ifdef HARDDEBUG
@@ -1235,7 +1227,7 @@ static int db_select_count_continue_tree(Process *p,
     }
 
     stack = get_any_stack(tb);
-    traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc);
+    traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
     release_stack(tb,stack);
 
     BUMP_REDS(p, 1000 - sc.max);
@@ -1245,7 +1237,7 @@ static int db_select_count_continue_tree(Process *p,
     }	
     key = GETKEY(tb, sc.lastobj);
     if (end_condition != NIL && 
-	(cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) {
+	(cmp_partly_bound(end_condition,key) > 0)) {
 	/* done anyway */
 	RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE);
     }
@@ -1283,7 +1275,6 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     struct select_count_context sc;
     struct mp_info mpi;
     Eterm lastkey = THE_NON_VALUE;
-    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1333,12 +1324,11 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     if (mpi.some_limitation) {
 	if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 	    lastkey = GETKEY(tb, this->dbterm.tpl);
-	    lk_base = this->dbterm.tpl;
 	}
 	sc.end_condition = mpi.least;
     }
     
-    traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc);
+    traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
     release_stack(tb,stack);
     BUMP_REDS(p, 1000 - sc.max);
     if (sc.max > 0) {
@@ -1387,7 +1377,6 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     struct select_context sc;
     struct mp_info mpi;
     Eterm lastkey = THE_NON_VALUE;
-    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1444,20 +1433,18 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 	if (mpi.some_limitation) {
 	    if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
-		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.least;
 	}
-	traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
+	traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc);
     } else {
 	if (mpi.some_limitation) {
 	    if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
-		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.most;
 	}
-	traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
+	traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc);
     }
     release_stack(tb,stack);
 
@@ -1585,7 +1572,7 @@ static int db_select_delete_continue_tree(Process *p,
     sc.keypos = tb->common.keypos;
 
     ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy));
-    traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc);
+    traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
 
     BUMP_REDS(p, 1000 - sc.max);
 
@@ -1594,7 +1581,7 @@ static int db_select_delete_continue_tree(Process *p,
     }	
     key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
     if (end_condition != NIL && 
-	cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */
+	cmp_partly_bound(end_condition,key) > 0) { /* done anyway */
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
     }
     /* Not done yet, let's trap. */
@@ -1629,7 +1616,6 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     struct select_delete_context sc;
     struct mp_info mpi;
     Eterm lastkey = THE_NON_VALUE;
-    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1682,12 +1668,11 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     if (mpi.some_limitation) {
 	if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) {
 	    lastkey = GETKEY(tb, this->dbterm.tpl);
-	    lk_base = this->dbterm.tpl;
 	}
 	sc.end_condition = mpi.least;
     }
 
-    traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc);
+    traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
     BUMP_REDS(p, 1000 - sc.max);
 
     if (sc.max > 0) {
@@ -1733,7 +1718,7 @@ static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     TreeDbTerm *this;
 
     *ret = NIL;
-    this = linkout_tree(tb, key, NULL);
+    this = linkout_tree(tb, key);
     if (this) {
         Eterm copy, *hp, *hend;
 
@@ -1844,9 +1829,7 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt,
     do_db_tree_foreach_offheap(tdbt->right, func, arg);
 }
 
-static TreeDbTerm *linkout_tree(DbTableTree *tb, 
-				Eterm key, Eterm* key_base)
-{
+static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key) {
     TreeDbTerm **tstack[STACK_NEED];
     int tpos = 0;
     int dstack[STACK_NEED+1];
@@ -1868,7 +1851,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb,
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp_key(tb, key, key_base, *this)) < 0) {
+	} else if ((c = cmp_key(tb, key, *this)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -1932,7 +1915,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp_key(tb,key,NULL,*this)) < 0) {
+	} else if ((c = cmp_key(tb,key,*this)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -2318,15 +2301,13 @@ done:
  * Find next and previous in sort order
  */
 
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack,
-			     Eterm key, Eterm* key_base)
-{
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) {
     TreeDbTerm *this;
     TreeDbTerm *tmp;
     Sint c;
 
     if(( this = TOP_NODE(stack)) != NULL) {
-	if (!cmp_key_eq(tb,key,key_base,this)) {
+	if (!cmp_key_eq(tb,key,this)) {
 	    /* Start from the beginning */
 	    stack->pos = stack->slot = 0;
 	}
@@ -2336,7 +2317,7 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack,
 	    return NULL;
 	for (;;) {
 	    PUSH_NODE(stack, this);
-	    if (( c = cmp_key(tb,key,key_base,this) ) > 0) {
+	    if (( c = cmp_key(tb,key,this) ) > 0) {
 		if (this->right == NULL) /* We are at the previos 
 					    and the element does
 					    not exist */
@@ -2376,15 +2357,13 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack,
     return this;
 }
 
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack,
-			     Eterm key, Eterm* key_base)
-{
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) {
     TreeDbTerm *this;
     TreeDbTerm *tmp;
     Sint c;
 
     if(( this = TOP_NODE(stack)) != NULL) {
-	if (!cmp_key_eq(tb,key,key_base,this)) {
+	if (!cmp_key_eq(tb,key,this)) {
 	    /* Start from the beginning */
 	    stack->pos = stack->slot = 0;
 	}
@@ -2394,7 +2373,7 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack,
 	    return NULL;
 	for (;;) {
 	    PUSH_NODE(stack, this);
-	    if (( c = cmp_key(tb,key,key_base,this) ) < 0) {
+	    if (( c = cmp_key(tb,key,this) ) < 0) {
 		if (this->left == NULL) /* We are at the next 
 					   and the element does
 					   not exist */
@@ -2447,8 +2426,7 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
 	return NULL;
     for (;;) {
 	PUSH_NODE(stack, this);
-	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
-				   this->dbterm.tpl) ) >= 0) {
+	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) >= 0) {
 	    if (this->right == NULL) {
 		do {
 		    tmp = POP_NODE(stack);
@@ -2481,8 +2459,7 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
 	return NULL;
     for (;;) {
 	PUSH_NODE(stack, this);
-	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
-				   this->dbterm.tpl) ) <= 0) {
+	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) <= 0) {
 	    if (this->left == NULL) {
 		do {
 		    tmp = POP_NODE(stack);
@@ -2513,10 +2490,10 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key)
     DbTreeStack* stack = get_static_stack(tb);
 
     if(!stack || EMPTY_NODE(stack)
-       || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) {
+       || !cmp_key_eq(tb, key, (this=TOP_NODE(stack)))) {
 
 	this = tb->root;
-	while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) {
+	while (this != NULL && (res = cmp_key(tb,key,this)) != 0) {
 	    if (res < 0)
 		this = this->left;
 	    else
@@ -2538,7 +2515,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key)
     Sint res;
 
     this = &tb->root;
-    while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) {
+    while ((*this) != NULL && (res = cmp_key(tb, key, *this)) != 0) {
 	if (res < 0)
 	    this = &((*this)->left);
 	else
@@ -2618,7 +2595,7 @@ db_finalize_dbterm_tree(int cret, DbUpdateHandle *handle)
  */
 static void traverse_backwards(DbTableTree *tb,
 			       DbTreeStack* stack,
-			       Eterm lastkey, Eterm* lk_base,
+			       Eterm lastkey,
 			       int (*doit)(DbTableTree *,
 					   TreeDbTerm *,
 					   void *,
@@ -2637,16 +2614,15 @@ static void traverse_backwards(DbTableTree *tb,
 	    this = this->right;
 	}
 	this = TOP_NODE(stack);
-	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl),
-			 this->dbterm.tpl);
+	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
 	if (!((*doit)(tb, this, context, 0)))
 	    return;
     } else {
-	next = find_prev(tb, stack, lastkey, lk_base);
+	next = find_prev(tb, stack, lastkey);
     }
 
     while ((this = next) != NULL) {
-	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
+	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
 	if (!((*doit)(tb, this, context, 0)))
 	    return;
     }
@@ -2657,7 +2633,7 @@ static void traverse_backwards(DbTableTree *tb,
  */
 static void traverse_forward(DbTableTree *tb,
 			     DbTreeStack* stack,
-			     Eterm lastkey, Eterm* lk_base,
+			     Eterm lastkey,
 			     int (*doit)(DbTableTree *,
 					 TreeDbTerm *,
 					 void *,
@@ -2676,15 +2652,15 @@ static void traverse_forward(DbTableTree *tb,
 	    this = this->left;
 	}
 	this = TOP_NODE(stack);
-	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
+	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
 	if (!((*doit)(tb, this, context, 1)))
 	    return;
     } else {
-	next = find_next(tb, stack, lastkey, lk_base);
+	next = find_next(tb, stack, lastkey);
     }
 
     while ((this = next) != NULL) {
-	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
+	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
 	if (!((*doit)(tb, this, context, 1)))
 	    return;
     }
@@ -2721,7 +2697,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 
 
 
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
 {
     Eterm* aa;
     Eterm* bb;
@@ -2746,12 +2722,12 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	aa = list_val(a);
 	bb = list_val(b);
 	while (1) {
-	    if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done)
+	    if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done)
 		return j;
 	    if (is_same(*aa, *bb))
 		return 0;
 	    if (is_not_list(*aa) || is_not_list(*bb))
-		return do_cmp_partly_bound(*aa, *bb, b_base, done);
+		return do_cmp_partly_bound(*aa, *bb, done);
 	    aa = list_val(*aa);
 	    bb = list_val(*bb);
 	}
@@ -2772,7 +2748,7 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 	    if (i < arityval(*bb)) return(-1);
 	    if (i > arityval(*bb)) return(1);
 	    while (i--) {
-		if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0
+		if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0
 		    || *done) 
 		    return j;
 	    }
@@ -2784,10 +2760,9 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
     }
 }
 
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base)
-{
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) {
     int done = 0;
-    Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done);
+    Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done);
 #ifdef HARDDEBUG
     erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key);
     if (ret < 0)
@@ -3013,12 +2988,10 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr,
     if (sc->end_condition != NIL && 
 	((forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
-			   this->dbterm.tpl) < 0) ||
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) ||
 	 (!forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
-			   this->dbterm.tpl) > 0))) {
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) {
 	return 0;
     }
     ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects,
@@ -3050,8 +3023,7 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr,
     /* Always backwards traversing */
     if (sc->end_condition != NIL && 
 	(cmp_partly_bound(sc->end_condition, 
-			  GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
-			  this->dbterm.tpl) > 0)) {
+			  GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0)) {
 	return 0;
     }
     ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
@@ -3077,12 +3049,10 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr,
     if (sc->end_condition != NIL && 
 	((forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
-			   this->dbterm.tpl) < 0) ||
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) ||
 	 (!forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
-			   this->dbterm.tpl) > 0))) {
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) {
 	return 0;
     }
 
@@ -3120,14 +3090,13 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
     
     if (sc->end_condition != NIL && 
 	cmp_partly_bound(sc->end_condition, 
-			 GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
-			 this->dbterm.tpl) > 0)
+			 GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0)
 	return 0;
     ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
 			  &this->dbterm, NULL, 0);
     if (ret == am_true) {
 	key = GETKEY(sc->tb, this->dbterm.tpl);
-	linkout_tree(sc->tb, key, this->dbterm.tpl);
+	linkout_tree(sc->tb, key);
 	sc->erase_lastterm = 1;
 	++sc->accum;
     }
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 8047711e6f..0bf9558ac9 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -367,7 +367,6 @@ typedef struct MatchVariable {
     Eterm term;
 #ifdef DEBUG
     Process* proc;
-    Eterm* base;
 #endif
 } MatchVariable;
 
@@ -1233,7 +1232,7 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp,
 {
     Eterm ret;
 
-    ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args,
+    ret = db_prog_match(p, mpsp, NIL, args, num_args,
 			in_flags, return_flags);
 #if defined(HARDDEBUG)
     if (is_non_value(ret)) {
@@ -1258,7 +1257,7 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp,
 {
     Eterm ret;
 
-    ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args,
+    ret = db_prog_match(p, mpsp, args, NULL, num_args,
 			ERTS_PAM_COPY_RESULT,
 			return_flags);
 #if defined(HARDDEBUG)
@@ -1755,7 +1754,7 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
 ** i.e. 'DCOMP_TRACE' was specified 
 */
 Eterm db_prog_match(Process *c_p, Binary *bprog,
-		    Eterm term, Eterm* base,
+		    Eterm term,
 		    Eterm *termp,
 		    int arity,
 		    enum erts_pam_run_flags in_flags,
@@ -1856,7 +1855,6 @@ restart:
     for (i=0; i<prog->num_bindings; i++) {
 	variables[i].term = THE_NON_VALUE;
 	variables[i].proc = NULL;
-	variables[i].base = base;
     }
 #endif
 
@@ -2156,7 +2154,7 @@ restart:
 	    break;
 	case matchPushVResult:
 	    if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV;
-	    /* Build (NULL-based) copy on callers heap */
+	    /* Build copy on callers heap */
 	    n = *pc++;
 	    ASSERT(is_value(variables[n].term));
 	    ASSERT(!variables[n].proc);
@@ -2164,14 +2162,12 @@ restart:
 	    *esp++ = variables[n].term;
 	    #ifdef DEBUG
 	    variables[n].proc = c_p;
-	    variables[n].base = NULL;
 	    #endif
 	    break;
 	case matchPushV:
 	case_matchPushV:
 	    n = *pc++;
 	    ASSERT(is_value(variables[n].term));
-	    ASSERT(!variables[n].base);
 	    *esp++ = variables[n].term;
 	    break;
 	case matchPushExpr:
@@ -5204,16 +5200,13 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
 			     int all, DbTerm* obj, Eterm** hpp, Uint extra)
 {
     Uint32 dummy;
-    Eterm* base;
     Eterm res;
 
     if (tb->compress) {
 	obj = db_alloc_tmp_uncompressed(tb, obj);
-	base = NULL;
     }
-    else base = NULL;
 
-    res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), base, NULL, 0,
+    res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0,
 			ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
 
     if (is_value(res) && hpp!=NULL) {
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 10899bb3e7..c0966655cd 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -432,7 +432,7 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards,
 Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
 		      int all, DbTerm* obj, Eterm** hpp, Uint extra);
 
-Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base,
+Eterm db_prog_match(Process *p, Binary *prog, Eterm term,
 		    Eterm *termp, int arity,
 		    enum erts_pam_run_flags in_flags,
 		    Uint32 *return_flags /* Zeroed on enter */);
-- 
cgit v1.2.3


From 37a9aa5f35466838af383d53490b040142468673 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 26 Jun 2015 14:22:14 +0200
Subject: erts: Fix ETS race between object deleter and table unfixer

causing the delete marked object to be left in the table
after safe_fixtable(_,false) has returned.

This is not super serious as the delete marked object
is quite benign and will be deleted at the next unfix operation.
---
 erts/emulator/beam/erl_db_hash.c | 60 +++++++++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 23 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 81b0c4465c..98a2e2842a 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -148,8 +148,11 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 }
 
 /* Remember a slot containing a pseudo-deleted item (INVALID_HASH)
-*/
-static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
+ * Return false if we got raced by unfixing thread
+ * and the object should be deleted for real.
+ */
+static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
+                                          erts_aint_t fixated_by_me)
 {
     erts_aint_t was_next;
     erts_aint_t exp_next;
@@ -160,12 +163,18 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
     fixd->slot = ix;
     was_next = erts_smp_atomic_read_acqb(&tb->fixdel);
     do { /* Lockless atomic insertion in linked list: */
-	exp_next = was_next;
+        if (NFIXED(tb) <= fixated_by_me) {
+            erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb,
+                         fixd, sizeof(FixedDeletion));
+            return 0; /* raced by unfixer */
+        }
+        exp_next = was_next;
 	fixd->next = (FixedDeletion*) exp_next;
-	was_next = erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
-						(erts_aint_t) fixd,
-						exp_next);
+	was_next = erts_smp_atomic_cmpxchg_mb(&tb->fixdel,
+                                              (erts_aint_t) fixd,
+                                              exp_next);
     }while (was_next != exp_next);
+    return 1;
 }
 
 
@@ -607,8 +616,8 @@ void db_unfix_table_hash(DbTableHash *tb)
 		       || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock)
 			   && !tb->common.is_thread_safe));
 restart:
-    fixdel = (FixedDeletion*) erts_smp_atomic_xchg_acqb(&tb->fixdel,
-							(erts_aint_t) NULL);
+    fixdel = (FixedDeletion*) erts_smp_atomic_xchg_mb(&tb->fixdel,
+                                                      (erts_aint_t) NULL);
     while (fixdel != NULL) {
 	FixedDeletion *fx = fixdel;
 	int ix = fx->slot;
@@ -1142,9 +1151,9 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
     while(b != 0) {
 	if (has_live_key(tb,b,key,hval)) {
 	    --nitems_diff;
-	    if (nitems_diff == -1 && IS_FIXED(tb)) {
+	    if (nitems_diff == -1 && IS_FIXED(tb)
+                && add_fixed_deletion(tb, ix, 0)) {
 		/* Pseudo remove (no need to keep several of same key) */
-		add_fixed_deletion(tb, ix);
 		b->hvalue = INVALID_HASH;
 	    } else {
 		*bp = b->next;
@@ -1196,9 +1205,8 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
 	    ++nkeys;
 	    if (db_eq(&tb->common,object, &b->dbterm)) {
 		--nitems_diff;
-		if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */
-		    add_fixed_deletion(tb,ix);
-		    b->hvalue = INVALID_HASH;
+		if (nkeys==1 && IS_FIXED(tb) && add_fixed_deletion(tb,ix,0)) {
+		    b->hvalue = INVALID_HASH;        /* Pseudo remove */
 		    bp = &b->next;
 		    b = b->next;
 		} else {
@@ -1820,14 +1828,17 @@ static int db_select_delete_hash(Process *p,
 	    int did_erase = 0;
 	    if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
 				&(*current)->dbterm, NULL, 0) == am_true) {
+                HashDbTerm *del;
 		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
 		    if (slot_ix != last_pseudo_delete) {
-			add_fixed_deletion(tb, slot_ix);
-			last_pseudo_delete = slot_ix;
+                        if (!add_fixed_deletion(tb, slot_ix, fixated_by_me))
+                            goto do_erase;
+                        last_pseudo_delete = slot_ix;
 		    }
 		    (*current)->hvalue = INVALID_HASH;
 		} else {
-		    HashDbTerm *del = *current;
+                do_erase:
+		    del = *current;
 		    *current = (*current)->next;
 		    free_term(tb, del);
 		    did_erase = 1;
@@ -1931,14 +1942,17 @@ static int db_select_delete_continue_hash(Process *p,
 	    int did_erase = 0;
 	    if (db_match_dbterm(&tb->common, p, mp, 0,
 				&(*current)->dbterm, NULL, 0) == am_true) {
+                HashDbTerm *del;
 		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
 		    if (slot_ix != last_pseudo_delete) {
-			add_fixed_deletion(tb, slot_ix);
+                        if (!add_fixed_deletion(tb, slot_ix, fixated_by_me))
+                            goto do_erase;
 			last_pseudo_delete = slot_ix;
 		    }
 		    (*current)->hvalue = INVALID_HASH;
 		} else {
-		    HashDbTerm *del = *current;
+                do_erase:
+		    del = *current;
 		    *current = (*current)->next;
 		    free_term(tb, del);
 		    did_erase = 1;
@@ -2089,9 +2103,9 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
             *ret = get_term_list(p, tb, key, hval, b, &bend);
             while (b != bend) {
                 --nitems_diff;
-                if (nitems_diff == -1 && IS_FIXED(tb)) {
+                if (nitems_diff == -1 && IS_FIXED(tb)
+                    && add_fixed_deletion(tb, ix, 0)) {
                     /* Pseudo remove (no need to keep several of same key) */
-                    add_fixed_deletion(tb, ix);
                     bp = &b->next;
                     b->hvalue = INVALID_HASH;
                     b = b->next;
@@ -2131,7 +2145,7 @@ int db_mark_all_deleted_hash(DbTable *tbl)
 
     for (i = 0; i < NACTIVE(tb); i++) {
 	if ((list = BUCKET(tb,i)) != NULL) {
-	    add_fixed_deletion(tb, i);
+	    add_fixed_deletion(tb, i, 0);
 	    do {
 		list->hvalue = INVALID_HASH;
 		list = list->next;
@@ -2908,8 +2922,8 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
     ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE));
 
     if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) {
-        if (IS_FIXED(tb)) {
-            add_fixed_deletion(tb, hash_to_ix(tb, b->hvalue));
+        if (IS_FIXED(tb) && add_fixed_deletion(tb, hash_to_ix(tb, b->hvalue),
+                                               0)) {
             b->hvalue = INVALID_HASH;
         } else {
             *bp = b->next;
-- 
cgit v1.2.3


From 3bda47f2f26dd417fbd65d5f46b2ab8411a2a41f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 1 Jul 2015 15:09:58 +0200
Subject: erts: Remove halfword !HEAP_ON_C_STACK

---
 erts/emulator/beam/beam_emu.c     |  9 +-----
 erts/emulator/beam/erl_arith.c    | 11 ++-----
 erts/emulator/beam/erl_bif_info.c |  9 ------
 erts/emulator/beam/erl_debug.c    | 25 ---------------
 erts/emulator/beam/erl_debug.h    |  6 ----
 erts/emulator/beam/erl_process.c  |  3 --
 erts/emulator/beam/erl_process.h  |  6 ----
 erts/emulator/beam/erl_term.h     |  1 -
 erts/emulator/beam/erl_trace.c    | 24 --------------
 erts/emulator/beam/erl_vm.h       |  8 -----
 erts/emulator/beam/global.h       | 67 ++++-----------------------------------
 erts/emulator/beam/io.c           | 35 ++------------------
 12 files changed, 13 insertions(+), 191 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index f4111c19f1..806b31654c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1165,11 +1165,7 @@ void process_main(void)
      */
     register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
     register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
-#if HEAP_ON_C_STACK
-    Eterm tmp_big[2];           /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */
-#else
-    Eterm *tmp_big;		/* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
-#endif
+    Eterm tmp_big[2];
 
     /*
      * X registers and floating point registers are located in
@@ -1261,9 +1257,6 @@ void process_main(void)
 
     reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
     freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
-#if !HEAP_ON_C_STACK
-    tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
-#endif
     ERL_BITS_RELOAD_STATEP(c_p);
     {
 	int reds;
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index b8c5ef9b09..3671025d22 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -42,15 +42,8 @@
 #  define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #endif
 
-#if !HEAP_ON_C_STACK
-#  define DECLARE_TMP(VariableName,N,P)  \
-     Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N))
-#else
-#  define DECLARE_TMP(VariableName,N,P) \
-     Eterm VariableName[2]
-#endif
-#  define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp)))
-
+#define DECLARE_TMP(VariableName,N,P)  Eterm VariableName[2]
+#define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp)))
 
 static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right);
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a3ff537aa1..9a132ee007 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -73,9 +73,6 @@ static char otp_version[] = ERLANG_OTP_VERSION;
 static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     "%s"
 				     " [erts-" ERLANG_VERSION "]"
-#if !HEAP_ON_C_STACK
-				     " [no-c-stack-objects]"
-#endif
 #ifndef OTP_RELEASE
 #ifdef ERLANG_GIT_VERSION
 				     " [source-" ERLANG_GIT_VERSION "]"
@@ -667,18 +664,12 @@ static Eterm pi_1_keys[] = {
 #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm))
 
 static Eterm pi_1_keys_list;
-#if HEAP_ON_C_STACK
 static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS];
-#endif
 
 static void
 process_info_init(void)
 {
-#if HEAP_ON_C_STACK
     Eterm *hp = &pi_1_keys_list_heap[0];
-#else
-    Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS);
-#endif
     int i;
 
     pi_1_keys_list = NIL;
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 77a1e3d7cb..434b6c6d7a 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -628,29 +628,4 @@ void print_memory_info(Process *p)
     }
     erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes);
 }
-#if !HEAP_ON_C_STACK && defined(DEBUG)
-Eterm *erts_debug_allocate_tmp_heap(int size, Process *p)
-{
-    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
-    int offset = sd->num_tmp_heap_used;
-
-    ASSERT(offset+size <= TMP_HEAP_SIZE);
-    return (sd->tmp_heap)+offset;
-}
-void erts_debug_use_tmp_heap(int size, Process *p)
-{
-    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
-
-    sd->num_tmp_heap_used += size;
-    ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE);
-}
-void erts_debug_unuse_tmp_heap(int size, Process *p)
-{
-    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
-
-    sd->num_tmp_heap_used -= size;
-    ASSERT(sd->num_tmp_heap_used >= 0);
-}
 #endif
-#endif
-
diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h
index 4905e64f07..f4259e7dae 100644
--- a/erts/emulator/beam/erl_debug.h
+++ b/erts/emulator/beam/erl_debug.h
@@ -92,10 +92,4 @@ extern void print_tagged_memory(Eterm *start, Eterm *end);
 extern void print_untagged_memory(Eterm *start, Eterm *end);
 extern void print_memory(Process *p);
 extern void print_memory_info(Process *p);
-#if defined(DEBUG) && !HEAP_ON_C_STACK
-extern Eterm *erts_debug_allocate_tmp_heap(int, Process *);
-extern void erts_debug_use_tmp_heap(int, Process *);
-extern void erts_debug_unuse_tmp_heap(int, Process *);
-#endif
-
 #endif /* _ERL_DEBUG_H_ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 135f09c04d..338be6d8e1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5495,9 +5495,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->f_reg_array =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
 					   MAX_REG * sizeof(FloatDef));
-#if !HEAP_ON_C_STACK
-    esdp->num_tmp_heap_used = 0;
-#endif
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
 	esdp->no = 0;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 20ffe7ea7c..ffbf3e2bff 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -631,12 +631,6 @@ struct ErtsSchedulerData_ {
     void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
     Process *free_process;
     ErtsThrPrgrData thr_progress_data;
-#endif
-#if !HEAP_ON_C_STACK
-    Eterm tmp_heap[TMP_HEAP_SIZE];
-    int num_tmp_heap_used;
-    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
-    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
 #endif
     ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 0d8e981da4..b928e4ca1e 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -23,7 +23,6 @@
 
 typedef UWord Wterm;  /* Full word terms */
 
-#define HEAP_ON_C_STACK 1
 struct erl_node_; /* Declared in erl_node_tables.h */
 
 /*
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index e1b03a057f..7baa3b1a54 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1677,12 +1677,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
     args = transformed_args;
 
     if (is_internal_port(*tracer_pid)) {
-#if HEAP_ON_C_STACK
 	Eterm local_heap[64+MAX_ARG];
-#else
-	Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM,
-				       sizeof(Eterm)*(64+MAX_ARG));
-#endif
 	hp = local_heap;
 
 	if (!erts_is_valid_tracer_port(*tracer_pid)) {
@@ -1696,9 +1691,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 #ifdef ERTS_SMP
 	    if (is_not_nil(tracee))
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
-#if !HEAP_ON_C_STACK
-	    erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
 #endif
 	    UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	    return 0;
@@ -1727,9 +1719,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 					    ERTS_PAM_TMP_RESULT, &return_flags);
 	    if (is_non_value(pam_result)) {
 		erts_match_set_release_result(p);
-#if !HEAP_ON_C_STACK
-		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
-#endif
 		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
@@ -1738,9 +1727,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	    /* Meta trace */
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
-#if !HEAP_ON_C_STACK
-		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
-#endif
 		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
@@ -1748,17 +1734,11 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	    /* Non-meta trace */
 	    if (*tracee_flags & F_TRACE_SILENT) { 
 		erts_match_set_release_result(p);
-#if !HEAP_ON_C_STACK
-		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
-#endif
 		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
-#if !HEAP_ON_C_STACK
-		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
-#endif
 		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
@@ -1802,9 +1782,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	send_to_port(p, mess, tracer_pid, tracee_flags);
 	erts_smp_mtx_unlock(&smq_mtx);
 	erts_match_set_release_result(p);
-#if !HEAP_ON_C_STACK
-	erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
-#endif
 	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	return *tracer_pid == NIL ? 0 : return_flags;
 
@@ -1823,7 +1800,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 #ifdef DEBUG
 	Eterm* limit;
 #endif
-
 	ASSERT(is_internal_pid(*tracer_pid));
 	
 	tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 2d4141ac8d..cfc978b5a5 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -40,14 +40,6 @@
 #define MAX_ARG 255	        /* Max number of arguments allowed */
 #define MAX_REG 1024            /* Max number of x(N) registers used */
 
-/* Scheduler stores data for temporary heaps if
-   !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary
-   heap data on the C stack or if we use the buffers in the scheduler data. */
-#define TMP_HEAP_SIZE 128            /* Number of Eterm in the schedulers
-				        small heap for transient heap data */
-#define ERL_ARITH_TMP_HEAP_SIZE 4    /* as does erl_arith... */
-#define BEAM_EMU_TMP_HEAP_SIZE  2    /* and beam_emu... */
-
 /*
  * The new arithmetic operations need some extra X registers in the register array.
  * so does the gc_bif's (i_gc_bif3 need 3 extra).
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 870afb414f..b4d02dd1dd 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1444,69 +1444,16 @@ erts_alloc_message_heap(Uint size,
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#if !HEAP_ON_C_STACK
-#  if defined(DEBUG)
-#    define DeclareTmpHeap(VariableName,Size,Process) \
-       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process)
-#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
-      Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process)
-#    define DeclareTmpHeapNoproc(VariableName,Size) \
-       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL)
-#    define UseTmpHeap(Size,Proc) \
-       do { \
-         erts_debug_use_tmp_heap((Size),(Proc)); \
-       } while (0)
-#    define UnUseTmpHeap(Size,Proc) \
-       do { \
-         erts_debug_unuse_tmp_heap((Size),(Proc)); \
-       } while (0)
-#    define UseTmpHeapNoproc(Size) \
-       do { \
-         erts_debug_use_tmp_heap(Size,NULL); \
-       } while (0)
-#    define UnUseTmpHeapNoproc(Size) \
-       do { \
-         erts_debug_unuse_tmp_heap(Size,NULL); \
-       } while (0)
-#  else
-#    define DeclareTmpHeap(VariableName,Size,Process) \
-       Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
-#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
-      Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
-#    define DeclareTmpHeapNoproc(VariableName,Size) \
-       Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used)
-#    define UseTmpHeap(Size,Proc) \
-       do { \
-         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \
-       } while (0)
-#    define UnUseTmpHeap(Size,Proc) \
-       do { \
-         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \
-       } while (0)
-#    define UseTmpHeapNoproc(Size) \
-       do { \
-         erts_get_scheduler_data()->num_tmp_heap_used += (Size); \
-       } while (0)
-#    define UnUseTmpHeapNoproc(Size) \
-       do { \
-         erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \
-       } while (0)
-
-
-#  endif
-
-#else
-#  define DeclareTmpHeap(VariableName,Size,Process) \
+#define DeclareTmpHeap(VariableName,Size,Process) \
      Eterm VariableName[Size]
-#  define DeclareTypedTmpHeap(Type,VariableName,Process)	\
+#define DeclareTypedTmpHeap(Type,VariableName,Process)	\
      Type VariableName[1]
-#  define DeclareTmpHeapNoproc(VariableName,Size) \
+#define DeclareTmpHeapNoproc(VariableName,Size) \
      Eterm VariableName[Size]
-#  define UseTmpHeap(Size,Proc) /* Nothing */
-#  define UnUseTmpHeap(Size,Proc) /* Nothing */
-#  define UseTmpHeapNoproc(Size) /* Nothing */
-#  define UnUseTmpHeapNoproc(Size) /* Nothing */
-#endif /* HEAP_ON_C_STACK */
+#define UseTmpHeap(Size,Proc) /* Nothing */
+#define UnUseTmpHeap(Size,Proc) /* Nothing */
+#define UseTmpHeapNoproc(Size) /* Nothing */
+#define UnUseTmpHeapNoproc(Size) /* Nothing */
 
 ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf);
 ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 6b4b90cb06..915d0f4328 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -6791,7 +6791,7 @@ int driver_monitor_process(ErlDrvPort drvport,
 {
     Port *prt;
     int ret;
-#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK))
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsSchedulerData *sched = erts_get_scheduler_data();
 #endif
 
@@ -6802,16 +6802,6 @@ int driver_monitor_process(ErlDrvPort drvport,
     /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
     ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
-
-#if !HEAP_ON_C_STACK
-    if (!sched) {
-	/* Need a separate allocation for the ref :( */
-	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
-				sizeof(Eterm)*REF_THING_SIZE);
-	ret = do_driver_monitor_process(prt,buf,process,monitor);
-	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
-    } else
-#endif
     {
 	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
 	UseTmpHeapNoproc(REF_THING_SIZE);
@@ -6864,7 +6854,7 @@ int driver_demonitor_process(ErlDrvPort drvport,
 {
     Port *prt;
     int ret;
-#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK))
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsSchedulerData *sched = erts_get_scheduler_data();
 #endif
 
@@ -6875,15 +6865,6 @@ int driver_demonitor_process(ErlDrvPort drvport,
     /* Now we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
     ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
-#if !HEAP_ON_C_STACK
-    if (!sched) {
-	/* Need a separate allocation for the ref :( */
-	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
-				sizeof(Eterm)*REF_THING_SIZE);
-	ret = do_driver_demonitor_process(prt,buf,monitor);
-	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
-    } else
-#endif
     {
 	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
 	UseTmpHeapNoproc(REF_THING_SIZE);
@@ -6919,7 +6900,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport,
 {
     Port *prt;
     ErlDrvTermData ret;
-#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK))
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsSchedulerData *sched = erts_get_scheduler_data();
 #endif
 
@@ -6930,16 +6911,6 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport,
     /* Now we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
     ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
-
-#if !HEAP_ON_C_STACK
-    if (!sched) {
-	/* Need a separate allocation for the ref :( */
-	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
-				sizeof(Eterm)*REF_THING_SIZE);
-	ret = do_driver_get_monitored_process(prt,buf,monitor);
-	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
-    } else
-#endif
     {
 	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
 	UseTmpHeapNoproc(REF_THING_SIZE);
-- 
cgit v1.2.3


From 0d69dcf52f190c29ba1778bc61f030e6379f4379 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 2 Jul 2015 10:10:20 +0200
Subject: Correct disassembly of the i_get_map_elements instruction

The emulator would crash.
---
 erts/emulator/beam/beam_debug.c | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index c756de8c8e..8a35ad17c6 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -669,7 +669,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_new_map_dII:
     case op_update_map_assoc_jsdII:
     case op_update_map_exact_jsdII:
-    case op_i_get_map_elements_fsI:
 	{
 	    int n = unpacked[-1];
 
@@ -693,6 +692,32 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
+    case op_i_get_map_elements_fsI:
+	{
+	    int n = unpacked[-1];
+
+	    while (n > 0) {
+		if (n % 3 == 1) {
+		    erts_print(to, to_arg, " %X", ap[0]);
+		} else if (!is_header(ap[0])) {
+		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
+		} else {
+		    switch ((ap[0] >> 2) & 0x03) {
+		    case R_REG_DEF:
+			erts_print(to, to_arg, " x(0)");
+			break;
+		    case X_REG_DEF:
+			erts_print(to, to_arg, " x(%d)", ap[0] >> 4);
+			break;
+		    case Y_REG_DEF:
+			erts_print(to, to_arg, " y(%d)", ap[0] >> 4);
+			break;
+		    }
+		}
+		ap++, size++, n--;
+	    }
+	}
+	break;
     }
     erts_print(to, to_arg, "\n");
 
-- 
cgit v1.2.3


From 54189af4a519b72883dde3263135baf6aba0f81f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 1 Jul 2015 12:18:54 +0200
Subject: beam_emu.c: Remove unused MoveGenDest macro

---
 erts/emulator/beam/beam_emu.c | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 806b31654c..0f9bfa1871 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -566,9 +566,6 @@ void** beam_ops;
 #define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2)
 #define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)
 
-#define MoveGenDest(src, dstp) \
-   if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; }
-
 #define MoveReturn(Src, Dest)       \
     (Dest) = (Src);                 \
     I = c_p->cp;                    \
-- 
cgit v1.2.3


From 45f469ca089096c8a59ca53d948d05d1a998386e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 30 Mar 2012 11:05:16 +0200
Subject: Change the meaning of 'x' in a transformation

The purpose of this series of commits is to improve code generation
for the Clang compiler.

As a first step we want to change the meaning of 'x' in a
transformation such as:

  operation Literal=q => move Literal x | operation x

Currently, a plain 'x' means reg[0] or x(0), which is the first
element in the X register array. That element is distinct from
r(0) which is a variable in process_main(). Therefore, since r(0)
and x(0) are currently distinct it is fine to use x(0) as a
scratch register.

However, in the next commit we will eliminate the separate variable
for storing the contents of X register zero (thus, x(0) and r(0)
will point to the same location in the X register array). Therefore,
we must use another scratch register in transformation. Redefine
a plain 'x' in a transformation to mean x(1023). Also define
SCRATCH_X_REG so that we can refer to the register by name from
C code.
---
 erts/emulator/beam/beam_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 10baab506a..f5f1147261 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4322,7 +4322,7 @@ gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
     for (i = 0; i < n; i++) {
 	op->a[3+2*i] = Rest[i];
 	op->a[3+2*i+1].type = TAG_x;
-	op->a[3+2*i+1].val = 0;	/* x(0); normally not used */
+	op->a[3+2*i+1].val = SCRATCH_X_REG; /* Ignore result */
     }
     return op;
 }
-- 
cgit v1.2.3


From 1f996cc46a8c935b8f4e33a4a96804f9fb852da3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 30 Mar 2012 11:38:37 +0200
Subject: Store r(0) and x(0) in the same location

As part of improving code generation for clang, we want to
eliminate the special variable that stores the content of X
register zero most of the time. In a future, that will allow us
to eliminate the special case of handling r(0) for most
instructions, thus reducing the code size and allow other
simplifcations.

Therefore, in this commit, eliminate the variable that is used
to store r(0) and make r(0) as synonym for x(0). I have chosen
to keep the r(0) define to keep the size of the diff managable.
---
 erts/emulator/beam/beam_emu.c | 100 +++---------------------------------------
 1 file changed, 7 insertions(+), 93 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0f9bfa1871..9a34d5f11a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -99,10 +99,7 @@ do {									\
 do {                                     \
   int i_;                                \
   int Arity_ = PC[-1];                   \
-  if (Arity_ > 0) {                      \
-	CHECK_TERM(r(0));                \
-  }                                      \
-  for (i_ = 1; i_ < Arity_; i_++) {      \
+  for (i_ = 0; i_ < Arity_; i_++) {      \
 	CHECK_TERM(x(i_));               \
   }                                      \
 } while (0)
@@ -293,7 +290,7 @@ void** beam_ops;
 #define Ib(N) (N)
 #define x(N) reg[N]
 #define y(N) E[N]
-#define r(N) x##N
+#define r(N) x(N)
 
 /*
  * Makes sure that there are StackNeed + HeapNeed + 1 words available
@@ -309,12 +306,10 @@ void** beam_ops;
      needed = (StackNeed) + 1; \
      if (E - HTOP < (needed + (HeapNeed))) { \
            SWAPOUT; \
-           reg[0] = r(0); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
            ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
-           r(0) = reg[0]; \
            SWAPIN; \
      } \
      E -= needed; \
@@ -363,12 +358,10 @@ void** beam_ops;
     unsigned need = (Nh);                                       		\
     if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
        SWAPOUT;                                                 		\
-       reg[0] = r(0);                                           		\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  		\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
-       r(0) = reg[0];                                           		\
        SWAPIN;                                                  		\
     }                                                           		\
     HEAP_SPACE_VERIFIED(need);                                                  \
@@ -386,12 +379,10 @@ void** beam_ops;
     unsigned need = (Nh);                                       \
     if (E - HTOP < need) {                                      \
        SWAPOUT;                                                 \
-       reg[0] = r(0);                                           \
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
-       r(0) = reg[0];                                           \
        SWAPIN;                                                  \
     }                                                           \
     HEAP_SPACE_VERIFIED(need);                             \
@@ -408,15 +399,11 @@ void** beam_ops;
     unsigned need = (Nh);						\
     if (E - HTOP < need) {						\
        SWAPOUT;								\
-       reg[0] = r(0);							\
        reg[Live] = Extra;						\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
-       if (Live > 0) {							\
-	   r(0) = reg[0];						\
-       }								\
        Extra = reg[Live];						\
        SWAPIN;								\
     }									\
@@ -439,7 +426,6 @@ void** beam_ops;
 #define MakeFun(FunP, NumFree)					\
   do {								\
      SWAPOUT;							\
-     reg[0] = r(0);						\
      r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree);	\
      SWAPIN;							\
   } while (0)
@@ -992,7 +978,6 @@ init_emulator(void)
  */
 
 #if defined(__GNUC__) && defined(sparc) && !defined(DEBUG)
-#  define REG_x0 asm("%l0")
 #  define REG_xregs asm("%l1")
 #  define REG_htop asm("%l2")
 #  define REG_stop asm("%l3")
@@ -1001,7 +986,6 @@ init_emulator(void)
 #  define REG_tmp_arg1 asm("%l6")
 #  define REG_tmp_arg2 asm("%l7")
 #else
-#  define REG_x0
 #  define REG_xregs
 #  define REG_htop
 #  define REG_stop
@@ -1126,11 +1110,6 @@ void process_main(void)
     ERTS_DECLARE_DUMMY(Eterm pid);
 #endif
 
-    /*
-     * X register zero; also called r(0)
-     */
-    register Eterm x0 REG_x0 = NIL;
-
     /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
      * in all other cases x0 is used.
      */
@@ -1262,7 +1241,7 @@ void process_main(void)
 	int i;
 
 	argp = c_p->arg_reg;
-	for (i = c_p->arity - 1; i > 0; i--) {
+	for (i = c_p->arity - 1; i >= 0; i--) {
 	    reg[i] = argp[i];
 	    CHECK_TERM(reg[i]);
 	}
@@ -1286,12 +1265,6 @@ void process_main(void)
 	}
 
 	next = (BeamInstr *) *I;
-	r(0) = c_p->arg_reg[0];
-#ifdef HARDDEBUG
-	if (c_p->arity > 0) {
-	    CHECK_TERM(r(0));
-	}
-#endif
 	SWAPIN;
 	ASSERT(VALID_INSTR(next));
 
@@ -1365,11 +1338,9 @@ void process_main(void)
 
 	    live = Arg(2);
 	    SWAPOUT;
-	    reg[0] = r(0);
 	    reg[live] = increment_reg_val;
 	    reg[live+1] = make_small(increment_val);
 	    result = erts_gc_mixed_plus(c_p, reg, live);
-	    r(0) = reg[0];
 	    SWAPIN;
 	    ERTS_HOLE_CHECK(c_p);
 	    if (is_value(result)) {
@@ -1383,11 +1354,9 @@ void process_main(void)
     do {                                 \
         Uint live = Arg(1);              \
         SWAPOUT;                         \
-        reg[0] = r(0);                   \
         reg[live] = (Arg1);              \
         reg[live+1] = (Arg2);            \
         result = (Func)(c_p, reg, live); \
-        r(0) = reg[0];                   \
         SWAPIN;                          \
         ERTS_HOLE_CHECK(c_p);            \
         if (is_value(result)) {          \
@@ -1706,7 +1675,6 @@ void process_main(void)
 
      PRE_BIF_SWAPOUT(c_p);
      c_p->fcalls = FCALLS - 1;
-     reg[0] = r(0);
      result = erl_send(c_p, r(0), x(1));
      PreFetch(0, next);
      ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -1714,7 +1682,6 @@ void process_main(void)
      PROCESS_MAIN_CHK_LOCKS(c_p);
      if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
 	 result = erts_gc_after_bif_call(c_p, result, reg, 2);
-	 r(0) = reg[0];
 	 E = c_p->stop;
      }
      HTOP = HEAP_TOP(c_p);
@@ -1727,7 +1694,6 @@ void process_main(void)
 	 SET_CP(c_p, I+1);
 	 SET_I(c_p->i);
 	 SWAPIN;
-	 r(0) = reg[0];
 	 Dispatch();
      }
      goto find_func_info;
@@ -1935,13 +1901,11 @@ void process_main(void)
      ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS,
 				   {
 				       SWAPOUT;
-				       reg[0] = r(0);
 				       PROCESS_MAIN_CHK_LOCKS(c_p);
 				   },
 				   {
 				       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 				       PROCESS_MAIN_CHK_LOCKS(c_p);
-				       r(0) = reg[0];
 				       SWAPIN;
 				   });
      if (is_non_value(ERL_MESSAGE_TERM(msgp))) {
@@ -2448,11 +2412,9 @@ void process_main(void)
  OpCase(new_map_dII): {
      Eterm res;
 
-     x(0) = r(0);
      SWAPOUT;
      res = new_map(c_p, reg, I-1);
      SWAPIN;
-     r(0) = x(0);
      StoreResult(res, Arg(0));
      Next(3+Arg(2));
  }
@@ -2543,12 +2505,10 @@ do {								\
      Eterm map;
 
      GetArg1(1, map);
-     x(0) = r(0);
      SWAPOUT;
      res = update_map_assoc(c_p, reg, map, I);
      SWAPIN;
      if (is_value(res)) {
-	 r(0) = x(0);
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
      } else {
@@ -2567,12 +2527,10 @@ do {								\
      Eterm map;
 
      GetArg1(1, map);
-     x(0) = r(0);
      SWAPOUT;
      res = update_map_exact(c_p, reg, map, I);
      SWAPIN;
      if (is_value(res)) {
-	 r(0) = x(0);
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
      } else {
@@ -2659,7 +2617,6 @@ do {								\
 	Uint live = (Uint) Arg(3);
 
 	GetArg1(2, arg);
-	reg[0] = r(0);
 	reg[live] = arg;
 	bf = (GcBifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
@@ -2671,7 +2628,6 @@ do {								\
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
-	r(0) = reg[0];
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
@@ -2694,7 +2650,6 @@ do {								\
 	Eterm result;
 	Uint live = (Uint) Arg(2);
 
-	reg[0] = r(0);
 	reg[live++] = tmp_arg1;
 	reg[live] = tmp_arg2;
 	bf = (GcBifFunction) Arg(1);
@@ -2707,7 +2662,6 @@ do {								\
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
-	r(0) = reg[0];
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
@@ -2732,7 +2686,6 @@ do {								\
 	Uint live = (Uint) Arg(3);
 
 	GetArg1(2, arg);
-	reg[0] = r(0);
 	reg[live++] = arg;
 	reg[live++] = tmp_arg1;
 	reg[live] = tmp_arg2;
@@ -2746,7 +2699,6 @@ do {								\
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
-	r(0) = reg[0];
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
@@ -2835,7 +2787,6 @@ do {								\
 	PreFetch(1, next);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	reg[0] = r(0);
 	result = (*bf)(c_p, reg, I);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2857,7 +2808,6 @@ do {								\
 	    SET_CP(c_p, I+2);
 	    SET_I(c_p->i);
 	    SWAPIN;
-	    r(0) = reg[0];
 	    Dispatch();
 	}
 
@@ -2967,11 +2917,9 @@ do {								\
      Uint live = Arg(1);
 
      SWAPOUT;
-     reg[0] = r(0);
      reg[live] = tmp_arg1;
      reg[live+1] = tmp_arg2;
      result = arith_func(c_p, reg, live);
-     r(0) = reg[0];
      SWAPIN;
      ERTS_HOLE_CHECK(c_p);
      if (is_value(result)) {
@@ -3166,10 +3114,8 @@ do {								\
      } else {
 	 Uint live = Arg(2);
 	 SWAPOUT;
-	 reg[0] = r(0);
 	 reg[live] = bnot_val;
 	 bnot_val = erts_gc_bnot(c_p, reg, live);
-	 r(0) = reg[0];
 	 SWAPIN;
 	 ERTS_HOLE_CHECK(c_p);
 	 if (is_nil(bnot_val)) {
@@ -3189,7 +3135,6 @@ do {								\
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_CP(c_p, I+1);
 	 SET_I(next);
 	 Dispatch();
@@ -3204,7 +3149,6 @@ do {								\
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
@@ -3220,7 +3164,6 @@ do {								\
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_I(next);
 	 Dispatch();
      }
@@ -3231,12 +3174,10 @@ do {								\
  OpCase(apply_I): {
      BeamInstr *next;
 
-     reg[0] = r(0);
      SWAPOUT;
      next = fixed_apply(c_p, reg, Arg(0));
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_CP(c_p, I+2);
 	 SET_I(next);
 	 Dispatch();
@@ -3248,12 +3189,10 @@ do {								\
  OpCase(apply_last_IP): {
      BeamInstr *next;
 
-     reg[0] = r(0);
      SWAPOUT;
      next = fixed_apply(c_p, reg, Arg(0));
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(1));
 	 SET_I(next);
@@ -3270,7 +3209,6 @@ do {								\
      next = apply_fun(c_p, r(0), x(1), reg);
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_CP(c_p, I+1);
 	 SET_I(next);
 	 Dispatchfun();
@@ -3285,7 +3223,6 @@ do {								\
      next = apply_fun(c_p, r(0), x(1), reg);
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
@@ -3301,7 +3238,6 @@ do {								\
      next = apply_fun(c_p, r(0), x(1), reg);
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_I(next);
 	 Dispatchfun();
      }
@@ -3312,12 +3248,9 @@ do {								\
      BeamInstr *next;
 
      SWAPOUT;
-     reg[0] = r(0);
-
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
      SWAPIN;
      if (next != NULL) {
-	 r(0) = reg[0];
 	 SET_CP(c_p, I+2);
 	 SET_I(next);
 	 Dispatchfun();
@@ -3329,11 +3262,9 @@ do {								\
      BeamInstr *next;
 
      SWAPOUT;
-     reg[0] = r(0);
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
      SWAPIN;
      if (next != NULL) {
-	r(0) = reg[0];
 	SET_CP(c_p, (BeamInstr *) E[0]);
 	E = ADD_BYTE_OFFSET(E, Arg(1));
 	SET_I(next);
@@ -3418,10 +3349,9 @@ do {								\
       */
 
      argp = c_p->arg_reg;
-     for (i = c_p->arity - 1; i > 0; i--) {
+     for (i = c_p->arity - 1; i >= 0; i--) {
 	 argp[i] = reg[i];
      }
-     c_p->arg_reg[0] = r(0);
      SWAPOUT;
      c_p->i = I;
      goto do_schedule1;
@@ -3531,7 +3461,6 @@ do {								\
  /* Fall through here */
 
  find_func_info: {
-     reg[0] = r(0);
      SWAPOUT;
      I = handle_error(c_p, I, reg, NULL);
      goto post_error_handling;
@@ -3549,9 +3478,7 @@ do {								\
      * code[4]: Not used
      */
     SWAPOUT;
-    reg[0] = r(0);
     I = call_error_handler(c_p, I-3, reg, am_undefined_function);
-    r(0) = reg[0];
     SWAPIN;
     if (I) {
 	Goto(*I);
@@ -3560,14 +3487,12 @@ do {								\
  /* Fall through */
  OpCase(error_action_code): {
     handle_error:
-     reg[0] = r(0);
      SWAPOUT;
      I = handle_error(c_p, NULL, reg, NULL);
  post_error_handling:
      if (I == 0) {
 	 goto do_schedule;
      } else {
-	 r(0) = reg[0];
 	 ASSERT(!is_value(r(0)));
 	 if (c_p->mbuf) {
 	     erts_garbage_collect(c_p, 0, reg+1, 3);
@@ -3610,7 +3535,6 @@ do {								\
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
-		reg[0] = r(0);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
 		    nif_bif_result = THE_NON_VALUE;
@@ -3649,7 +3573,7 @@ do {								\
 	    bif_nif_arity = I[-1];
             ASSERT(bif_nif_arity <= 4);
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-	    reg[0] = r(0);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	    {
 		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -3679,7 +3603,6 @@ do {								\
 		Goto(*I);
 	    } else if (c_p->freason == TRAP) {
 		SET_I(c_p->i);
-		r(0) = reg[0];
 		if (c_p->flags & F_HIBERNATE_SCHED) {
 		    c_p->flags &= ~F_HIBERNATE_SCHED;
 		    goto do_schedule;
@@ -4126,10 +4049,8 @@ do {								\
      Uint res;
 
      SWAPOUT;
-     reg[0] = r(0);
      reg[live] = tmp_arg2;
      res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3));
-     r(0) = reg[0];
      SWAPIN;
      if (is_non_value(res)) {
 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
@@ -4706,7 +4627,7 @@ do {								\
      Uint hole_size;
 
      OpCase(bs_context_to_binary_r): {
-	 context_to_binary_context = x0;
+	 context_to_binary_context = r(0);
 	 I -= 2;
 	 goto do_context_to_binary;
      }
@@ -4737,7 +4658,7 @@ do {								\
      Next(2);
 
      OpCase(i_bs_get_binary_all_reuse_rfI): {
-	 context_to_binary_context = x0;
+	 context_to_binary_context = r(0);
 	 goto do_bs_get_binary_all_reuse;
      }
 
@@ -4883,9 +4804,7 @@ do {								\
      BeamInstr real_I;
      ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
      SWAPOUT;
-     reg[0] = r(0);
      real_I = erts_generic_breakpoint(c_p, I, reg);
-     r(0) = reg[0];
      SWAPIN;
      ASSERT(VALID_INSTR(real_I));
      Goto(real_I);
@@ -5103,7 +5022,6 @@ do {								\
      SWAPOUT;
      c_p->fcalls = FCALLS;
      c_p->def_arg_reg[4] = -neg_o_reds;
-     reg[0] = r(0);
      c_p = hipe_mode_switch(c_p, cmd, reg);
      reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
      freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
@@ -5120,7 +5038,6 @@ do {								\
 	 /*fall through*/
        case HIPE_MODE_SWITCH_RES_CALL_BEAM:
 	 SET_I(c_p->i);
-	 r(0) = reg[0];
 	 Dispatch();
        case HIPE_MODE_SWITCH_RES_CALL_CLOSURE:
 	 /* This can be used to call any function value, but currently it's
@@ -5131,7 +5048,6 @@ do {								\
 	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
 	     SWAPIN;
 	     if (next != NULL) {
-		 r(0) = reg[0];
 		 SET_I(next);
 		 Dispatchfun();
 	     }
@@ -5191,9 +5107,7 @@ do {								\
 
  OpCase(i_debug_breakpoint): {
      SWAPOUT;
-     reg[0] = r(0);
      I = call_error_handler(c_p, I-3, reg, am_breakpoint);
-     r(0) = reg[0];
      SWAPIN;
      if (I) {
 	 Goto(*I);
-- 
cgit v1.2.3


From e1019cbba7a66788a068b6df1a1caf2d643ef65b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 30 Mar 2012 11:54:04 +0200
Subject: Eliminate R_REG_DEF

---
 erts/emulator/beam/beam_debug.c | 22 ++--------------------
 erts/emulator/beam/beam_emu.c   | 14 --------------
 erts/emulator/beam/beam_load.c  |  8 ++------
 erts/emulator/beam/erl_term.h   |  2 --
 4 files changed, 4 insertions(+), 42 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8a35ad17c6..00d0b27178 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -452,21 +452,13 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 's':		/* Any source (tagged constant or register) */
 	    tag = beam_reg_tag(*ap);
 	    if (tag == X_REG_DEF) {
-		if (reg_index(*ap) == 0) {
-		    erts_print(to, to_arg, "x[0]");
-		} else {
-		    erts_print(to, to_arg, "x(%d)", reg_index(*ap));
-		}
+		erts_print(to, to_arg, "x(%d)", reg_index(*ap));
 		ap++;
 		break;
 	    } else if (tag == Y_REG_DEF) {
 		erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE);
 		ap++;
 		break;
-	    } else if (tag == R_REG_DEF) {
-		erts_print(to, to_arg, "x(0)");
-		ap++;
-		break;
 	    }
 	    /*FALLTHROUGH*/
 	case 'a':		/* Tagged atom */
@@ -483,18 +475,11 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 'd':		/* Destination (x(0), x(N), y(N)) */
 	    switch (beam_reg_tag(*ap)) {
 	    case X_REG_DEF:
-		if (reg_index(*ap) == 0) {
-		    erts_print(to, to_arg, "x[0]");
-		} else {
-		    erts_print(to, to_arg, "x(%d)", reg_index(*ap));
-		}
+		erts_print(to, to_arg, "x(%d)", reg_index(*ap));
 		break;
 	    case Y_REG_DEF:
 		erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE);
 		break;
-	    case R_REG_DEF:
-		erts_print(to, to_arg, "x(0)");
-		break;
 	    }
 	    ap++;
 	    break;
@@ -677,9 +662,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
 		} else {
 		    switch ((ap[0] >> 2) & 0x03) {
-		    case R_REG_DEF:
-			erts_print(to, to_arg, " x(0)");
-			break;
 		    case X_REG_DEF:
 			erts_print(to, to_arg, " x(%d)", ap[0] >> 4);
 			break;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9a34d5f11a..059c443450 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -160,8 +160,6 @@ do {                                     \
     stb_reg = (DestDesc);                           \
     CHECK_TERM(Result);                             \
     switch (beam_reg_tag(stb_reg)) {                \
-    case R_REG_DEF:                                 \
-      r(0) = (Result); break;                       \
     case X_REG_DEF:                                 \
       xb(x_reg_offset(stb_reg)) = (Result); break;  \
     default:                                        \
@@ -186,8 +184,6 @@ do {                                     \
     stb_next = (BeamInstr *) *I;                                 \
     CHECK_TERM(Result);                                      \
     switch (beam_reg_tag(stb_reg)) {                         \
-    case R_REG_DEF:                                          \
-      r(0) = (Result); Goto(stb_next);                       \
     case X_REG_DEF:                                          \
       xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
     default:                                                 \
@@ -521,7 +517,6 @@ void** beam_ops;
    do { \
      tr = Arg(pos); \
      switch (beam_reg_tag(tr)) { \
-     case R_REG_DEF: tr = r(0); break; \
      case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \
      case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \
      } \
@@ -2391,9 +2386,6 @@ void process_main(void)
      do {
 	 Eterm term = *I++;
 	 switch (term & _TAG_IMMED1_MASK) {
-	 case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
-	     *hp++ = r(0);
-	     break;
 	 case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
 	     *hp++ = x(term >> _TAG_IMMED1_SIZE);
 	     break;
@@ -2422,9 +2414,6 @@ void process_main(void)
 #define PUT_TERM_REG(term, desc)				\
 do {								\
     switch ((desc) & _TAG_IMMED1_MASK) {			\
-    case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
-	r(0) = (term);						\
-	break;							\
     case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
 	x((desc) >> _TAG_IMMED1_SIZE) = (term);			\
 	break;							\
@@ -6502,9 +6491,6 @@ static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx)
 do {								\
     Eterm src = (Eterm)(term);					\
     switch (src & _TAG_IMMED1_MASK) {				\
-    case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
-	dest = x(0);						\
-	break;							\
     case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
 	dest = x(src >> _TAG_IMMED1_SIZE);			\
 	break;							\
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index f5f1147261..495e92d600 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2167,7 +2167,7 @@ load_code(LoaderState* stp)
 	    case 's':	/* Any source (tagged constant or register) */
 		switch (tag) {
 		case TAG_r:
-		    code[ci++] = make_rreg();
+		    code[ci++] = make_xreg(0);
 		    break;
 		case TAG_x:
 		    code[ci++] = make_xreg(tmp_op->a[arg].val);
@@ -2193,7 +2193,7 @@ load_code(LoaderState* stp)
 	    case 'd':	/* Destination (x(0), x(N), y(N) */
 		switch (tag) {
 		case TAG_r:
-		    code[ci++] = make_rreg();
+		    code[ci++] = make_xreg(0);
 		    break;
 		case TAG_x:
 		    code[ci++] = make_xreg(tmp_op->a[arg].val);
@@ -2357,10 +2357,6 @@ load_code(LoaderState* stp)
 		ci++;
 		break;
 	    case TAG_r:
-		CodeNeed(1);
-		code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) |
-		    TAG_PRIMARY_HEADER;
-		break;
 	    case TAG_x:
 		CodeNeed(1);
 		code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index b928e4ca1e..cff2430547 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1037,11 +1037,9 @@ _ET_DECLARE_CHECKED(Uint,catch_val,Eterm)
 
 #define X_REG_DEF	0
 #define Y_REG_DEF	1
-#define R_REG_DEF	2
 
 #define beam_reg_tag(x)	((x) & 3)
 
-#define make_rreg()	R_REG_DEF
 #define make_xreg(ix)	(((ix) * sizeof(Eterm)) | X_REG_DEF)
 #define make_yreg(ix)	(((ix) * sizeof(Eterm)) | Y_REG_DEF)
 
-- 
cgit v1.2.3


From f0923b143ecfdacfe4873fc1e96c8ec69726a4d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 16 Jun 2015 06:39:25 +0200
Subject: Allow X and Y registers to be overloaded with any literal

Consider the try_case_end instruction:

  try_case_end s

The 's' operand type means that the operand can either be a
literal of one of the types atom, integer, or empty list, or
a register. That worked well before R12. In R12 additional
types of literals where introduced. Because of way the
overloading was done, an 's' operand cannot handle the
new types of literals. Therefore, code such as the following
is necessary in ops.tab to avoid giving an 's' operand a
literal:

  try_case_end Literal=q => move Literal x | try_case_end x

While this work, it is error-prone in that it is easy to
forget to add that kind of rule. It would also be complicated
in case we wanted to introduce a new kind of addition operator
such as:

  i_plus jssd

Since there are two 's' operands, two scratch registers and
two 'move' instructions would be needed.

Therefore, we'll need to find a smarter way to find tag
register operands. We will overload the pid and port tags
for X and Y register, respectively. That works because pids
and port are immediate values (fit in one word), and there
are no literals for pids and ports.
---
 erts/emulator/beam/beam_debug.c |  77 +++++++++++-----------
 erts/emulator/beam/beam_emu.c   | 139 +++++++++++++++++++++-------------------
 erts/emulator/beam/beam_load.c  |  22 ++++---
 erts/emulator/beam/erl_term.c   |   6 +-
 erts/emulator/beam/erl_term.h   |  50 +++++++--------
 erts/emulator/beam/ops.tab      |  29 +--------
 6 files changed, 153 insertions(+), 170 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 00d0b27178..243ccb9afd 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -432,31 +432,33 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     while (*sign) {
 	switch (*sign) {
 	case 'r':		/* x(0) */
-	    erts_print(to, to_arg, "x(0)");
+	    erts_print(to, to_arg, "r(0)");
 	    break;
 	case 'x':		/* x(N) */
-	    if (reg_index(ap[0]) == 0) {
-		erts_print(to, to_arg, "x[0]");
-	    } else {
-		erts_print(to, to_arg, "x(%d)", reg_index(ap[0]));
+	    {
+		Uint n = ap[0] / sizeof(Eterm);
+		erts_print(to, to_arg, "x(%d)", n);
+		ap++;
 	    }
-	    ap++;
 	    break;
 	case 'y':		/* y(N) */
-	    erts_print(to, to_arg, "y(%d)", reg_index(ap[0]) - CP_SIZE);
-	    ap++;
+	    {
+		Uint n = ap[0] / sizeof(Eterm) - CP_SIZE;
+		erts_print(to, to_arg, "y(%d)", n);
+		ap++;
+	    }
 	    break;
 	case 'n':		/* Nil */
 	    erts_print(to, to_arg, "[]");
 	    break;
 	case 's':		/* Any source (tagged constant or register) */
-	    tag = beam_reg_tag(*ap);
-	    if (tag == X_REG_DEF) {
-		erts_print(to, to_arg, "x(%d)", reg_index(*ap));
+	    tag = loader_tag(*ap);
+	    if (tag == LOADER_X_REG) {
+		erts_print(to, to_arg, "x(%d)", loader_x_reg_index(*ap));
 		ap++;
 		break;
-	    } else if (tag == Y_REG_DEF) {
-		erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE);
+	    } else if (tag == LOADER_Y_REG) {
+		erts_print(to, to_arg, "y(%d)", loader_y_reg_index(*ap) - CP_SIZE);
 		ap++;
 		break;
 	    }
@@ -473,12 +475,14 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    ap++;
 	    break;
 	case 'd':		/* Destination (x(0), x(N), y(N)) */
-	    switch (beam_reg_tag(*ap)) {
-	    case X_REG_DEF:
-		erts_print(to, to_arg, "x(%d)", reg_index(*ap));
+	    switch (loader_tag(*ap)) {
+	    case LOADER_X_REG:
+		erts_print(to, to_arg, "x(%d)",
+			   loader_x_reg_index(*ap));
 		break;
-	    case Y_REG_DEF:
-		erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE);
+	    case LOADER_Y_REG:
+		erts_print(to, to_arg, "y(%d)",
+			   loader_y_reg_index(*ap) - CP_SIZE);
 		break;
 	    }
 	    ap++;
@@ -546,7 +550,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    ap++;
 	    break;
 	case 'l':		/* fr(N) */
-	    erts_print(to, to_arg, "fr(%d)", reg_index(ap[0]));
+	    erts_print(to, to_arg, "fr(%d)", loader_reg_index(ap[0]));
 	    ap++;
 	    break;
 	default:
@@ -658,17 +662,16 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    int n = unpacked[-1];
 
 	    while (n > 0) {
-		if (!is_header(ap[0])) {
+		switch (loader_tag(ap[0])) {
+		case LOADER_X_REG:
+		    erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0]));
+		    break;
+		case LOADER_Y_REG:
+		    erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0]));
+		    break;
+		default:
 		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
-		} else {
-		    switch ((ap[0] >> 2) & 0x03) {
-		    case X_REG_DEF:
-			erts_print(to, to_arg, " x(%d)", ap[0] >> 4);
-			break;
-		    case Y_REG_DEF:
-			erts_print(to, to_arg, " y(%d)", ap[0] >> 4);
-			break;
-		    }
+		    break;
 		}
 		ap++, size++, n--;
 	    }
@@ -681,18 +684,16 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    while (n > 0) {
 		if (n % 3 == 1) {
 		    erts_print(to, to_arg, " %X", ap[0]);
-		} else if (!is_header(ap[0])) {
-		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
 		} else {
-		    switch ((ap[0] >> 2) & 0x03) {
-		    case R_REG_DEF:
-			erts_print(to, to_arg, " x(0)");
+		    switch (loader_tag(ap[0])) {
+		    case LOADER_X_REG:
+			erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0]));
 			break;
-		    case X_REG_DEF:
-			erts_print(to, to_arg, " x(%d)", ap[0] >> 4);
+		    case LOADER_Y_REG:
+			erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]));
 			break;
-		    case Y_REG_DEF:
-			erts_print(to, to_arg, " y(%d)", ap[0] >> 4);
+		    default:
+			erts_print(to, to_arg, " %T", (Eterm) ap[0]);
 			break;
 		    }
 		}
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 059c443450..a74da2d872 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -154,17 +154,19 @@ do {                                     \
  * Store a result into a register given a destination descriptor.
  */
 
-#define StoreResult(Result, DestDesc)               \
-  do {                                              \
-    Eterm stb_reg;                                  \
-    stb_reg = (DestDesc);                           \
-    CHECK_TERM(Result);                             \
-    switch (beam_reg_tag(stb_reg)) {                \
-    case X_REG_DEF:                                 \
-      xb(x_reg_offset(stb_reg)) = (Result); break;  \
-    default:                                        \
-      yb(y_reg_offset(stb_reg)) = (Result); break;  \
-    }                                               \
+#define StoreResult(Result, DestDesc)			\
+  do {							\
+    Eterm stb_reg;					\
+    stb_reg = (DestDesc);				\
+    CHECK_TERM(Result);					\
+    switch (loader_tag(stb_reg)) {			\
+    case LOADER_X_REG:					\
+      x(loader_x_reg_index(stb_reg)) = (Result);	\
+      break;						\
+    default:						\
+      y(loader_y_reg_index(stb_reg)) = (Result);	\
+      break;						\
+    }							\
   } while (0)
 
 #define StoreSimpleDest(Src, Dest) Dest = (Src)
@@ -175,20 +177,22 @@ do {                                     \
  * be just before the next instruction.
  */
  
-#define StoreBifResult(Dst, Result)                          \
-  do {                                                       \
-    BeamInstr* stb_next;                                         \
-    Eterm stb_reg;                                           \
-    stb_reg = Arg(Dst);                                      \
-    I += (Dst) + 2;                                          \
-    stb_next = (BeamInstr *) *I;                                 \
-    CHECK_TERM(Result);                                      \
-    switch (beam_reg_tag(stb_reg)) {                         \
-    case X_REG_DEF:                                          \
-      xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
-    default:                                                 \
-      yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
-    }                                                        \
+#define StoreBifResult(Dst, Result)			\
+  do {							\
+    BeamInstr* stb_next;				\
+    Eterm stb_reg;					\
+    stb_reg = Arg(Dst);					\
+    I += (Dst) + 2;					\
+    stb_next = (BeamInstr *) *I;			\
+    CHECK_TERM(Result);					\
+    switch (loader_tag(stb_reg)) {			\
+    case LOADER_X_REG:					\
+      x(loader_x_reg_index(stb_reg)) = (Result);	\
+      Goto(stb_next);					\
+    default:						\
+      y(loader_y_reg_index(stb_reg)) = (Result);	\
+      Goto(stb_next);					\
+    }							\
   } while (0)
 
 #define ClauseFail() goto jump_f
@@ -513,14 +517,19 @@ void** beam_ops;
     ASSERT(VALID_INSTR(Dst));  \
     Goto(Dst)
 
-#define GetR(pos, tr) \
-   do { \
-     tr = Arg(pos); \
-     switch (beam_reg_tag(tr)) { \
-     case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \
-     case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \
-     } \
-     CHECK_TERM(tr); \
+#define GetR(pos, tr)				\
+   do {						\
+     tr = Arg(pos);				\
+     switch (loader_tag(tr)) {			\
+     case LOADER_X_REG:				\
+        tr = x(loader_x_reg_index(tr));		\
+        break;					\
+     case LOADER_Y_REG:				\
+        ASSERT(loader_y_reg_index(tr) >= 1);	\
+        tr = y(loader_y_reg_index(tr));		\
+        break;					\
+     }						\
+     CHECK_TERM(tr);				\
    } while (0)
 
 #define GetArg1(N, Dst) GetR((N), Dst)
@@ -2385,12 +2394,12 @@ void process_main(void)
 
      do {
 	 Eterm term = *I++;
-	 switch (term & _TAG_IMMED1_MASK) {
-	 case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
-	     *hp++ = x(term >> _TAG_IMMED1_SIZE);
+	 switch (loader_tag(term)) {
+	 case LOADER_X_REG:
+	     *hp++ = x(loader_x_reg_index(term));
 	     break;
-	 case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
-	     *hp++ = y(term >> _TAG_IMMED1_SIZE);
+	 case LOADER_Y_REG:
+	     *hp++ = y(loader_y_reg_index(term));
 	     break;
 	 default:
 	     *hp++ = term;
@@ -2411,19 +2420,19 @@ void process_main(void)
      Next(3+Arg(2));
  }
 
-#define PUT_TERM_REG(term, desc)				\
-do {								\
-    switch ((desc) & _TAG_IMMED1_MASK) {			\
-    case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
-	x((desc) >> _TAG_IMMED1_SIZE) = (term);			\
-	break;							\
-    case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
-	y((desc) >> _TAG_IMMED1_SIZE) = (term);			\
-	break;							\
-    default:							\
-	ASSERT(0);						\
-	break;							\
-    }								\
+#define PUT_TERM_REG(term, desc)		\
+do {						\
+    switch (loader_tag(desc)) {			\
+    case LOADER_X_REG:				\
+	x(loader_x_reg_index(desc)) = (term);	\
+	break;					\
+    case LOADER_Y_REG:				\
+	y(loader_y_reg_index(desc)) = (term);	\
+	break;					\
+    default:					\
+	ASSERT(0);				\
+	break;					\
+    }						\
 } while(0)
 
  OpCase(i_get_map_elements_fsI): {
@@ -6487,20 +6496,20 @@ static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx)
     return vs ? *vs : THE_NON_VALUE;
 }
 
-#define GET_TERM(term, dest)					\
-do {								\
-    Eterm src = (Eterm)(term);					\
-    switch (src & _TAG_IMMED1_MASK) {				\
-    case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
-	dest = x(src >> _TAG_IMMED1_SIZE);			\
-	break;							\
-    case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
-	dest = y(src >> _TAG_IMMED1_SIZE);			\
-	break;							\
-    default:							\
-	dest = src;						\
-	break;							\
-    }								\
+#define GET_TERM(term, dest)			\
+do {						\
+    Eterm src = (Eterm)(term);			\
+    switch (loader_tag(src)) {			\
+    case LOADER_X_REG:				\
+        dest = x(loader_x_reg_index(src));	\
+	break;					\
+    case LOADER_Y_REG:				\
+        dest = y(loader_y_reg_index(src));	\
+	break;					\
+    default:					\
+	dest = src;				\
+	break;					\
+    }						\
 } while(0)
 
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 495e92d600..3cd80c4ec2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2167,13 +2167,13 @@ load_code(LoaderState* stp)
 	    case 's':	/* Any source (tagged constant or register) */
 		switch (tag) {
 		case TAG_r:
-		    code[ci++] = make_xreg(0);
+		    code[ci++] = make_loader_x_reg(0);
 		    break;
 		case TAG_x:
-		    code[ci++] = make_xreg(tmp_op->a[arg].val);
+		    code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
 		    break;
 		case TAG_y:
-		    code[ci++] = make_yreg(tmp_op->a[arg].val);
+		    code[ci++] = make_loader_y_reg(tmp_op->a[arg].val);
 		    break;
 		case TAG_i:
 		    code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val);
@@ -2184,6 +2184,10 @@ load_code(LoaderState* stp)
 		case TAG_n:
 		    code[ci++] = NIL;
 		    break;
+		case TAG_q:
+		    new_literal_patch(stp, ci);
+		    code[ci++] = tmp_op->a[arg].val;
+		    break;
 		default:
 		    LoadError1(stp, "bad tag %d for general source",
 			       tmp_op->a[arg].type);
@@ -2193,13 +2197,13 @@ load_code(LoaderState* stp)
 	    case 'd':	/* Destination (x(0), x(N), y(N) */
 		switch (tag) {
 		case TAG_r:
-		    code[ci++] = make_xreg(0);
+		    code[ci++] = make_loader_x_reg(0);
 		    break;
 		case TAG_x:
-		    code[ci++] = make_xreg(tmp_op->a[arg].val);
+		    code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
 		    break;
 		case TAG_y:
-		    code[ci++] = make_yreg(tmp_op->a[arg].val);
+		    code[ci++] = make_loader_y_reg(tmp_op->a[arg].val);
 		    break;
 		default:
 		    LoadError1(stp, "bad tag %d for destination",
@@ -2359,13 +2363,11 @@ load_code(LoaderState* stp)
 	    case TAG_r:
 	    case TAG_x:
 		CodeNeed(1);
-		code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
-		    (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+		code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
 		break;
 	    case TAG_y:
 		CodeNeed(1);
-		code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
-		    (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+		code[ci++] = make_loader_y_reg(tmp_op->a[arg].val);
 		break;
 	    case TAG_n:
 		CodeNeed(1);
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 89459fb278..3a5fbcc284 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -173,9 +173,7 @@ ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr);
 ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer);
 ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP);
 ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch);
-ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg);
-ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg);
-ET_DEFINE_CHECKED(Uint,x_reg_index,Uint,_is_xreg);
-ET_DEFINE_CHECKED(Uint,y_reg_index,Uint,_is_yreg);
+ET_DEFINE_CHECKED(Uint,loader_x_reg_index,Uint,_is_loader_x_reg);
+ET_DEFINE_CHECKED(Uint,loader_y_reg_index,Uint,_is_loader_y_reg);
 
 #endif	/* ET_DEBUG */
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index cff2430547..089eecf024 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1027,42 +1027,40 @@ _ET_DECLARE_CHECKED(Uint,catch_val,Eterm)
 /*
  * Overloaded tags.
  *
- * SMALL = 15
- * ATOM/NIL=7
+ * In the loader, we want to tag a term in a way so that it can
+ * be any literal (atom/integer/float/tuple/list/binary) or a
+ * register.
  *
- * Note that the two least significant bits in SMALL/ATOM/NIL always are 3;
- * thus, we can distinguish register from literals by looking at only these
- * two bits.
+ * We can achive that by overloading the PID and PORT tags to
+ * mean X and Y registers. That works because there are no
+ * pid or port literals.
  */
 
-#define X_REG_DEF	0
-#define Y_REG_DEF	1
+#define _LOADER_TAG_XREG _TAG_IMMED1_PID
+#define _LOADER_TAG_YREG _TAG_IMMED1_PORT
+#define _LOADER_TAG_SIZE _TAG_IMMED1_SIZE
+#define _LOADER_MASK _TAG_IMMED1_MASK
 
-#define beam_reg_tag(x)	((x) & 3)
+#define LOADER_X_REG _LOADER_TAG_XREG
+#define LOADER_Y_REG _LOADER_TAG_YREG
 
-#define make_xreg(ix)	(((ix) * sizeof(Eterm)) | X_REG_DEF)
-#define make_yreg(ix)	(((ix) * sizeof(Eterm)) | Y_REG_DEF)
+#define make_loader_x_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_XREG)
+#define make_loader_y_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_YREG)
 
-#define _is_xreg(x)	(beam_reg_tag(x) == X_REG_DEF)
-#define _is_yreg(x)	(beam_reg_tag(x) == Y_REG_DEF)
+#define loader_reg_index(R) ((R) >> _LOADER_TAG_SIZE)
 
-#define _unchecked_x_reg_offset(R)	((R) - X_REG_DEF)
-_ET_DECLARE_CHECKED(Uint,x_reg_offset,Uint)
-#define x_reg_offset(R)	_ET_APPLY(x_reg_offset,(R))
+#define loader_tag(T) ((T) & _LOADER_MASK)
 
-#define _unchecked_y_reg_offset(R)	((R) - Y_REG_DEF)
-_ET_DECLARE_CHECKED(Uint,y_reg_offset,Uint)
-#define y_reg_offset(R)	_ET_APPLY(y_reg_offset,(R))
+#define _is_loader_x_reg(x) (loader_tag(x) == _LOADER_TAG_XREG)
+#define _is_loader_y_reg(x) (loader_tag(x) == _LOADER_TAG_YREG)
 
-#define reg_index(R) ((R) / sizeof(Eterm))
+#define _unchecked_loader_x_reg_index(R) ((R) >> _LOADER_TAG_SIZE)
+_ET_DECLARE_CHECKED(Uint,loader_x_reg_index,Uint)
+#define loader_x_reg_index(R) _ET_APPLY(loader_x_reg_index,(R))
 
-#define _unchecked_x_reg_index(R)	((R) >> 2)
-_ET_DECLARE_CHECKED(Uint,x_reg_index,Uint)
-#define x_reg_index(R)	_ET_APPLY(x_reg_index,(R))
-
-#define _unchecked_y_reg_index(R)	((R) >> 2)
-_ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
-#define y_reg_index(R)	_ET_APPLY(y_reg_index,(R))
+#define _unchecked_loader_y_reg_index(R)  ((R) >> _LOADER_TAG_SIZE)
+_ET_DECLARE_CHECKED(Uint,loader_y_reg_index,Uint)
+#define loader_y_reg_index(R) _ET_APPLY(loader_y_reg_index,(R))
 
 /*
  * Backwards compatibility definitions:
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 1d32e72247..4fa6a087e2 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -237,12 +237,10 @@ try Y F => catch Y F
 try_case Y => try_end Y
 try_end y
 
-try_case_end Literal=q => move Literal x | try_case_end x
 try_case_end s
 
 # Destructive set tuple element
 
-set_tuple_element Lit=q Tuple Pos => move Lit x | set_tuple_element x Tuple Pos
 set_tuple_element s d P
 
 # Get tuple element
@@ -273,8 +271,8 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
 
 jump f
 
-case_end Literal=cq => move Literal x | case_end x
-badmatch Literal=cq => move Literal x | badmatch x
+case_end Literal=c => move Literal x | case_end x
+badmatch Literal=c => move Literal x | badmatch x
 
 case_end r
 case_end x
@@ -849,7 +847,6 @@ is_boolean f y
 
 is_function2 Fail=f acq Arity => jump Fail
 is_function2 Fail=f Fun a => jump Fail
-is_function2 Fail Fun Literal=q => move Literal x | is_function2 Fail Fun x
 
 is_function2 f s s
 %macro: is_function2 IsFunction2 -fail_action
@@ -1122,11 +1119,8 @@ bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst
 
 bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst)
 
-bif1 Fail Bif Literal=q Dst => move Literal x | bif1 Fail Bif x Dst
 bif1 p Bif S1 Dst => bif1_body Bif S1 Dst
 
-bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst
-
 bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst
 bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst
 
@@ -1439,8 +1433,6 @@ i_bs_private_append j I d
 # Storing integers into binaries.
 #
 
-bs_put_integer Fail=j Sz=s Unit=u Flags=u Literal=q => \
-	move Literal x | bs_put_integer Fail Sz Unit Flags x
 bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
 			gen_put_integer(Fail, Sz, Unit, Flags, Src)
 
@@ -1454,32 +1446,22 @@ i_new_bs_put_integer_imm j I I s
 # Utf8/utf16/utf32 support. (R12B-5)
 #
 
-bs_utf8_size Fail Literal=q Dst=d => \
-	move Literal x | bs_utf8_size Fail x Dst
 bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst
 
 i_bs_utf8_size s d
 
-bs_utf16_size Fail Literal=q Dst=d => \
-	move Literal x | bs_utf16_size Fail x Dst
 bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst
 
 i_bs_utf16_size s d
 
-bs_put_utf8 Fail=j Flags=u Literal=q => \
-	move Literal x | bs_put_utf8 Fail Flags x
 bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
 
 i_bs_put_utf8 j s
 
-bs_put_utf16 Fail=j Flags=u Literal=q => \
-	move Literal x | bs_put_utf16 Fail Flags x
 bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src
 
 i_bs_put_utf16 j I s
 
-bs_put_utf32 Fail=j Flags=u Literal=q => \
-	move Literal x | bs_put_utf32 Fail Flags x
 bs_put_utf32 Fail=j Flags=u Src=s => \
    i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
 
@@ -1490,9 +1472,6 @@ i_bs_validate_unicode j s
 #
 bs_put_float Fail Sz=q Unit Flags Val => badarg Fail
 
-bs_put_float Fail=j Sz Unit=u Flags=u Literal=q => \
-	move Literal x | bs_put_float Fail Sz Unit Flags x
-
 bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_float(Fail, Sz, Unit, Flags, Src)
 
@@ -1506,8 +1485,6 @@ i_new_bs_put_float_imm j I I s
 # Storing binaries into binaries.
 #
 
-bs_put_binary Fail Sz Unit Flags Literal=q => \
- 	move Literal x | bs_put_binary Fail Sz Unit Flags x
 bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_binary(Fail, Sz, Unit, Flags, Src)
 
@@ -1731,8 +1708,6 @@ gc_bif2 Fail I Bif S1 S2 Dst => \
 gc_bif3 Fail I Bif S1 S2 S3 Dst => \
 	gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst)
 
-i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D
-
 i_gc_bif1 j I s I d
 
 ii_gc_bif2/6
-- 
cgit v1.2.3


From 135c9821e8806defa455a47208fb897fff1a3ff8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 23 Jun 2015 08:53:50 +0200
Subject: Make the 'r' operand type optional

The 'r' type is now mandatory. That means in order to handle
both of the following instructions:

  move x(0) y(7)
  move x(1) y(7)

we would need to define two specific operations in ops.tab:

  move r y
  move x y

We want to make 'r' operands optional. That is, if we have
only this specific instruction:

  move x y

it will match both of the following instructions:

  move x(0) y(7)
  move x(1) y(7)

Make 'r' optional allows us to save code space when we don't
want to make handling of x(0) a special case, but we can still
use 'r' to optimize commonly used instructions.
---
 erts/emulator/beam/beam_debug.c |   6 -
 erts/emulator/beam/beam_emu.c   | 355 ++++++-------------------------
 erts/emulator/beam/beam_load.c  |  66 ++++--
 erts/emulator/beam/erl_vm.h     |   1 +
 erts/emulator/beam/ops.tab      | 456 ++++++++++++++--------------------------
 5 files changed, 271 insertions(+), 613 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 243ccb9afd..30e3765502 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -569,7 +569,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
     unpacked = ap;
     ap = addr + size;
     switch (op) {
-    case op_i_select_val_lins_rfI:
     case op_i_select_val_lins_xfI:
     case op_i_select_val_lins_yfI:
 	{
@@ -589,7 +588,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
-    case op_i_select_val_bins_rfI:
     case op_i_select_val_bins_xfI:
     case op_i_select_val_bins_yfI:
 	{
@@ -603,7 +601,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
-    case op_i_select_tuple_arity_rfI:
     case op_i_select_tuple_arity_xfI:
     case op_i_select_tuple_arity_yfI:
         {
@@ -628,7 +625,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
             }
         }
         break;
-    case op_i_jump_on_val_rfII:
     case op_i_jump_on_val_xfII:
     case op_i_jump_on_val_yfII:
 	{
@@ -640,7 +636,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
-    case op_i_jump_on_val_zero_rfI:
     case op_i_jump_on_val_zero_xfI:
     case op_i_jump_on_val_zero_yfI:
 	{
@@ -652,7 +647,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
-    case op_i_put_tuple_rI:
     case op_i_put_tuple_xI:
     case op_i_put_tuple_yI:
     case op_new_map_dII:
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a74da2d872..e5179830de 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -556,12 +556,12 @@ void** beam_ops;
 #define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2)
 #define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)
 
-#define MoveReturn(Src, Dest)       \
-    (Dest) = (Src);                 \
-    I = c_p->cp;                    \
-    ASSERT(VALID_INSTR(*c_p->cp));  \
-    c_p->cp = 0;                    \
-    CHECK_TERM(r(0));               \
+#define MoveReturn(Src)				\
+    x(0) = (Src);				\
+    I = c_p->cp;				\
+    ASSERT(VALID_INSTR(*c_p->cp));		\
+    c_p->cp = 0;				\
+    CHECK_TERM(r(0));				\
     Goto(*I)
 
 #define DeallocateReturn(Deallocate)       \
@@ -573,26 +573,26 @@ void** beam_ops;
     Goto(*I);                              \
   } while (0)
 
-#define MoveDeallocateReturn(Src, Dest, Deallocate)  \
-    (Dest) = (Src);                                  \
+#define MoveDeallocateReturn(Src, Deallocate)	\
+    x(0) = (Src);				\
     DeallocateReturn(Deallocate)
 
-#define MoveCall(Src, Dest, CallDest, Size)	\
-    (Dest) = (Src);				\
+#define MoveCall(Src, CallDest, Size)		\
+    x(0) = (Src);				\
     SET_CP(c_p, I+Size+1);			\
-    SET_I((BeamInstr *) CallDest);			\
+    SET_I((BeamInstr *) CallDest);		\
     Dispatch();
 
-#define MoveCallLast(Src, Dest, CallDest, Deallocate)	\
-    (Dest) = (Src);					\
-    RESTORE_CP(E);					\
-    E = ADD_BYTE_OFFSET(E, (Deallocate));		\
-    SET_I((BeamInstr *) CallDest);				\
+#define MoveCallLast(Src, CallDest, Deallocate)	\
+    x(0) = (Src);				\
+    RESTORE_CP(E);				\
+    E = ADD_BYTE_OFFSET(E, (Deallocate));	\
+    SET_I((BeamInstr *) CallDest);		\
     Dispatch();
 
-#define MoveCallOnly(Src, Dest, CallDest)	\
-    (Dest) = (Src);				\
-    SET_I((BeamInstr *) CallDest);			\
+#define MoveCallOnly(Src, CallDest)		\
+    x(0) = (Src);				\
+    SET_I((BeamInstr *) CallDest);		\
     Dispatch();
 
 #define MoveJump(Src)				\
@@ -676,8 +676,8 @@ void** beam_ops;
     if (is_not_list(Src)) { Fail; }                     \
     A(Need, Alive)
 
-#define IsNonemptyListTestHeap(Src, Need, Alive, Fail)  \
-    if (is_not_list(Src)) { Fail; }                     \
+#define IsNonemptyListTestHeap(Need, Alive, Fail)  \
+    if (is_not_list(x(0))) { Fail; }			\
     TestHeap(Need, Alive)
 
 #define IsTuple(X, Action) if (is_not_tuple(X)) Action
@@ -1316,17 +1316,13 @@ void process_main(void)
 	    Uint live;
 	    Eterm result;
 
-	OpCase(i_increment_yIId):
-	    increment_reg_val = yb(Arg(0));
-	    goto do_increment;
-
 	OpCase(i_increment_xIId):
 	    increment_reg_val = xb(Arg(0));
 	    goto do_increment;
 
-	OpCase(i_increment_rIId):
-	    increment_reg_val = r(0);
-	    I--;
+	OpCase(i_increment_yIId):
+	    increment_reg_val = yb(Arg(0));
+	    goto do_increment;
 
 	do_increment:
 	    increment_val = Arg(1);
@@ -1335,7 +1331,6 @@ void process_main(void)
 		ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
 		if (MY_IS_SSMALL(i)) {
 		    result = make_small(i);
-		store_result:
 		    StoreBifResult(3, result);
 		}
 	    }
@@ -1348,7 +1343,7 @@ void process_main(void)
 	    SWAPIN;
 	    ERTS_HOLE_CHECK(c_p);
 	    if (is_value(result)) {
-		goto store_result;
+		StoreBifResult(3, result);
 	    }
 	    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
 	    goto find_func_info;
@@ -1464,47 +1459,37 @@ void process_main(void)
     {
 	Eterm is_eq_exact_lit_val;
 
-    OpCase(i_is_eq_exact_literal_xfc):
-	is_eq_exact_lit_val = xb(Arg(0));
-	I++;
+    OpCase(i_is_eq_exact_literal_fxc):
+	is_eq_exact_lit_val = xb(Arg(1));
 	goto do_is_eq_exact_literal;
 
-    OpCase(i_is_eq_exact_literal_yfc):
-	is_eq_exact_lit_val = yb(Arg(0));
-	I++;
+    OpCase(i_is_eq_exact_literal_fyc):
+	is_eq_exact_lit_val = yb(Arg(1));
 	goto do_is_eq_exact_literal;
 
-    OpCase(i_is_eq_exact_literal_rfc):
-	is_eq_exact_lit_val = r(0);
-
     do_is_eq_exact_literal:
-	if (!eq(Arg(1), is_eq_exact_lit_val)) {
+	if (!eq(Arg(2), is_eq_exact_lit_val)) {
 	    ClauseFail();
 	}
-	Next(2);
+	Next(3);
     }
 
     {
 	Eterm is_ne_exact_lit_val;
 
-    OpCase(i_is_ne_exact_literal_xfc):
-	is_ne_exact_lit_val = xb(Arg(0));
-	I++;
+    OpCase(i_is_ne_exact_literal_fxc):
+	is_ne_exact_lit_val = xb(Arg(1));
 	goto do_is_ne_exact_literal;
 
-    OpCase(i_is_ne_exact_literal_yfc):
-	is_ne_exact_lit_val = yb(Arg(0));
-	I++;
+    OpCase(i_is_ne_exact_literal_fyc):
+	is_ne_exact_lit_val = yb(Arg(1));
 	goto do_is_ne_exact_literal;
 
-    OpCase(i_is_ne_exact_literal_rfc):
-	is_ne_exact_lit_val = r(0);
-
     do_is_ne_exact_literal:
-	if (eq(Arg(1), is_ne_exact_lit_val)) {
+	if (eq(Arg(2), is_ne_exact_lit_val)) {
 	    ClauseFail();
 	}
-	Next(2);
+	Next(3);
     }
 
  OpCase(move_window3_xxxy): {
@@ -1553,7 +1538,7 @@ void process_main(void)
      NextPF(6, next);
  }
 
- OpCase(i_move_call_only_fcr): {
+ OpCase(i_move_call_only_fc): {
      r(0) = Arg(1);
  }
  /* FALL THROUGH */
@@ -1563,7 +1548,7 @@ void process_main(void)
      Dispatch();
  }
 
- OpCase(i_move_call_last_fPcr): {
+ OpCase(i_move_call_last_fPc): {
      r(0) = Arg(2);
  }
  /* FALL THROUGH */
@@ -1575,7 +1560,7 @@ void process_main(void)
      Dispatch();
  }
 
- OpCase(i_move_call_crf): {
+ OpCase(i_move_call_cf): {
      r(0) = Arg(0);
      I++;
  }
@@ -1587,7 +1572,7 @@ void process_main(void)
      Dispatch();
  }
 
- OpCase(i_move_call_ext_last_ePcr): {
+ OpCase(i_move_call_ext_last_ePc): {
      r(0) = Arg(2);
  }
  /* FALL THROUGH */
@@ -1603,7 +1588,7 @@ void process_main(void)
     DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
     Dispatchx();
 
- OpCase(i_move_call_ext_cre): {
+ OpCase(i_move_call_ext_ce): {
      r(0) = Arg(0);
      I++;
  }
@@ -1613,7 +1598,7 @@ void process_main(void)
     DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
     Dispatchx();
 
- OpCase(i_move_call_ext_only_ecr): {
+ OpCase(i_move_call_ext_only_ec): {
      r(0) = Arg(1);
  }
  /* FALL THROUGH */
@@ -1707,29 +1692,23 @@ void process_main(void)
 	Eterm element_index;
 	Eterm element_tuple;
 
-    OpCase(i_element_xjsd):
-	element_tuple = xb(Arg(0));
-	I++;
+    OpCase(i_element_jxsd):
+	element_tuple = xb(Arg(1));
 	goto do_element;
 
-    OpCase(i_element_yjsd):
-	element_tuple = yb(Arg(0));
-	I++;
+    OpCase(i_element_jysd):
+	element_tuple = yb(Arg(1));
 	goto do_element;
 
-    OpCase(i_element_rjsd):
-	element_tuple = r(0);
-	/* Fall through */
-
     do_element:
-	GetArg1(1, element_index);
+	GetArg1(2, element_index);
 	if (is_small(element_index) && is_tuple(element_tuple)) {
 	    Eterm* tp = tuple_val(element_tuple);
 
 	    if ((signed_val(element_index) >= 1) &&
 		(signed_val(element_index) <= arityval(*tp))) {
 		Eterm result = tp[signed_val(element_index)];
-		StoreBifResult(2, result);
+		StoreBifResult(3, result);
 	    }
 	}
     }
@@ -1743,29 +1722,24 @@ void process_main(void)
     {
 	Eterm fast_element_tuple;
 
-    OpCase(i_fast_element_rjId):
-	fast_element_tuple = r(0);
+    OpCase(i_fast_element_jxId):
+     fast_element_tuple = xb(Arg(1));
+     goto do_fast_element;
+
+    OpCase(i_fast_element_jyId):
+     fast_element_tuple = yb(Arg(1));
+     goto do_fast_element;
 
     do_fast_element:
 	if (is_tuple(fast_element_tuple)) {
 	    Eterm* tp = tuple_val(fast_element_tuple);
-	    Eterm pos = Arg(1);	/* Untagged integer >= 1 */
+	    Eterm pos = Arg(2);	/* Untagged integer >= 1 */
 	    if (pos <= arityval(*tp)) {
 		Eterm result = tp[pos];
-		StoreBifResult(2, result);
+		StoreBifResult(3, result);
 	    }
 	}
      goto badarg;
-
-    OpCase(i_fast_element_xjId):
-     fast_element_tuple = xb(Arg(0));
-     I++;
-     goto do_fast_element;
-
-    OpCase(i_fast_element_yjId):
-     fast_element_tuple = yb(Arg(0));
-     I++;
-     goto do_fast_element;
  }
 
  OpCase(catch_yf):
@@ -1871,7 +1845,7 @@ void process_main(void)
      * Pick up the next message and place it in x(0).
      * If no message, jump to a wait or wait_timeout instruction.
      */
- OpCase(i_loop_rec_fr):
+ OpCase(i_loop_rec_f):
  {
      BeamInstr *next;
      ErlMessage* msgp;
@@ -2183,10 +2157,6 @@ void process_main(void)
      select_val2 = xb(Arg(0));
      goto do_select_tuple_arity2;
 
- OpCase(i_select_tuple_arity2_rfAAff):
-     select_val2 = r(0);
-     I--;
-
  do_select_tuple_arity2:
      if (is_not_tuple(select_val2)) {
 	 goto select_val2_fail;
@@ -2202,10 +2172,6 @@ void process_main(void)
      select_val2 = xb(Arg(0));
      goto do_select_val2;
 
- OpCase(i_select_val2_rfccff):
-     select_val2 = r(0);
-     I--;
-
  do_select_val2:
      if (select_val2 == Arg(2)) {
 	 I += 3;
@@ -2229,10 +2195,6 @@ void process_main(void)
      select_val = yb(Arg(0));
      goto do_select_tuple_arity;
 
- OpCase(i_select_tuple_arity_rfI):
-     select_val = r(0);
-     I--;
-
  do_select_tuple_arity:
      if (is_tuple(select_val)) {
 	 select_val = *tuple_val(select_val);
@@ -2249,10 +2211,6 @@ void process_main(void)
      select_val = yb(Arg(0));
      goto do_linear_search;
 
- OpCase(i_select_val_lins_rfI):
-     select_val = r(0);
-     I--;
-
  do_linear_search: {
      BeamInstr *vs = &Arg(3);
      int ix = 0;
@@ -2279,10 +2237,6 @@ void process_main(void)
      select_val = yb(Arg(0));
      goto do_binary_search;
      
- OpCase(i_select_val_bins_rfI):
-     select_val = r(0);
-     I--;
-
  do_binary_search:
  {
      struct Pairs {
@@ -2343,10 +2297,6 @@ void process_main(void)
      jump_on_val_zero_index = xb(Arg(0));
      goto do_jump_on_val_zero_index;
 
- OpCase(i_jump_on_val_zero_rfI):
-     jump_on_val_zero_index = r(0);
-     I--;
-
  do_jump_on_val_zero_index:
      if (is_small(jump_on_val_zero_index)) {
 	 jump_on_val_zero_index = signed_val(jump_on_val_zero_index);
@@ -2371,10 +2321,6 @@ void process_main(void)
      jump_on_val_index = xb(Arg(0));
      goto do_jump_on_val_index;
 
- OpCase(i_jump_on_val_rfII):
-     jump_on_val_index = r(0);
-     I--;
-
  do_jump_on_val_index:
      if (is_small(jump_on_val_index)) {
 	 jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3));
@@ -3441,18 +3387,8 @@ do {						\
     {
 	Eterm badmatch_val;
 
-    OpCase(badmatch_y):
-	badmatch_val = yb(Arg(0));
-	goto do_badmatch;
-
     OpCase(badmatch_x):
 	badmatch_val = xb(Arg(0));
-	goto do_badmatch;
-
-    OpCase(badmatch_r):
-	badmatch_val = r(0);
-
-    do_badmatch:
 	c_p->fvalue = badmatch_val;
 	c_p->freason = BADMATCH;
     }
@@ -3627,16 +3563,6 @@ do {						\
 
     OpCase(case_end_x):
 	case_end_val = xb(Arg(0));
-	goto do_case_end;
-
-    OpCase(case_end_y):
-	case_end_val = yb(Arg(0));
-	goto do_case_end;
-
-    OpCase(case_end_r):
-	case_end_val = r(0);
-
-    do_case_end:
 	c_p->fvalue = case_end_val;
 	c_p->freason = EXC_CASE_CLAUSE;
 	goto find_func_info;
@@ -3692,11 +3618,6 @@ do {						\
 	 goto do_bs_init_bits;
      }
 
-     OpCase(i_bs_init_bits_fail_rjId): {
-	 num_bits_term = r(0);
-	 alloc = 0;
-	 goto do_bs_init_bits;
-     }
      OpCase(i_bs_init_bits_fail_yjId): {
 	 num_bits_term = yb(Arg(0));
 	 I++;
@@ -3823,12 +3744,6 @@ do {						\
 	 goto do_bs_init;
      }
 
-     OpCase(i_bs_init_fail_rjId): {
-	 tmp_arg1 = r(0);
-	 tmp_arg2 = 0;
-	 goto do_bs_init;
-     }
-
      OpCase(i_bs_init_fail_yjId): {
 	 tmp_arg1 = yb(Arg(0));
 	 tmp_arg2 = 0;
@@ -4202,9 +4117,6 @@ do {						\
      Uint slots;
      Eterm context;
 
-     OpCase(i_bs_start_match2_rfIId): {
-	 context = r(0);
-
      do_start_match:
 	 slots = Arg(2);
 	 if (!is_boxed(context)) {
@@ -4251,7 +4163,7 @@ do {						\
 	     ClauseFail();
 	 }
 	 NextPF(4, next);
-     }
+
      OpCase(i_bs_start_match2_xfIId): {
 	 context = xb(Arg(0));
 	 I++;
@@ -4264,18 +4176,6 @@ do {						\
      }
  }
 
- OpCase(bs_test_zero_tail2_fr): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     
-     PreFetch(1, next);
-     _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0));
-     if (_mb->size != _mb->offset) {
-	 ClauseFail();
-     }
-     NextPF(1, next);
- }
-
  OpCase(bs_test_zero_tail2_fx): {
      BeamInstr *next;
      ErlBinMatchBuffer *_mb;
@@ -4288,16 +4188,6 @@ do {						\
      NextPF(2, next);
  }
 
- OpCase(bs_test_tail_imm2_frI): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     PreFetch(2, next);
-     _mb = ms_matchbuffer(r(0));
-     if (_mb->size - _mb->offset != Arg(1)) {
-	 ClauseFail();
-     }
-     NextPF(2, next);
- }
  OpCase(bs_test_tail_imm2_fxI): {
      BeamInstr *next;
      ErlBinMatchBuffer *_mb;
@@ -4309,16 +4199,6 @@ do {						\
      NextPF(3, next);
  }
 
- OpCase(bs_test_unit_frI): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     PreFetch(2, next);
-     _mb = ms_matchbuffer(r(0));
-     if ((_mb->size - _mb->offset) % Arg(1)) {
-	 ClauseFail();
-     }
-     NextPF(2, next);
- }
  OpCase(bs_test_unit_fxI): {
      BeamInstr *next;
      ErlBinMatchBuffer *_mb;
@@ -4330,16 +4210,6 @@ do {						\
      NextPF(3, next);
  }
 
- OpCase(bs_test_unit8_fr): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     PreFetch(1, next);
-     _mb = ms_matchbuffer(r(0));
-     if ((_mb->size - _mb->offset) & 7) {
-	 ClauseFail();
-     }
-     NextPF(1, next);
- }
  OpCase(bs_test_unit8_fx): {
      BeamInstr *next;
      ErlBinMatchBuffer *_mb;
@@ -4354,19 +4224,11 @@ do {						\
  {
      Eterm bs_get_integer8_context;
 
- OpCase(i_bs_get_integer_8_rfd): {
-	 bs_get_integer8_context = r(0);
-	 goto do_bs_get_integer_8;
-     }
-
  OpCase(i_bs_get_integer_8_xfd): {
-	 bs_get_integer8_context = xb(Arg(0));
-	 I++;
-     }
-
- do_bs_get_integer_8: {
 	 ErlBinMatchBuffer *_mb;
 	 Eterm _result;
+	 bs_get_integer8_context = xb(Arg(0));
+	 I++;
 	 _mb = ms_matchbuffer(bs_get_integer8_context);
 	 if (_mb->size - _mb->offset < 8) {
 	     ClauseFail();
@@ -4384,15 +4246,10 @@ do {						\
  {
      Eterm bs_get_integer_16_context;
 
- OpCase(i_bs_get_integer_16_rfd):
-     bs_get_integer_16_context = r(0);
-     goto do_bs_get_integer_16;
-
  OpCase(i_bs_get_integer_16_xfd):
      bs_get_integer_16_context = xb(Arg(0));
      I++;
 
- do_bs_get_integer_16:
      {
 	 ErlBinMatchBuffer *_mb;
 	 Eterm _result;
@@ -4413,17 +4270,10 @@ do {						\
  {
      Eterm bs_get_integer_32_context;
 
- OpCase(i_bs_get_integer_32_rfId):
-     bs_get_integer_32_context = r(0);
-     goto do_bs_get_integer_32;
-
-     
  OpCase(i_bs_get_integer_32_xfId):
      bs_get_integer_32_context = xb(Arg(0));
      I++;
 
-
- do_bs_get_integer_32:
      {
 	 ErlBinMatchBuffer *_mb;
 	 Uint32 _integer;
@@ -4452,13 +4302,6 @@ do {						\
      }
  }
 
- /* Operands: Size Live Fail Flags Dst */
- OpCase(i_bs_get_integer_imm_rIIfId): {
-     tmp_arg1 = r(0);
-     /* Operands: Size Live Fail Flags Dst */
-     goto do_bs_get_integer_imm_test_heap;
- }
-
  /* Operands: x(Reg) Size Live Fail Flags Dst */
  OpCase(i_bs_get_integer_imm_xIIfId): {
      tmp_arg1 = xb(Arg(0));
@@ -4481,15 +4324,6 @@ do {						\
      goto do_bs_get_integer_imm;
  }
 
- /* Operands: Size Fail Flags Dst */
- OpCase(i_bs_get_integer_small_imm_rIfId): {
-     tmp_arg1 = r(0);
-     tmp_arg2 = Arg(0);
-     I++;
-     /* Operands: Fail Flags Dst */
-     goto do_bs_get_integer_imm;
- }
-
  /* Operands: x(Reg) Size Fail Flags Dst */
  OpCase(i_bs_get_integer_small_imm_xIfId): {
      tmp_arg1 = xb(Arg(0));
@@ -4563,11 +4397,6 @@ do {						\
      Eterm get_utf8_context;
 
      /* Operands: MatchContext Fail Dst */
- OpCase(i_bs_get_utf8_rfd): {
-	 get_utf8_context = r(0);
-	 goto do_bs_get_utf8;
-     }
-
  OpCase(i_bs_get_utf8_xfd): {
 	 get_utf8_context = xb(Arg(0));
 	 I++;
@@ -4578,7 +4407,7 @@ do {						\
       * Operands: Fail Dst
       */
 
- do_bs_get_utf8: {
+     {
 	 Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context));
 	 if (is_non_value(result)) {
 	     ClauseFail();
@@ -4591,12 +4420,7 @@ do {						\
      Eterm get_utf16_context;
 
      /* Operands: MatchContext Fail Flags Dst */
- OpCase(i_bs_get_utf16_rfId): {
-	 get_utf16_context = r(0);
-	 goto do_bs_get_utf16;
-     }
-
- OpCase(i_bs_get_utf16_xfId): {
+     OpCase(i_bs_get_utf16_xfId): {
 	 get_utf16_context = xb(Arg(0));
 	 I++;
      }
@@ -4605,7 +4429,7 @@ do {						\
       * get_utf16_context = match_context
       * Operands: Fail Flags Dst
       */
- do_bs_get_utf16: {
+     {
 	 Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context),
 					  Arg(1));
 	 if (is_non_value(result)) {
@@ -4624,26 +4448,10 @@ do {						\
      Uint orig;
      Uint hole_size;
 
-     OpCase(bs_context_to_binary_r): {
-	 context_to_binary_context = r(0);
-	 I -= 2;
-	 goto do_context_to_binary;
-     }
-
-     /* Unfortunately, inlining can generate this instruction. */
-     OpCase(bs_context_to_binary_y): {
-	 context_to_binary_context = yb(Arg(0));
-	 goto do_context_to_binary0;
-     }
-
-     OpCase(bs_context_to_binary_x): {
+     OpCase(bs_context_to_binary_x):
 	 context_to_binary_context = xb(Arg(0));
-     
-     do_context_to_binary0:
 	 I--;
-     }
 
- do_context_to_binary:
      if (is_boxed(context_to_binary_context) &&
 	 header_is_bin_matchstate(*boxed_val(context_to_binary_context))) {
 	 ErlBinMatchState* ms;
@@ -4655,17 +4463,11 @@ do {						\
      }
      Next(2);
 
-     OpCase(i_bs_get_binary_all_reuse_rfI): {
-	 context_to_binary_context = r(0);
-	 goto do_bs_get_binary_all_reuse;
-     }
-
      OpCase(i_bs_get_binary_all_reuse_xfI): {
 	 context_to_binary_context = xb(Arg(0));
 	 I++;
      }
 
- do_bs_get_binary_all_reuse:
      mb = ms_matchbuffer(context_to_binary_context);
      size = mb->size - mb->offset;
      if (size % Arg(1) != 0) {
@@ -4693,16 +4495,11 @@ do {						\
  {
      Eterm match_string_context;
 
-     OpCase(i_bs_match_string_rfII): {
-	 match_string_context = r(0);
-	 goto do_bs_match_string;
-     }
      OpCase(i_bs_match_string_xfII): {
 	 match_string_context = xb(Arg(0));
 	 I++;
      }
 
- do_bs_match_string:
      {
 	 BeamInstr *next;
 	 byte* bytes;
@@ -4730,14 +4527,6 @@ do {						\
      }
  }
 
- OpCase(i_bs_save2_rI): {
-     BeamInstr *next;
-     ErlBinMatchState *_ms;
-     PreFetch(1, next);
-     _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
-     _ms->save_offset[Arg(0)] = _ms->mb.offset;
-     NextPF(1, next);
- }
  OpCase(i_bs_save2_xI): {
      BeamInstr *next;
      ErlBinMatchState *_ms;
@@ -4747,14 +4536,6 @@ do {						\
      NextPF(2, next);
  }
 
- OpCase(i_bs_restore2_rI): {
-     BeamInstr *next;
-     ErlBinMatchState *_ms;
-     PreFetch(1, next);
-     _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
-     _ms->mb.offset = _ms->save_offset[Arg(0)];
-     NextPF(1, next);
- }
  OpCase(i_bs_restore2_xI): {
      BeamInstr *next;
      ErlBinMatchState *_ms;
@@ -5030,7 +4811,9 @@ do {						\
      switch( c_p->def_arg_reg[3] ) {
        case HIPE_MODE_SWITCH_RES_RETURN:
 	 ASSERT(is_value(reg[0]));
-	 MoveReturn(reg[0], r(0));
+	 SET_I(c_p->cp);
+	 c_p->cp = 0;
+	 Goto(*I);
        case HIPE_MODE_SWITCH_RES_CALL_EXPORTED:
 	 c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()];
 	 /*fall through*/
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3cd80c4ec2..3671baa552 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1847,9 +1847,7 @@ load_code(LoaderState* stp)
 	    case TAG_o:
 		break;
 	    case TAG_x:
-		if (last_op->a[arg].val == 0) {
-		    last_op->a[arg].type = TAG_r;
-		} else if (last_op->a[arg].val >= MAX_REG) {
+		if (last_op->a[arg].val >= MAX_REG) {
 		    LoadError1(stp, "invalid x register number: %u",
 			       last_op->a[arg].val);
 		}
@@ -2055,7 +2053,42 @@ load_code(LoaderState* stp)
 		if (((opc[specific].mask[0] & mask[0]) == mask[0]) &&
 		    ((opc[specific].mask[1] & mask[1]) == mask[1]) &&
 		    ((opc[specific].mask[2] & mask[2]) == mask[2])) {
-		    break;
+
+		    if (!opc[specific].involves_r) {
+			break;	/* No complications - match */
+		    }
+
+		    /*
+		     * The specific operation uses the 'r' operand,
+		     * which is shorthand for x(0). Now things
+		     * get complicated. First we must check whether
+		     * all operands that should be of type 'r' use
+		     * x(0) (as opposed to some other X register).
+		     */
+		    for (arg = 0; arg < arity; arg++) {
+			if (opc[specific].involves_r & (1 << arg) &&
+			    tmp_op->a[arg].type == TAG_x) {
+			    if (tmp_op->a[arg].val != 0) {
+				break; /* Other X register than 0 */
+			    }
+			}
+		    }
+
+		    if (arg == arity) {
+			/*
+			 * All 'r' operands use x(0) in the generic
+			 * operation. That means a match. Now we
+			 * will need to rewrite the generic instruction
+			 * to actually use 'r' instead of 'x(0)'.
+			 */
+			for (arg = 0; arg < arity; arg++) {
+			    if (opc[specific].involves_r & (1 << arg) &&
+				tmp_op->a[arg].type == TAG_x) {
+				tmp_op->a[arg].type = TAG_r;
+			    }
+			}
+			break;	/* Match */
+		    }
 		}
 		specific++;
 	    }
@@ -2166,9 +2199,6 @@ load_code(LoaderState* stp)
 		break;
 	    case 's':	/* Any source (tagged constant or register) */
 		switch (tag) {
-		case TAG_r:
-		    code[ci++] = make_loader_x_reg(0);
-		    break;
 		case TAG_x:
 		    code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
 		    break;
@@ -2196,9 +2226,6 @@ load_code(LoaderState* stp)
 		break;
 	    case 'd':	/* Destination (x(0), x(N), y(N) */
 		switch (tag) {
-		case TAG_r:
-		    code[ci++] = make_loader_x_reg(0);
-		    break;
 		case TAG_x:
 		    code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
 		    break;
@@ -2360,7 +2387,6 @@ load_code(LoaderState* stp)
 		stp->labels[tmp_op->a[arg].val].patches = ci;
 		ci++;
 		break;
-	    case TAG_r:
 	    case TAG_x:
 		CodeNeed(1);
 		code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
@@ -2455,7 +2481,6 @@ load_code(LoaderState* stp)
 	    stp->on_load = ci;
 	    break;
 	case op_bs_put_string_II:
-	case op_i_bs_match_string_rfII:
 	case op_i_bs_match_string_xfII:
 	    new_string_patch(stp, ci-1);
 	    break;
@@ -2693,17 +2718,17 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
     op->next = NULL;
 
     if (Index.type == TAG_i && Index.val > 0 &&
-	(Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) {
+	(Tuple.type == TAG_x || Tuple.type == TAG_y)) {
 	op->op = genop_i_fast_element_4;
-	op->a[0] = Tuple;
-	op->a[1] = Fail;
+	op->a[0] = Fail;
+	op->a[1] = Tuple;
 	op->a[2].type = TAG_u;
 	op->a[2].val = Index.val;
 	op->a[3] = Dst;
     } else {
 	op->op = genop_i_element_4;
-	op->a[0] = Tuple;
-	op->a[1] = Fail;
+	op->a[0] = Fail;
+	op->a[1] = Tuple;
 	op->a[2] = Index;
 	op->a[3] = Dst;
     }
@@ -4798,7 +4823,8 @@ transform_engine(LoaderState* st)
 	    if (var[i].type != instr->a[ap].type)
 		goto restart;
 	    switch (var[i].type) {
-	    case TAG_r: case TAG_n: break;
+	    case TAG_n:
+		break;
 	    default:
 		if (var[i].val != instr->a[ap].val)
 		    goto restart;
@@ -5865,7 +5891,7 @@ make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamIns
     fp[4] = arity;
 #ifdef HIPE
     if (native) {
-	fp[5] = BeamOpCode(op_move_return_nr);
+	fp[5] = BeamOpCode(op_move_return_n);
 	hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5);
     }
 #endif
@@ -6301,7 +6327,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 #ifdef HIPE
 	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
 #else
-	op = (Eterm) BeamOpCode(op_move_return_nr);
+	op = (Eterm) BeamOpCode(op_move_return_n);
 #endif
 	fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op);
     }
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index cfc978b5a5..98f27a1725 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -141,6 +141,7 @@
 typedef struct op_entry {
    char* name;			/* Name of instruction. */
    Uint32 mask[3];		/* Signature mask. */
+   unsigned involves_r;		/* Needs special attention when matching. */
    int sz;			/* Number of loaded words. */
    char* pack;			/* Instructions for packing engine. */
    char* sign;			/* Signature string. */
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4fa6a087e2..335f8e2db5 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -39,8 +39,8 @@ too_old_compiler | never() =>
 # necessary.) Since the instructions don't work correctly in R12B, simply
 # refuse to load the module.
 
-func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler
-func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler
+func_info M=a a==am_module_info A=u==0 | label L | move n x==0 => too_old_compiler
+func_info M=a a==am_module_info A=u==1 | label L | move n x==0 => too_old_compiler
 
 # The undocumented and unsupported guard BIF is_constant/1 was removed
 # in R13. The is_constant/2 operation is marked as obsolete in genop.tab,
@@ -77,14 +77,14 @@ return
 # there is no line/1 instruction between the move and the call.
 #
 
-move S r | line Loc | call_ext Ar Func => \
-     line Loc | move S r | call_ext Ar Func
-move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \
-     line Loc | move S r | call_ext_last Ar Func D
-move S r | line Loc | call_ext_only Ar Func=u$is_bif => \
-     line Loc | move S r | call_ext_only Ar Func
-move S r | line Loc | call Ar Func => \
-     line Loc | move S r | call Ar Func
+move S X0=x==0 | line Loc | call_ext Ar Func => \
+     line Loc | move S X0 | call_ext Ar Func
+move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_bif D => \
+     line Loc | move S X0 | call_ext_last Ar Func D
+move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_bif => \
+     line Loc | move S X0 | call_ext_only Ar Func
+move S X0=x==0 | line Loc | call Ar Func => \
+     line Loc | move S X0 | call Ar Func
 
 #
 # A tail-recursive call to an external function (non-BIF) will
@@ -167,31 +167,24 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
 select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-i_select_val_bins r f I
 i_select_val_bins x f I
 i_select_val_bins y f I
 
-i_select_val_lins r f I
 i_select_val_lins x f I
 i_select_val_lins y f I
 
-i_select_val2 r f c c f f
 i_select_val2 x f c c f f
 i_select_val2 y f c c f f
 
-i_select_tuple_arity r f I
 i_select_tuple_arity x f I
 i_select_tuple_arity y f I
 
-i_select_tuple_arity2 r f A A f f
 i_select_tuple_arity2 x f A A f f
 i_select_tuple_arity2 y f A A f f
 
-i_jump_on_val_zero r f I
 i_jump_on_val_zero x f I
 i_jump_on_val_zero y f I
 
-i_jump_on_val r f I I
 i_jump_on_val x f I I
 i_jump_on_val y f I I
 
@@ -203,30 +196,13 @@ is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \
 %macro: get_list GetList -pack
 get_list x x x
 get_list x x y
-get_list x x r
 get_list x y x
 get_list x y y
-get_list x y r
-get_list x r x
-get_list x r y
 
 get_list y x x
 get_list y x y
-get_list y x r
 get_list y y x
 get_list y y y
-get_list y y r
-get_list y r x
-get_list y r y
-
-get_list r x x
-get_list r x y
-get_list r x r
-get_list r y x
-get_list r y y
-get_list r y r
-get_list r r x
-get_list r r y
 
 # Old-style catch.
 catch y f
@@ -247,21 +223,15 @@ set_tuple_element s d P
 
 %macro: i_get_tuple_element GetTupleElement -pack
 i_get_tuple_element x P x
-i_get_tuple_element r P x
 i_get_tuple_element y P x
-i_get_tuple_element x P r
-i_get_tuple_element y P r
 
 %cold
-i_get_tuple_element r P r
 i_get_tuple_element x P y
-i_get_tuple_element r P y
 i_get_tuple_element y P y
 %hot
 
 %macro: is_number IsNumber -fail_action
 %cold
-is_number f r
 is_number f x
 is_number f y
 %hot
@@ -271,16 +241,12 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
 
 jump f
 
-case_end Literal=c => move Literal x | case_end x
-badmatch Literal=c => move Literal x | badmatch x
+case_end NotInX=cy => move NotInX x | case_end x
+badmatch NotInX=cy => move NotInX x | badmatch x
 
-case_end r
 case_end x
-case_end y
 
-badmatch r
 badmatch x
-badmatch y
 
 if_end
 raise s s
@@ -289,7 +255,7 @@ raise s s
 badarg j
 system_limit j
 
-move C=cxy r | jump Lbl => move_jump Lbl C
+move C=cxy x==0 | jump Lbl => move_jump Lbl C
 
 %macro: move_jump MoveJump -nonext
 move_jump f n
@@ -307,8 +273,6 @@ move_window/6
 
 # x -> y
 
-move S1=r S2=y | move X1=x Y1=y => move2 S1 S2 X1 Y1
-
 move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \
     move_window X1 X2 X3 Y1 Y3
 
@@ -329,12 +293,13 @@ move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
 move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
 
-move S1=x S2=r | move S3=x S4=x => move2 S1 S2 S3 S4
-move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
-move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1
+move X1=x X2=x | move X3=x Y1=y => move2 X1 X2 X3 Y1
+
+move S1=x S2=x | move X1=x Y1=y => move2 S1 S2 X1 Y1
+move S1=y S2=x | move X1=x Y1=y => move2 S1 S2 X1 Y1
 
-move Y1=y X1=x | move S1=r D1=x => move2 Y1 X1 S1 D1
-move S1=r D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1
+move Y1=y X1=x | move S1=x D1=x => move2 Y1 X1 S1 D1
+move S1=x D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1
 
 move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3
 move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3
@@ -351,14 +316,13 @@ move2 x y x y
 move2 y x y x
 move2 x x x x
 
-move2 x r x x
+move2 x x x y
 
-move2 x r x y
-move2 r y x y
-move2 y r x y
+move2 x y x y
+move2 y x x y
 
-move2 r x y x
-move2 y x r x
+move2 x x y x
+move2 y x x x
 
 %macro: move3 Move3
 move3 x y x y x y
@@ -373,20 +337,14 @@ move S=c D=y => move S x | move x D
 %macro:move Move -pack -gen_dest
 move x x
 move x y
-move x r
 move y x
-move y r
-move r x
-move r y
-move c r
 move c x
 move n x
-move n r
 move y y
 
 # Receive operations.
 
-loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src
+loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail
 
 label L | wait_timeout Fail Src | smp_already_locked(L) => label L | i_wait_timeout_locked Fail Src
 wait_timeout Fail Src => i_wait_timeout Fail Src
@@ -401,7 +359,7 @@ label L | timeout | smp_already_locked(L) => label L | timeout_locked
 remove_message
 timeout
 timeout_locked
-i_loop_rec f r
+i_loop_rec f
 loop_rec_end f
 wait f
 wait_locked f
@@ -419,29 +377,25 @@ send
 # Optimized comparisons with one immediate/literal operand.
 #
 
-is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C
-is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C
+is_eq_exact Lbl R=xy C=ian => i_is_eq_exact_immed Lbl R C
+is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C
 
-is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C
-is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C
+is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C
+is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C
 
 %macro: i_is_eq_exact_immed EqualImmed -fail_action
-i_is_eq_exact_immed f r c
 i_is_eq_exact_immed f x c
 i_is_eq_exact_immed f y c
 
-i_is_eq_exact_literal r f c
-i_is_eq_exact_literal x f c
-i_is_eq_exact_literal y f c
+i_is_eq_exact_literal f x c
+i_is_eq_exact_literal f y c
 
 %macro: i_is_ne_exact_immed NotEqualImmed -fail_action
-i_is_ne_exact_immed f r c
 i_is_ne_exact_immed f x c
 i_is_ne_exact_immed f y c
 
-i_is_ne_exact_literal r f c
-i_is_ne_exact_literal x f c
-i_is_ne_exact_literal y f c
+i_is_ne_exact_literal f x c
+i_is_ne_exact_literal f y c
 
 #
 # Common Compare Specializations
@@ -449,31 +403,21 @@ i_is_ne_exact_literal y f c
 # to keep the instruction set small-ish
 #
 
-is_eq_exact Lbl S1=xy S2=r => is_eq_exact Lbl S2 S1
-is_eq_exact Lbl S1=rx S2=xy => i_is_eq_exact_spec Lbl S1 S2
+is_eq_exact Lbl S1=y S2=x => is_eq_exact Lbl S2 S1
+is_eq_exact Lbl S1=x S2=xy => i_is_eq_exact_spec Lbl S1 S2
 %macro: i_is_eq_exact_spec EqualExact -fail_action
 
 i_is_eq_exact_spec f x x
 i_is_eq_exact_spec f x y
-i_is_eq_exact_spec f r x
-i_is_eq_exact_spec f r y
-%cold
-i_is_eq_exact_spec f r r
-%hot
 
-is_lt Lbl S1=rxc S2=rxc  => i_is_lt_spec Lbl S1 S2
+is_lt Lbl S1=xc S2=xc  => i_is_lt_spec Lbl S1 S2
 
 %macro: i_is_lt_spec IsLessThan -fail_action
 
 i_is_lt_spec f x x
-i_is_lt_spec f x r
 i_is_lt_spec f x c
-i_is_lt_spec f r x
-i_is_lt_spec f r c
 i_is_lt_spec f c x
-i_is_lt_spec f c r
 %cold
-i_is_lt_spec f r r
 i_is_lt_spec f c c
 %hot
 
@@ -523,7 +467,6 @@ i_put_tuple Dst Arity Puts=* | put S => \
 i_put_tuple/2
 
 %macro:i_put_tuple PutTuple -pack -goto:do_put_tuple
-i_put_tuple r I
 i_put_tuple x I
 i_put_tuple y I
 
@@ -540,48 +483,25 @@ put_list x n x
 put_list y n x
 put_list x x x
 put_list y x x
-put_list x x r
-put_list y r r
 
 put_list y y x
 put_list x y x
-put_list r x x
-put_list r y x
-put_list r x r
-put_list y y r
-put_list y r x
-put_list r n x
-
-put_list x r x
-put_list x y r
-put_list y x r
-put_list y x x
 
-put_list x r r
+put_list y x x
 
 # put_list SrcReg Constant Dst
-put_list r c r
-put_list r c x
-put_list r c y
 
-put_list x c r
 put_list x c x
 put_list x c y
 
-put_list y c r
 put_list y c x
 put_list y c y
 
 # put_list Constant SrcReg Dst
-put_list c r r
-put_list c r x
-put_list c r y
 
-put_list c x r
 put_list c x x
 put_list c x y
 
-put_list c y r
 put_list c y x
 put_list c y y
 
@@ -590,18 +510,12 @@ put_list s s d
 %hot
 
 %macro: i_fetch FetchArgs -pack
-i_fetch c r
 i_fetch c x
 i_fetch c y
-i_fetch r c
-i_fetch r x
-i_fetch r y
 i_fetch x c
-i_fetch x r
 i_fetch x x
 i_fetch x y
 i_fetch y c
-i_fetch y r
 i_fetch y x
 i_fetch y y
 
@@ -629,27 +543,27 @@ return_trace
 # Note: There is no 'move_return y r', since there never are any y registers
 # when we do move_return (if we have y registers, we must do move_deallocate_return).
 
-move S r | return => move_return S r
+move S x==0 | return => move_return S
 
 %macro: move_return MoveReturn -nonext
-move_return x r
-move_return c r
-move_return n r
+move_return x
+move_return c
+move_return n
 
-move S r | deallocate D | return => move_deallocate_return S r D
+move S x==0 | deallocate D | return => move_deallocate_return S D
 
 %macro: move_deallocate_return MoveDeallocateReturn -pack -nonext
-move_deallocate_return x r Q
-move_deallocate_return y r Q
-move_deallocate_return c r Q
-move_deallocate_return n r Q
+move_deallocate_return x Q
+move_deallocate_return y Q
+move_deallocate_return c Q
+move_deallocate_return n Q
 
 deallocate D | return => deallocate_return D
 
 %macro: deallocate_return DeallocateReturn -nonext
 deallocate_return Q
 
-test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y
+test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y
 
 %macro: test_heap_1_put_list TestHeapPutList -pack
 test_heap_1_put_list I y
@@ -658,18 +572,16 @@ test_heap_1_put_list I y
 
 is_tuple Fail Literal=q => move Literal x | is_tuple Fail x
 is_tuple Fail=f c => jump Fail
-is_tuple Fail=f S=rxy | test_arity Fail=f S=rxy Arity => is_tuple_of_arity Fail S Arity
+is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity
 
 %macro:is_tuple_of_arity IsTupleOfArity -fail_action
 
 is_tuple_of_arity f x A
 is_tuple_of_arity f y A
-is_tuple_of_arity f r A
 
 %macro: is_tuple IsTuple -fail_action
 is_tuple f x
 is_tuple f y
-is_tuple f r
 
 test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity
 test_arity Fail=f c Arity => jump Fail
@@ -677,7 +589,6 @@ test_arity Fail=f c Arity => jump Fail
 %macro: test_arity IsArity -fail_action
 test_arity f x A
 test_arity f y A
-test_arity f r A
 
 is_tuple_of_arity Fail=f Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \
   is_tuple_of_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P
@@ -726,46 +637,40 @@ is_integer Fail=f i =>
 is_integer Fail=f an => jump Fail
 is_integer Fail Literal=q => move Literal x | is_integer Fail x
 
-is_integer Fail=f S=rx | allocate Need Regs => is_integer_allocate Fail S Need Regs
+is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs
 
 %macro: is_integer_allocate IsIntegerAllocate -fail_action
 is_integer_allocate f x I I
-is_integer_allocate f r I I
 
 %macro: is_integer IsInteger -fail_action
 is_integer f x
 is_integer f y
-is_integer f r
 
 is_list Fail=f n =>
 is_list Fail Literal=q => move Literal x | is_list Fail x
 is_list Fail=f c => jump Fail
 %macro: is_list IsList -fail_action
-is_list f r
 is_list f x
 %cold
 is_list f y
 %hot
 
-is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
+is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
 
 %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
 is_nonempty_list_allocate f x I t
-is_nonempty_list_allocate f r I t
 
-is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2
+is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2
 
 %macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack
-is_non_empty_list_test_heap f r I t
+is_non_empty_list_test_heap f I t
 
 %macro: is_nonempty_list IsNonemptyList -fail_action
 is_nonempty_list f x
 is_nonempty_list f y
-is_nonempty_list f r
 
 %macro: is_atom IsAtom -fail_action
 is_atom f x
-is_atom f r
 %cold
 is_atom f y
 %hot
@@ -773,7 +678,6 @@ is_atom Fail=f a =>
 is_atom Fail=f niq => jump Fail
 
 %macro: is_float IsFloat -fail_action
-is_float f r
 is_float f x
 %cold
 is_float f y
@@ -787,12 +691,10 @@ is_nil Fail=f qia => jump Fail
 %macro: is_nil IsNil -fail_action
 is_nil f x
 is_nil f y
-is_nil f r
 
 is_binary Fail Literal=q => move Literal x | is_binary Fail x
 is_binary Fail=f c => jump Fail
 %macro: is_binary IsBinary -fail_action
-is_binary f r
 is_binary f x
 %cold
 is_binary f y
@@ -804,7 +706,6 @@ is_bitstr Fail Term => is_bitstring Fail Term
 is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x
 is_bitstring Fail=f c => jump Fail
 %macro: is_bitstring IsBitstring -fail_action
-is_bitstring f r
 is_bitstring f x
 %cold
 is_bitstring f y
@@ -812,7 +713,6 @@ is_bitstring f y
 
 is_reference Fail=f cq => jump Fail
 %macro: is_reference IsRef -fail_action
-is_reference f r
 is_reference f x
 %cold
 is_reference f y
@@ -820,7 +720,6 @@ is_reference f y
 
 is_pid Fail=f cq => jump Fail
 %macro: is_pid IsPid -fail_action
-is_pid f r
 is_pid f x
 %cold
 is_pid f y
@@ -828,7 +727,6 @@ is_pid f y
 
 is_port Fail=f cq => jump Fail
 %macro: is_port IsPort -fail_action
-is_port f r
 is_port f x
 %cold
 is_port f y
@@ -840,7 +738,6 @@ is_boolean Fail=f ac => jump Fail
 
 %cold
 %macro: is_boolean IsBoolean -fail_action
-is_boolean f r
 is_boolean f x
 is_boolean f y
 %hot
@@ -986,76 +883,76 @@ call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate
 %unless USE_VM_PROBES
 
 call_ext Arity u$func:erlang:dt_get_tag/0 => \
-    move a=am_undefined r
+    move a=am_undefined x=0
 call_ext_last Arity u$func:erlang:dt_get_tag/0 D => \
-    move a=am_undefined r | deallocate D | return
+    move a=am_undefined x=0 | deallocate D | return
 call_ext_only Arity u$func:erlang:dt_get_tag/0 => \
-    move a=am_undefined r | return
-
-move Any r | call_ext Arity u$func:erlang:dt_put_tag/1 => \
-    move a=am_undefined r
-move Any r | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \
-    move a=am_undefined r | deallocate D | return
-move Any r | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \
-    move a=am_undefined r | return
+    move a=am_undefined x=0 | return
+
+move Any x==0 | call_ext Arity u$func:erlang:dt_put_tag/1 => \
+    move a=am_undefined x=0
+move Any x==0 | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \
+    move a=am_undefined x=0 | deallocate D | return
+move Any x==0 | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \
+    move a=am_undefined x=0 | return
 call_ext Arity u$func:erlang:dt_put_tag/1 => \
-    move a=am_undefined r
+    move a=am_undefined x=0
 call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \
-    move a=am_undefined r | deallocate D | return
+    move a=am_undefined x=0 | deallocate D | return
 call_ext_only Arity u$func:erlang:dt_put_tag/1 => \
-    move a=am_undefined r | return
+    move a=am_undefined x=0 | return
 
 call_ext Arity u$func:erlang:dt_get_tag_data/0 => \
-    move a=am_undefined r
+    move a=am_undefined x=0
 call_ext_last Arity u$func:erlang:dt_get_tag_data/0 D => \
-    move a=am_undefined r | deallocate D | return
+    move a=am_undefined x=0 | deallocate D | return
 call_ext_only Arity u$func:erlang:dt_get_tag_data/0 => \
-    move a=am_undefined r | return
-
-move Any r | call_ext Arity u$func:erlang:dt_spread_tag/1 => \
-    move a=am_true r
-move Any r | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \
-    move a=am_true r | deallocate D | return
-move Any r | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \
-    move a=am_true r | return
+    move a=am_undefined x=0 | return
+
+move Any x==0 | call_ext Arity u$func:erlang:dt_spread_tag/1 => \
+    move a=am_true x=0
+move Any x==0 | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \
+    move a=am_true x=0 | deallocate D | return
+move Any x==0 | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \
+    move a=am_true x=0 | return
 call_ext Arity u$func:erlang:dt_spread_tag/1 => \
-    move a=am_true r
+    move a=am_true x=0
 call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \
-    move a=am_true r | deallocate D | return
+    move a=am_true x=0 | deallocate D | return
 call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \
-    move a=am_true r | return
-
-move Any r | call_ext Arity u$func:erlang:dt_restore_tag/1 => \
-    move a=am_true r
-move Any r | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \
-    move a=am_true r | deallocate D | return
-move Any r | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \
-    move a=am_true r | return
+    move a=am_true x=0 | return
+
+move Any x==0 | call_ext Arity u$func:erlang:dt_restore_tag/1 => \
+    move a=am_true x=0
+move Any x==0 | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \
+    move a=am_true x=0 | deallocate D | return
+move Any x==0 | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \
+    move a=am_true x=0 | return
 call_ext Arity u$func:erlang:dt_restore_tag/1 => \
-    move a=am_true r
+    move a=am_true x=0
 call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \
-    move a=am_true r | deallocate D | return
+    move a=am_true x=0 | deallocate D | return
 call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \
-    move a=am_true r | return
-
-move Any r | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
-    move Any r
-move Any r | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \
-    move Any r | deallocate D | return
-move Any r | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
-    move Any r | return
+    move a=am_true x=0 | return
+
+move Any x==0 | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
+    move Any x=0
+move Any x==0 | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \
+    move Any x=0 | deallocate D | return
+move Any x==0 | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
+    move Any x=0 | return
 call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 =>
 call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \
     deallocate D | return
 call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
     return
 
-move Any r | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \
-    move Any r
-move Any r | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \
-    move Any r | deallocate D | return
-move Any r | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \
-    move Any r | return
+move Any x==0 | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \
+    move Any x=0
+move Any x==0 | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \
+    move Any x=0 | deallocate D | return
+move Any x==0 | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \
+    move Any x=0 | return
 call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 =>
 call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \
     deallocate D | return
@@ -1063,7 +960,7 @@ call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \
     return
 
 # Can happen after one of the transformations above.
-move Discarded r | move Something r => move Something r
+move Discarded x==0 | move Something x==0 => move Something x=0
 
 %endif
 
@@ -1088,9 +985,9 @@ call_ext_only Ar=u Bif=u$is_bif => \
 # with call instructions.
 #
 
-move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func
-move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r
-move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r 
+move S=c x==0 | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S Func
+move S=c x==0 | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S
+move S=c x==0 | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S
 
 call_ext Ar Func        => i_call_ext Func
 call_ext_last Ar Func D => i_call_ext_last Func D
@@ -1117,7 +1014,7 @@ bif0 u$bif:erlang:node/0 Dst=d => node Dst
 
 bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst
 
-bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst)
+bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2, Dst)
 
 bif1 p Bif S1 Dst => bif1_body Bif S1 Dst
 
@@ -1127,24 +1024,20 @@ bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst
 i_get s d
 
 %macro: self Self
-self r
 self x
 self y
 
 %macro: node Node
-node r
 node x
 %cold
 node y
 %hot
 
-i_fast_element r j I d
-i_fast_element x j I d
-i_fast_element y j I d
+i_fast_element j x I d
+i_fast_element j y I d
 
-i_element r j s d
-i_element x j s d
-i_element y j s d
+i_element j x s d
+i_element j y s d
 
 bif1 f b s d
 bif1_body b s d
@@ -1155,37 +1048,37 @@ i_bif2_body b d
 # Internal calls.
 #
 
-move S=c r | call Ar P=f => i_move_call S r P
-move S=s r | call Ar P=f => move_call S r P
+move S=c x==0 | call Ar P=f => i_move_call S P
+move S=s x==0 | call Ar P=f => move_call S P
 
-i_move_call c r f
+i_move_call c f
 
 %macro:move_call MoveCall -arg_f -size -nonext
-move_call/3
+move_call/2
 
-move_call x r f
-move_call y r f
+move_call x f
+move_call y f
 
-move S=c r | call_last Ar P=f D => i_move_call_last P D S r
-move S r | call_last Ar P=f D => move_call_last S r P D
+move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S
+move S x==0 | call_last Ar P=f D => move_call_last S P D
 
-i_move_call_last f P c r
+i_move_call_last f P c
 
 %macro:move_call_last MoveCallLast -arg_f -nonext -pack
 
-move_call_last/4
-move_call_last x r f Q
-move_call_last y r f Q
+move_call_last/3
+move_call_last x f Q
+move_call_last y f Q
 
-move S=c r | call_only Ar P=f => i_move_call_only P S r
-move S=x r | call_only Ar P=f => move_call_only S r P
+move S=c x==0 | call_only Ar P=f => i_move_call_only P S
+move S=x x==0 | call_only Ar P=f => move_call_only S P
 
-i_move_call_only f c r
+i_move_call_only f c
 
 %macro:move_call_only MoveCallOnly -arg_f -nonext
-move_call_only/3
+move_call_only/2
 
-move_call_only x r f
+move_call_only x f
 
 call Ar Func        => i_call Func
 call_last Ar Func D => i_call_last Func D
@@ -1199,9 +1092,9 @@ i_call_ext e
 i_call_ext_last e P
 i_call_ext_only e
 
-i_move_call_ext c r e
-i_move_call_ext_last e P c r
-i_move_call_ext_only e c r
+i_move_call_ext c e
+i_move_call_ext_last e P c
+i_move_call_ext_only e c
 
 # Fun calls.
 
@@ -1221,7 +1114,6 @@ i_make_fun I t
 %macro: is_function IsFunction -fail_action
 is_function f x
 is_function f y
-is_function f r
 is_function Fail=f c => jump Fail
 
 func_info M F A => i_func_info u M F A
@@ -1233,126 +1125,102 @@ func_info M F A => i_func_info u M F A
 %cold
 bs_start_match2 Fail=f ica X Y D => jump Fail
 bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
-i_bs_start_match2 r f I I d
 i_bs_start_match2 x f I I d
 i_bs_start_match2 y f I I d
 
 bs_save2 Reg Index => gen_bs_save(Reg, Index)
-i_bs_save2 r I
 i_bs_save2 x I
 
 bs_restore2 Reg Index => gen_bs_restore(Reg, Index)
-i_bs_restore2 r I
 i_bs_restore2 x I
 
 # Matching integers
 bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val
 
-i_bs_match_string r f I I
 i_bs_match_string x f I I
 
 # Fetching integers from binaries.
-bs_get_integer2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \
+bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-i_bs_get_integer_small_imm r I f I d
 i_bs_get_integer_small_imm x I f I d
-i_bs_get_integer_imm r I I f I d
 i_bs_get_integer_imm x I I f I d
 i_bs_get_integer f I I d
-i_bs_get_integer_8 r f d
 i_bs_get_integer_8 x f d
-i_bs_get_integer_16 r f d
 i_bs_get_integer_16 x f d
-i_bs_get_integer_32 r f I d
 i_bs_get_integer_32 x f I d
 
 # Fetching binaries from binaries.
-bs_get_binary2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \
+bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
 %macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -gen_dest
 %macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -gen_dest
 %macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action -gen_dest
 
-i_bs_get_binary_imm2 f r I I I d
 i_bs_get_binary_imm2 f x I I I d
-i_bs_get_binary2 f r I s I d
 i_bs_get_binary2 f x I s I d
-i_bs_get_binary_all2 f r I I d
 i_bs_get_binary_all2 f x I I d
-i_bs_get_binary_all_reuse r f I
 i_bs_get_binary_all_reuse x f I
 
 # Fetching float from binaries.
-bs_get_float2 Fail=f Ms=rx Live=u Sz=s Unit=u Flags=u Dst=d => \
+bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
 		gen_get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-bs_get_float2 Fail=f Ms=rx Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
+bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
 
 %macro: i_bs_get_float2 BsGetFloat2 -fail_action -gen_dest
-i_bs_get_float2 f r I s I d
 i_bs_get_float2 f x I s I d
 
 # Miscellanous
 
-bs_skip_bits2 Fail=f Ms=rx Sz=s Unit=u Flags=u => \
-			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
-bs_skip_bits2 Fail=f Ms=rx Sz=q Unit=u Flags=u => \
+bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \
 			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
 
 %macro: i_bs_skip_bits_imm2 BsSkipBitsImm2 -fail_action
-i_bs_skip_bits_imm2 f r I
 i_bs_skip_bits_imm2 f x I
 
 %macro: i_bs_skip_bits2 BsSkipBits2 -fail_action
-i_bs_skip_bits2 f r x I
-i_bs_skip_bits2 f r y I
 i_bs_skip_bits2 f x x I
-i_bs_skip_bits2 f x r I
 i_bs_skip_bits2 f x y I
 
 %macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action
-i_bs_skip_bits_all2 f r I
 i_bs_skip_bits_all2 f x I 
 
-bs_test_tail2 Fail=f Ms=rx Bits=u==0 => bs_test_zero_tail2 Fail Ms
-bs_test_tail2 Fail=f Ms=rx Bits=u => bs_test_tail_imm2 Fail Ms Bits
-bs_test_zero_tail2 f r
+bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms
+bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits
 bs_test_zero_tail2 f x
-bs_test_tail_imm2 f r I
 bs_test_tail_imm2 f x I
 
 bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms
-bs_test_unit f r I
 bs_test_unit f x I
-bs_test_unit8 f r
 bs_test_unit8 f x
 
-bs_context_to_binary r
+# An y register operand for bs_context_to_binary is rare,
+# but can happen because of inlining.
+
+bs_context_to_binary Y=y => move Y x | bs_context_to_binary x
+
 bs_context_to_binary x
-bs_context_to_binary y
 
 #
 # Utf8/utf16/utf32 support. (R12B-5)
 #
-bs_get_utf8 Fail=f Ms=rx u u Dst=d => i_bs_get_utf8 Ms Fail Dst
-i_bs_get_utf8 r f d
+bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst
 i_bs_get_utf8 x f d
 
-bs_skip_utf8 Fail=f Ms=rx u u => i_bs_get_utf8 Ms Fail x
+bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x
 
-bs_get_utf16 Fail=f Ms=rx u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
-bs_skip_utf16 Fail=f Ms=rx u Flags=u => i_bs_get_utf16 Ms Fail Flags x
+bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
+bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x
 
-i_bs_get_utf16 r f I d
 i_bs_get_utf16 x f I d
 
-bs_get_utf32 Fail=f Ms=rx Live=u Flags=u Dst=d => \
+bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
 	i_fetch Dst Ms | \
 	i_bs_validate_unicode_retract Fail
-bs_skip_utf32 Fail=f Ms=rx Live=u Flags=u => \
+bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \
 	i_fetch x Ms | \
 	i_bs_validate_unicode_retract Fail
@@ -1379,9 +1247,8 @@ bs_init2 Fail Sz=u Words Regs Flags Dst => \
 bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
   i_bs_init_fail Sz Fail Regs Dst
 bs_init2 Fail Sz Words Regs Flags Dst => \
-  i_fetch Sz r | i_bs_init_fail_heap Words Fail Regs Dst
+  i_fetch Sz x=0 | i_bs_init_fail_heap Words Fail Regs Dst
 
-i_bs_init_fail r j I d
 i_bs_init_fail x j I d
 i_bs_init_fail y j I d
 
@@ -1402,9 +1269,8 @@ bs_init_bits Fail Sz=u Words Regs Flags Dst =>  i_bs_init_bits_heap Sz Words Reg
 bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
   i_bs_init_bits_fail Sz Fail Regs Dst
 bs_init_bits Fail Sz Words Regs Flags Dst => \
-  i_fetch Sz r | i_bs_init_bits_fail_heap Words Fail Regs Dst
+  i_fetch Sz x=0 | i_bs_init_bits_fail_heap Words Fail Regs Dst
 
-i_bs_init_bits_fail r j I d
 i_bs_init_bits_fail x j I d
 i_bs_init_bits_fail y j I d
 
@@ -1577,7 +1443,6 @@ is_map Fail Lit=q | literal_is_map(Lit) =>
 is_map Fail cq => jump Fail
 
 %macro: is_map IsMap -fail_action
-is_map f r
 is_map f x
 is_map f y
 
@@ -1588,35 +1453,25 @@ has_map_fields Fail Src Size Rest=* => \
 
 ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
 
-get_map_elements Fail Src=rxy Size=u==2 Rest=* => \
+get_map_elements Fail Src=xy Size=u==2 Rest=* => \
     gen_get_map_element(Fail, Src, Size, Rest)
 get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
    gen_get_map_elements(Fail, Src, Size, Rest)
 
 i_get_map_elements f s I
 
-i_get_map_element Fail Src=rxy Key=ry Dst => \
+i_get_map_element Fail Src=xy Key=y Dst => \
     move Key x | i_get_map_element Fail Src x Dst
 
 %macro: i_get_map_element_hash GetMapElementHash -fail_action
-i_get_map_element_hash f r c I r
-i_get_map_element_hash f x c I r
-i_get_map_element_hash f y c I r
-i_get_map_element_hash f r c I x
 i_get_map_element_hash f x c I x
 i_get_map_element_hash f y c I x
-i_get_map_element_hash f r c I y
 i_get_map_element_hash f x c I y
 i_get_map_element_hash f y c I y
 
 %macro: i_get_map_element GetMapElement -fail_action
-i_get_map_element f r x r
-i_get_map_element f x x r
-i_get_map_element f y x r
-i_get_map_element f r x x
 i_get_map_element f x x x
 i_get_map_element f y x x
-i_get_map_element f r x y
 i_get_map_element f x x y
 i_get_map_element f y x y
 
@@ -1625,9 +1480,9 @@ i_get_map_element f y x y
 # the i_increment/4 instruction (in bodies, not in guards).
 #
 
-gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \
+gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=x Dst => \
 	gen_increment(Reg, Int, Live, Dst)
-gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \
+gc_bif2 p Live u$bif:erlang:splus/2 Reg=x Int=i Dst => \
 	gen_increment(Reg, Int, Live, Dst)
 
 gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \
@@ -1662,7 +1517,6 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst
 gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst
 
-i_increment r I I d
 i_increment x I I d
 i_increment y I I d
 
-- 
cgit v1.2.3


From c76b297463feee133adad510128d312536150392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 16 Jun 2015 10:06:20 +0200
Subject: Eliminate the use of i_fetch in arithmetic instructions

The i_fetch instruction fetches two operands and places them
in the tmp_arg1 and tmp_arg2 variables. The next instruction
(such as i_plus) does not have to handle different types of
operands, but can get get them simply from the tmp_arg*
variables. Thus, i_fetch was introduced as a way to temper
a potentail combinatorial explosion.

Unfortunately, clang will generate terrible code because of
the tmp_arg1 and tmp_arg2 variables being live across multiple
instructions. Note that Clang has no way to predict the control
flow from one instruction to another. Clang must assume that
any instruction can jump to any other instruction. Somehow GCC
manages to cope with this situation much better.

Therefore, to improve the quality of the code generated by clang, we
must eliminate all uses of the tmp_arg1 and tmp_arg2 variables. This
commit eliminates the use of i_fetch in combination with the
arithmetic and logical instructions.

While we are touching the code for the bsr and bsl instructions,
also move the tmp_big[] array from top scope of process main into
the block that encloses the bsr and bsl instructions.
---
 erts/emulator/beam/beam_emu.c | 319 ++++++++++++++++++------------------------
 erts/emulator/beam/ops.tab    |  93 +++++++-----
 2 files changed, 200 insertions(+), 212 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index e5179830de..82cc030878 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1145,7 +1145,6 @@ void process_main(void)
      */
     register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
     register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
-    Eterm tmp_big[2];
 
     /*
      * X registers and floating point registers are located in
@@ -1158,8 +1157,6 @@ void process_main(void)
      */
     int neg_o_reds = 0;
 
-    Eterm (*arith_func)(Process* p, Eterm* reg, Uint live);
-
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
     static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES };
 #else
@@ -1307,9 +1304,6 @@ void process_main(void)
 #endif
 #include "beam_hot.h"
 
-#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
-#define ARITH_FUNC(name) erts_gc_##name
-
 	{
 	    Eterm increment_reg_val;
 	    Eterm increment_val;
@@ -1349,81 +1343,71 @@ void process_main(void)
 	    goto find_func_info;
 	}
 
-#define DO_BIG_ARITH(Func,Arg1,Arg2)     \
-    do {                                 \
-        Uint live = Arg(1);              \
-        SWAPOUT;                         \
-        reg[live] = (Arg1);              \
-        reg[live+1] = (Arg2);            \
-        result = (Func)(c_p, reg, live); \
-        SWAPIN;                          \
-        ERTS_HOLE_CHECK(c_p);            \
-        if (is_value(result)) {          \
-            StoreBifResult(4,result);    \
-        }                                \
-        goto lb_Cl_error;                \
-    } while(0)
+#define DO_OUTLINED_ARITH_2(name, Op1, Op2)	\
+ do {						\
+     Eterm result;				\
+     Uint live = Arg(1);			\
+						\
+     SWAPOUT;					\
+     reg[live] = Op1;				\
+     reg[live+1] = Op2;				\
+     result = erts_gc_##name(c_p, reg, live);	\
+     SWAPIN;					\
+     ERTS_HOLE_CHECK(c_p);			\
+     if (is_value(result)) {			\
+	 StoreBifResult(4, result);		\
+     }						\
+     goto lb_Cl_error;				\
+ } while (0)
 
- OpCase(i_plus_jIxxd):
  {
+     Eterm PlusOp1, PlusOp2;
      Eterm result;
 
-     if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
-	 Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3)));
-	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-	 if (MY_IS_SSMALL(i)) {
-	     result = make_small(i);
-             StoreBifResult(4, result);
-	 }
-     }
-     DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3)));
- }
+ OpCase(i_plus_jIxxd):
+     PlusOp1 = xb(Arg(2));
+     PlusOp2 = xb(Arg(3));
+     goto do_plus;
 
- OpCase(i_plus_jId):
- {
-     Eterm result;
+ OpCase(i_plus_jIssd):
+     GetArg2(2, PlusOp1, PlusOp2);
+     goto do_plus;
 
-     if (is_both_small(tmp_arg1, tmp_arg2)) {
-	 Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2);
+ do_plus:
+     if (is_both_small(PlusOp1, PlusOp2)) {
+	 Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
 	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
 	 if (MY_IS_SSMALL(i)) {
 	     result = make_small(i);
-	     STORE_ARITH_RESULT(result);
+             StoreBifResult(4, result);
 	 }
      }
-     arith_func = ARITH_FUNC(mixed_plus);
-     goto do_big_arith2;
+     DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2);
  }
 
- OpCase(i_minus_jIxxd):
  {
+     Eterm MinusOp1, MinusOp2;
      Eterm result;
 
-     if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
-	 Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3)));
-	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-	 if (MY_IS_SSMALL(i)) {
-	     result = make_small(i);
-             StoreBifResult(4, result);
-	 }
-     }
-     DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3)));
- }
+ OpCase(i_minus_jIxxd):
+     MinusOp1 = xb(Arg(2));
+     MinusOp2 = xb(Arg(3));
+     goto do_minus;
 
- OpCase(i_minus_jId):
- {
-     Eterm result;
+ OpCase(i_minus_jIssd):
+     GetArg2(2, MinusOp1, MinusOp2);
+     goto do_minus;
 
-     if (is_both_small(tmp_arg1, tmp_arg2)) {
-	 Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2);
+ do_minus:
+     if (is_both_small(MinusOp1, MinusOp2)) {
+	 Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
 	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
 	 if (MY_IS_SSMALL(i)) {
 	     result = make_small(i);
-	     STORE_ARITH_RESULT(result);
+             StoreBifResult(4, result);
 	 }
      }
-     arith_func = ARITH_FUNC(mixed_minus);
-     goto do_big_arith2;
+     DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2);
  }
 
  OpCase(i_is_lt_f):
@@ -2767,109 +2751,81 @@ do {						\
   * Arithmetic operations.
   */
 
- OpCase(i_times_jId):
+ OpCase(i_times_jIssd):
  {
-     arith_func = ARITH_FUNC(mixed_times);
-     goto do_big_arith2;
+     Eterm Op1, Op2;
+     GetArg2(2, Op1, Op2);
+     DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2);
  }
 
- OpCase(i_m_div_jId):
+ OpCase(i_m_div_jIssd):
  {
-     arith_func = ARITH_FUNC(mixed_div);
-     goto do_big_arith2;
+     Eterm Op1, Op2;
+     GetArg2(2, Op1, Op2);
+     DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2);
  }
 
- OpCase(i_int_div_jId):
+ OpCase(i_int_div_jIssd):
  {
-     Eterm result;
+     Eterm Op1, Op2;
 
-     if (tmp_arg2 == SMALL_ZERO) {
+     GetArg2(2, Op1, Op2);
+     if (Op2 == SMALL_ZERO) {
 	 goto badarith;
-     } else if (is_both_small(tmp_arg1, tmp_arg2)) {
-	 Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2);
+     } else if (is_both_small(Op1, Op2)) {
+	 Sint ires = signed_val(Op1) / signed_val(Op2);
 	 if (MY_IS_SSMALL(ires)) {
-	     result = make_small(ires);
-	     STORE_ARITH_RESULT(result);
+	     Eterm result = make_small(ires);
+	     StoreBifResult(4, result);
 	 }
      }
-     arith_func = ARITH_FUNC(int_div);
-     goto do_big_arith2;
+     DO_OUTLINED_ARITH_2(int_div, Op1, Op2);
  }
 
- OpCase(i_rem_jIxxd):
  {
-     Eterm result;
+     Eterm RemOp1, RemOp2;
 
-     if (xb(Arg(3)) == SMALL_ZERO) {
-	 goto badarith;
-     } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
-	 result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3))));
+ OpCase(i_rem_jIxxd):
+     RemOp1 = xb(Arg(2));
+     RemOp2 = xb(Arg(3));
+     goto do_rem;
+
+ OpCase(i_rem_jIssd):
+     GetArg2(2, RemOp1, RemOp2);
+     goto do_rem;
+
+ do_rem:
+     if (RemOp2 == SMALL_ZERO) {
+         goto badarith;
+     } else if (is_both_small(RemOp1, RemOp2)) {
+         Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2));
          StoreBifResult(4, result);
+     } else {
+	 DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2);
      }
-     DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3)));
  }
 
- OpCase(i_rem_jId):
  {
-     Eterm result;
-
-     if (tmp_arg2 == SMALL_ZERO) {
-	 goto badarith;
-     } else if (is_both_small(tmp_arg1, tmp_arg2)) {
-	 result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2));
-	 STORE_ARITH_RESULT(result);
-     } else {
-	 arith_func = ARITH_FUNC(int_rem);
-	 goto do_big_arith2;
-     }
- }
+     Eterm BandOp1, BandOp2;
 
  OpCase(i_band_jIxcd):
- {
-     Eterm result;
+     BandOp1 = xb(Arg(2));
+     BandOp2 = Arg(3);
+     goto do_band;
+
+ OpCase(i_band_jIssd):
+     GetArg2(2, BandOp1, BandOp2);
+     goto do_band;
 
-     if (is_both_small(xb(Arg(2)), Arg(3))) {
+ do_band:
+     if (is_both_small(BandOp1, BandOp2)) {
          /*
           * No need to untag -- TAG & TAG == TAG.
           */
-         result = xb(Arg(2)) & Arg(3);
+         Eterm result = BandOp1 & BandOp2;
          StoreBifResult(4, result);
      }
-     DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3));
- }
-
- OpCase(i_band_jId):
- {
-     Eterm result;
-
-     if (is_both_small(tmp_arg1, tmp_arg2)) {
-	 /*
-	  * No need to untag -- TAG & TAG == TAG.
-	  */
-	 result = tmp_arg1 & tmp_arg2;
-	 STORE_ARITH_RESULT(result);
-     }
-     arith_func = ARITH_FUNC(band);
-     goto do_big_arith2;
- }
-
-#undef DO_BIG_ARITH
-
- do_big_arith2:
- {
-     Eterm result;
-     Uint live = Arg(1);
-
-     SWAPOUT;
-     reg[live] = tmp_arg1;
-     reg[live+1] = tmp_arg2;
-     result = arith_func(c_p, reg, live);
-     SWAPIN;
-     ERTS_HOLE_CHECK(c_p);
-     if (is_value(result)) {
-	 STORE_ARITH_RESULT(result);
-     }
-     goto lb_Cl_error;
+     DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2);
  }
 
  /*
@@ -2890,97 +2846,102 @@ do {						\
      goto find_func_info;
  }
 
- OpCase(i_bor_jId):
+ OpCase(i_bor_jIssd):
  {
-     Eterm result;
+     Eterm Op1, Op2;
 
-     if (is_both_small(tmp_arg1, tmp_arg2)) {
+     GetArg2(2, Op1, Op2);
+     if (is_both_small(Op1, Op2)) {
 	 /*
 	  * No need to untag -- TAG | TAG == TAG.
 	  */
-	 result = tmp_arg1 | tmp_arg2;
-	 STORE_ARITH_RESULT(result);
+	 Eterm result = Op1 | Op2;
+	 StoreBifResult(4, result);
      }
-     arith_func = ARITH_FUNC(bor);
-     goto do_big_arith2;
+     DO_OUTLINED_ARITH_2(bor, Op1, Op2);
  }
 
- OpCase(i_bxor_jId):
+ OpCase(i_bxor_jIssd):
  {
-     Eterm result;
+     Eterm Op1, Op2;
 
-     if (is_both_small(tmp_arg1, tmp_arg2)) {
+     GetArg2(2, Op1, Op2);
+     if (is_both_small(Op1, Op2)) {
 	 /*
 	  * We could extract the tag from one argument, but a tag extraction
 	  * could mean a shift.  Therefore, play it safe here.
 	  */
-	 result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2));
-	 STORE_ARITH_RESULT(result);
+	 Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2));
+	 StoreBifResult(4, result);
      }
-     arith_func = ARITH_FUNC(bxor);
-     goto do_big_arith2;
+     DO_OUTLINED_ARITH_2(bxor, Op1, Op2);
  }
 
  {
+     Eterm Op1, Op2;
      Sint i;
      Sint ires;
      Eterm* bigp;
+     Eterm tmp_big[2];
 
-     OpCase(i_bsr_jId):
-	 if (is_small(tmp_arg2)) {
-	     i = -signed_val(tmp_arg2);
-	     if (is_small(tmp_arg1)) {
+     OpCase(i_bsr_jIssd):
+         GetArg2(2, Op1, Op2);
+	 if (is_small(Op2)) {
+	     i = -signed_val(Op2);
+	     if (is_small(Op1)) {
 		 goto small_shift;
-	     } else if (is_big(tmp_arg1)) {
+	     } else if (is_big(Op1)) {
 		 if (i == 0) {
-		     StoreBifResult(2, tmp_arg1);
+		     StoreBifResult(4, Op1);
 		 }
 		 goto big_shift;
 	     }
-	 } else if (is_big(tmp_arg2)) {
+	 } else if (is_big(Op2)) {
 	     /*
 	      * N bsr NegativeBigNum == N bsl MAX_SMALL
 	      * N bsr PositiveBigNum == N bsl MIN_SMALL
 	      */
-	     tmp_arg2 = make_small(bignum_header_is_neg(*big_val(tmp_arg2)) ?
+	     Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ?
 				   MAX_SMALL : MIN_SMALL);
 	     goto do_bsl;
 	}
      goto badarith;
      
-     OpCase(i_bsl_jId):
+     OpCase(i_bsl_jIssd):
+         GetArg2(2, Op1, Op2);
+
  do_bsl:
-	 if (is_small(tmp_arg2)) {
-	     i = signed_val(tmp_arg2);
+	 if (is_small(Op2)) {
+	     i = signed_val(Op2);
 
-	     if (is_small(tmp_arg1)) {
+	     if (is_small(Op1)) {
 	     small_shift:
-		 ires = signed_val(tmp_arg1);
+		 ires = signed_val(Op1);
 	     
 		 if (i == 0 || ires == 0) {
-		     StoreBifResult(2, tmp_arg1);
+		     StoreBifResult(4, Op1);
 		 } else if (i < 0)  { /* Right shift */
 		     i = -i;
 		     if (i >= SMALL_BITS-1) {
-			 tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
+			 Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
 		     } else {
-			 tmp_arg1 = make_small(ires >> i);
+			 Op1 = make_small(ires >> i);
 		     }
-		     StoreBifResult(2, tmp_arg1);
+		     StoreBifResult(4, Op1);
 		 } else if (i < SMALL_BITS-1) { /* Left shift */
 		     if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) ||
 			 ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) {
-			 tmp_arg1 = make_small(ires << i);
-			 StoreBifResult(2, tmp_arg1);
+			 Op1 = make_small(ires << i);
+			 StoreBifResult(4, Op1);
 		     }
 		 }
-		 tmp_arg1 = small_to_big(ires, tmp_big);
+		 Op1 = small_to_big(ires, tmp_big);
 
 	     big_shift:
 		 if (i > 0) {	/* Left shift. */
-		     ires = big_size(tmp_arg1) + (i / D_EXP);
+		     ires = big_size(Op1) + (i / D_EXP);
 		 } else {	/* Right shift. */
-		     ires = big_size(tmp_arg1);
+		     ires = big_size(Op1);
 		     if (ires <= (-i / D_EXP))
 			 ires = 3; /* ??? */
 		     else
@@ -2998,14 +2959,14 @@ do {						\
 			 c_p->freason = SYSTEM_LIMIT;
 			 goto lb_Cl_error;
 		     }
-		     TestHeapPreserve(ires+1, Arg(1), tmp_arg1);
+		     TestHeapPreserve(ires+1, Arg(1), Op1);
 		     bigp = HTOP;
-		     tmp_arg1 = big_lshift(tmp_arg1, i, bigp);
-		     if (is_big(tmp_arg1)) {
+		     Op1 = big_lshift(Op1, i, bigp);
+		     if (is_big(Op1)) {
 			 HTOP += bignum_header_arity(*HTOP) + 1;
 		     }
 		     HEAP_SPACE_VERIFIED(0);
-		     if (is_nil(tmp_arg1)) {
+		     if (is_nil(Op1)) {
 			 /*
 			  * This result must have been only slight larger
 			  * than allowed since it wasn't caught by the
@@ -3015,25 +2976,25 @@ do {						\
 			 goto lb_Cl_error;
 		     }
 		     ERTS_HOLE_CHECK(c_p);
-		     StoreBifResult(2, tmp_arg1);
+		     StoreBifResult(4, Op1);
 		 }
-	     } else if (is_big(tmp_arg1)) {
+	     } else if (is_big(Op1)) {
 		 if (i == 0) {
-		     StoreBifResult(2, tmp_arg1);
+		     StoreBifResult(4, Op1);
 		 }
 		 goto big_shift;
 	     }
-	 } else if (is_big(tmp_arg2)) {
-	     if (bignum_header_is_neg(*big_val(tmp_arg2))) {
+	 } else if (is_big(Op2)) {
+	     if (bignum_header_is_neg(*big_val(Op2))) {
 		 /*
 		  * N bsl NegativeBigNum is either 0 or -1, depending on
 		  * the sign of N. Since we don't believe this case
 		  * is common, do the calculation with the minimum
 		  * amount of code.
 		  */
-		 tmp_arg2 = make_small(MIN_SMALL);
+		 Op2 = make_small(MIN_SMALL);
 		 goto do_bsl;
-	     } else if (is_small(tmp_arg1) || is_big(tmp_arg1)) {
+	     } else if (is_small(Op1) || is_big(Op1)) {
 		 /*
 		  * N bsl PositiveBigNum is too large to represent.
 		  */
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 335f8e2db5..e4a757ec8b 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1475,68 +1475,95 @@ i_get_map_element f y x x
 i_get_map_element f x x y
 i_get_map_element f y x y
 
+#
+# Convert the plus operations to a generic plus instruction.
+#
+gen_plus/5
+gen_minus/5
+
+gc_bif1 Fail Live u$bif:erlang:splus/1 Src Dst => \
+   gen_plus Fail Live Src i Dst
+gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \
+   gen_plus Fail Live S1 S2 Dst
+
+gc_bif1 Fail Live u$bif:erlang:sminus/1 Src Dst => \
+   gen_minus Fail Live i Src Dst
+gc_bif2 Fail Live u$bif:erlang:sminus/2 S1 S2 Dst => \
+   gen_minus Fail Live S1 S2 Dst
+
 #
 # Optimize addition and subtraction of small literals using
 # the i_increment/4 instruction (in bodies, not in guards).
 #
 
-gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=x Dst => \
+gen_plus p Live Int=i Reg=d Dst => \
 	gen_increment(Reg, Int, Live, Dst)
-gc_bif2 p Live u$bif:erlang:splus/2 Reg=x Int=i Dst => \
+gen_plus p Live Reg=d Int=i Dst => \
 	gen_increment(Reg, Int, Live, Dst)
 
-gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \
-	negation_is_small(Int) => \
+gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \
 	gen_increment_from_minus(Reg, Int, Live, Dst)
 
 #
 # GCing arithmetic instructions.
 #
 
-gc_bif2 Fail I u$bif:erlang:splus/2 S1=x S2=x Dst=d => i_plus Fail I S1 S2 Dst
-gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst
-gc_bif2 Fail I u$bif:erlang:sminus/2 S1=x S2=x Dst=d => i_minus Fail I S1 S2 Dst
-gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst
-gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst
-gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst
+gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst
 
-gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst
-gc_bif2 Fail I u$bif:erlang:rem/2 S1=x S2=x Dst=d => i_rem Fail I S1 S2 Dst
-gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst
+gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst
 
-gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst
-gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst
+gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \
+  i_times Fail Live S1 S2 Dst
 
-gc_bif2 Fail I u$bif:erlang:band/2 S1=x S2=c Dst=d => i_band Fail I S1 S2 Dst
-gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst
-gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst
-gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst
+gc_bif2 Fail Live u$bif:erlang:div/2 S1 S2 Dst => \
+  i_m_div Fail Live S1 S2 Dst
+gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \
+  i_int_div Fail Live S1 S2 Dst
 
-gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
+gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \
+  i_rem Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \
+  i_bsl Fail Live S1 S2 Dst
+gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \
+  i_bsr Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \
+  i_band Fail Live S1 S2 Dst
 
-gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst
-gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst
+gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \
+  i_bor Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \
+  i_bxor Fail Live S1 S2 Dst
+
+gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 
 i_increment x I I d
 i_increment y I I d
 
 i_plus j I x x d
-i_plus j I d
+i_plus j I s s d
+
 i_minus j I x x d
-i_minus j I d
-i_times j I d
-i_m_div j I d
-i_int_div j I d
+i_minus j I s s d
+
+i_times j I s s d
+
+i_m_div j I s s d
+i_int_div j I s s d
+
 i_rem j I x x d
-i_rem j I d
+i_rem j I s s d
 
-i_bsl j I d
-i_bsr j I d
+i_bsl j I s s d
+i_bsr j I s s d
 
 i_band j I x c d
-i_band j I d
-i_bor j I d
-i_bxor j I d
+i_band j I s s d
+
+i_bor j I s s d
+i_bxor j I s s d
 
 i_int_bnot j s I d
 
-- 
cgit v1.2.3


From 3c583d590589b64ab543102f2542bda0337ce2f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 Jun 2015 06:52:50 +0200
Subject: Eliminate the use of i_fetch for relational operators

---
 erts/emulator/beam/beam_emu.c | 39 ++-----------------------
 erts/emulator/beam/ops.tab    | 67 ++++++++++++++-----------------------------
 2 files changed, 25 insertions(+), 81 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 82cc030878..b63fa038d2 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -651,6 +651,9 @@ void** beam_ops;
 #define EqualImmed(X, Y, Action) if (X != Y) { Action; }
 #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
 #define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; }
+#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; }
+#define Equal(X, Y, Action) if (!CMP_EQ(X,Y)) { Action; }
+#define NotEqual(X, Y, Action) if (!CMP_NE(X,Y)) { Action; }
 #define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; }
 #define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; }
 
@@ -1410,36 +1413,6 @@ void process_main(void)
      DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2);
  }
 
- OpCase(i_is_lt_f):
-    if (CMP_GE(tmp_arg1, tmp_arg2)) {
-	ClauseFail();
-    }
-    Next(1);
-
- OpCase(i_is_ge_f):
-    if (CMP_LT(tmp_arg1, tmp_arg2)) {
-	ClauseFail();
-    }
-    Next(1);
-
- OpCase(i_is_eq_f):
-    if (CMP_NE(tmp_arg1, tmp_arg2)) {
-	ClauseFail();
-    }
-    Next(1);
-
- OpCase(i_is_ne_f):
-    if (CMP_EQ(tmp_arg1, tmp_arg2)) {
-	ClauseFail();
-    }
-    Next(1);
-
- OpCase(i_is_eq_exact_f):
-    if (!EQ(tmp_arg1, tmp_arg2)) {
-	ClauseFail();
-    }
-    Next(1);
-
     {
 	Eterm is_eq_exact_lit_val;
 
@@ -3276,12 +3249,6 @@ do {						\
      NextPF(3, next);
  }
 
- OpCase(i_is_ne_exact_f):
-    if (EQ(tmp_arg1, tmp_arg2)) {
-	ClauseFail();
-    }
-    Next(1);
-
  OpCase(normal_exit): {
      SWAPOUT;
      c_p->freason = EXC_NORMAL;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index e4a757ec8b..57fb1ea026 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -397,59 +397,36 @@ i_is_ne_exact_immed f y c
 i_is_ne_exact_literal f x c
 i_is_ne_exact_literal f y c
 
-#
-# Common Compare Specializations
-# We don't do all of them since we want
-# to keep the instruction set small-ish
-#
-
-is_eq_exact Lbl S1=y S2=x => is_eq_exact Lbl S2 S1
-is_eq_exact Lbl S1=x S2=xy => i_is_eq_exact_spec Lbl S1 S2
-%macro: i_is_eq_exact_spec EqualExact -fail_action
-
-i_is_eq_exact_spec f x x
-i_is_eq_exact_spec f x y
-
-is_lt Lbl S1=xc S2=xc  => i_is_lt_spec Lbl S1 S2
-
-%macro: i_is_lt_spec IsLessThan -fail_action
-
-i_is_lt_spec f x x
-i_is_lt_spec f x c
-i_is_lt_spec f c x
+is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y
+%macro: is_eq_exact EqualExact -fail_action
+is_eq_exact f x x
+is_eq_exact f x y
+is_eq_exact f s s
+
+%macro: is_lt IsLessThan -fail_action
+is_lt f x x
+is_lt f x c
+is_lt f c x
 %cold
-i_is_lt_spec f c c
+is_lt f s s
 %hot
 
-is_ge Lbl S1=xc S2=xc => i_is_ge_spec Lbl S1 S2
-
-%macro: i_is_ge_spec IsGreaterEqual -fail_action
-
-i_is_ge_spec f x x
-i_is_ge_spec f x c
-i_is_ge_spec f c x
+%macro: is_ge IsGreaterEqual -fail_action
+is_ge f x x
+is_ge f x c
+is_ge f c x
 %cold
-i_is_ge_spec f c c
+is_ge f s s
 %hot
 
-#
-# All other comparisons.
-#
-
-is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
-is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
+%macro: is_ne_exact NotEqualExact -fail_action
+is_ne_exact f s s
 
-is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl
-is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl
-is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl
-is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl
+%macro: is_eq Equal -fail_action
+is_eq f s s
 
-i_is_eq_exact f
-i_is_ne_exact f
-i_is_lt f
-i_is_ge f
-i_is_eq f
-i_is_ne f
+%macro: is_ne NotEqual -fail_action
+is_ne f s s
 
 #
 # Putting things.
-- 
cgit v1.2.3


From 30204739a047ab96d2b7d59ae461d4cbb2131509 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 Jun 2015 10:34:39 +0200
Subject: Eliminate the use of i_fetch for BIF instructions

---
 erts/emulator/beam/beam_emu.c  | 68 ++++++++++++++++++++++++------------------
 erts/emulator/beam/beam_load.c | 28 ++++++-----------
 erts/emulator/beam/ops.tab     | 21 +++++++------
 3 files changed, 58 insertions(+), 59 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b63fa038d2..d260c74255 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2513,12 +2513,10 @@ do {						\
     {
 	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
 	GcBifFunction bf;
-	Eterm arg;
 	Eterm result;
 	Uint live = (Uint) Arg(3);
 
-	GetArg1(2, arg);
-	reg[live] = arg;
+	GetArg1(2, x(live));
 	bf = (GcBifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
@@ -2538,12 +2536,12 @@ do {						\
 	    SET_I((BeamInstr *) Arg(0));
 	    Goto(*I);
 	}
-	reg[0] = arg;
+	x(0) = x(live);
 	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
 	goto post_error_handling;
     }
 
- OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1
+ OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1
 			    and i_gc_bif3 */
     {
 	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
@@ -2551,8 +2549,13 @@ do {						\
 	Eterm result;
 	Uint live = (Uint) Arg(2);
 
-	reg[live++] = tmp_arg1;
-	reg[live] = tmp_arg2;
+	GetArg2(3, x(live), x(live+1));
+	/*
+	 * XXX This calling convention does not make sense. 'live'
+	 * should point out the first argument, not the second
+	 * (i.e. 'live' should not be incremented below).
+	 */
+	live++;
 	bf = (GcBifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
@@ -2566,30 +2569,34 @@ do {						\
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
-	    StoreBifResult(3, result);
+	    StoreBifResult(5, result);
 	}
 	if (Arg(0) != 0) {
 	    SET_I((BeamInstr *) Arg(0));
 	    Goto(*I);
 	}
-	reg[0] = tmp_arg1;
-	reg[1] = tmp_arg2;
+	live--;
+	x(0) = x(live);
+	x(1) = x(live+1);
 	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
 	goto post_error_handling;
     }
 
- OpCase(i_gc_bif3_jIsId):
+ OpCase(i_gc_bif3_jIIssd):
     {
 	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
 	GcBifFunction bf;
-	Eterm arg;
 	Eterm result;
-	Uint live = (Uint) Arg(3);
+	Uint live = (Uint) Arg(2);
 
-	GetArg1(2, arg);
-	reg[live++] = arg;
-	reg[live++] = tmp_arg1;
-	reg[live] = tmp_arg2;
+	x(live) = x(SCRATCH_X_REG);
+	GetArg2(3, x(live+1), x(live+2));
+	/*
+	 * XXX This calling convention does not make sense. 'live'
+	 * should point out the first argument, not the third
+	 * (i.e. 'live' should not be incremented below).
+	 */
+	live += 2;
 	bf = (GcBifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
@@ -2603,15 +2610,16 @@ do {						\
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
-	    StoreBifResult(4, result);
+	    StoreBifResult(5, result);
 	}
 	if (Arg(0) != 0) {
 	    SET_I((BeamInstr *) Arg(0));
 	    Goto(*I);
 	}
-	reg[0] = arg;
-	reg[1] = tmp_arg1;
-	reg[2] = tmp_arg2;
+	live -= 2;
+	x(0) = x(live);
+	x(1) = x(live+1);
+	x(2) = x(live+2);
 	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
 	goto post_error_handling;
     }
@@ -2619,12 +2627,13 @@ do {						\
  /*
   * Guards bifs and, or, xor in guards.
   */
- OpCase(i_bif2_fbd):
+ OpCase(i_bif2_fbssd):
     {
-	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+	Eterm tmp_reg[2];
 	Eterm (*bf)(Process*, Eterm*);
 	Eterm result;
 
+	GetArg2(2, tmp_reg[0], tmp_reg[1]);
 	bf = (BifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2636,7 +2645,7 @@ do {						\
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
-	    StoreBifResult(2, result);
+	    StoreBifResult(4, result);
 	}
 	SET_I((BeamInstr *) Arg(0));
 	Goto(*I);
@@ -2645,12 +2654,13 @@ do {						\
  /*
   * Guards bifs and, or, xor, relational operators in body.
   */
- OpCase(i_bif2_body_bd):
+ OpCase(i_bif2_body_bssd):
     {
-	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+	Eterm tmp_reg[2];
 	Eterm (*bf)(Process*, Eterm*);
 	Eterm result;
 
+	GetArg2(1, tmp_reg[0], tmp_reg[1]);
 	bf = (BifFunction) Arg(0);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2661,10 +2671,10 @@ do {						\
 	ERTS_HOLE_CHECK(c_p);
 	if (is_value(result)) {
 	    ASSERT(!is_CP(result));
-	    StoreBifResult(1, result);
+	    StoreBifResult(3, result);
 	}
-	reg[0] = tmp_arg1;
-	reg[1] = tmp_arg2;
+	reg[0] = tmp_reg[0];
+	reg[1] = tmp_reg[1];
 	SWAPOUT;
 	I = handle_error(c_p, I, reg, bf);
 	goto post_error_handling;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3671baa552..c90576333a 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3921,9 +3921,7 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx)
 /*
  * Rewrite gc_bifs with one parameter (the common case). Utilized
  * in ops.tab to rewrite instructions calling bif's in guards
- * to use a garbage collecting implementation. The instructions
- * are sometimes once again rewritten to handle literals (putting the
- * parameter in the mostly unused r[0] before the instruction is executed).
+ * to use a garbage collecting implementation.
  */
 static GenOp*
 gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
@@ -3978,10 +3976,6 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 
 /*
  * This is used by the ops.tab rule that rewrites gc_bifs with two parameters.
- * The instruction returned is then again rewritten to an i_load instruction
- * followed by i_gc_bif2_jIId, to handle literals properly.
- * As opposed to the i_gc_bif1_jIsId, the instruction  i_gc_bif2_jIId is
- * always rewritten, regardless of if there actually are any literals.
  */
 static GenOp*
 gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
@@ -4008,23 +4002,19 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 	op->a[2].val = stp->import[Bif.val].arity;
 	return op;
     }
-    op->op = genop_ii_gc_bif2_6;
+    op->op = genop_i_gc_bif2_6;
     op->arity = 6;
     op->a[0] = Fail;
     op->a[1].type = TAG_u;
-    op->a[2] = S1;
-    op->a[3] = S2;
-    op->a[4] = Live;
+    op->a[2] = Live;
+    op->a[3] = S1;
+    op->a[4] = S2;
     op->a[5] = Dst;
     return op;
 }
 
 /*
  * This is used by the ops.tab rule that rewrites gc_bifs with three parameters.
- * The instruction returned is then again rewritten to a move instruction that
- * uses r[0] for temp storage, followed by an i_load instruction,
- * followed by i_gc_bif3_jIsId, to handle literals properly. Rewriting
- * always occur, as with the gc_bif2 counterpart.
  */
 static GenOp*
 gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
@@ -4055,10 +4045,10 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
     op->arity = 7;
     op->a[0] = Fail;
     op->a[1].type = TAG_u;
-    op->a[2] = S1;
-    op->a[3] = S2;
-    op->a[4] = S3;
-    op->a[5] = Live;
+    op->a[2] = Live;
+    op->a[3] = S1;
+    op->a[4] = S2;
+    op->a[5] = S3;
     op->a[6] = Dst;
     op->next = NULL;
     return op;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 57fb1ea026..492d4d240a 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -995,8 +995,8 @@ bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2,
 
 bif1 p Bif S1 Dst => bif1_body Bif S1 Dst
 
-bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst
-bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst
+bif2 p Bif S1 S2 Dst => i_bif2_body Bif S1 S2 Dst
+bif2 Fail Bif S1 S2 Dst => i_bif2 Fail Bif S1 S2 Dst
 
 i_get s d
 
@@ -1018,8 +1018,8 @@ i_element j y s d
 
 bif1 f b s d
 bif1_body b s d
-i_bif2 f b d
-i_bif2_body b d
+i_bif2 f b s s d
+i_bif2_body b s s d
 
 #
 # Internal calls.
@@ -1568,17 +1568,16 @@ gc_bif3 Fail I Bif S1 S2 S3 Dst => \
 
 i_gc_bif1 j I s I d
 
-ii_gc_bif2/6
-
-ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D
-
-i_gc_bif2 j I I d
+i_gc_bif2 j I I s s d
 
 ii_gc_bif3/7
 
-ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D
+# A specific instruction can only have 6 operands, so we must
+# pass one of the arguments in an x register.
+ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \
+  move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst
 
-i_gc_bif3 j I s I d
+i_gc_bif3 j I I s s d
 
 #
 # The following instruction is specially handled in beam_load.c
-- 
cgit v1.2.3


From bede3941be8629efa4d91755c085a91b1416d432 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 Jun 2015 15:31:36 +0200
Subject: Eliminate use of i_fetch for bit syntax instructions

---
 erts/emulator/beam/beam_emu.c  | 207 +++++++++++++++++++++--------------------
 erts/emulator/beam/beam_load.c |  27 ++----
 erts/emulator/beam/ops.tab     |  29 +++---
 3 files changed, 129 insertions(+), 134 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index d260c74255..26e28832cf 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3548,11 +3548,10 @@ do {						\
 	 goto do_bs_init_bits_known;
      }
 
-     OpCase(i_bs_init_bits_fail_heap_IjId): {
-	 /* tmp_arg1 was fetched by an i_fetch instruction */
-	 num_bits_term = tmp_arg1;
-	 alloc = Arg(0);
-	 I++;
+     OpCase(i_bs_init_bits_fail_heap_sIjId): {
+	 GetArg1(0, num_bits_term);
+	 alloc = Arg(1);
+	 I += 2;
 	 goto do_bs_init_bits;
      }
 
@@ -3675,46 +3674,48 @@ do {						\
  }
 
  {
-     OpCase(i_bs_init_fail_heap_IjId): {
-	 /* tmp_arg1 was fetched by an i_fetch instruction */
-	 tmp_arg2 = Arg(0);
-	 I++;
+     Eterm BsOp1, BsOp2;
+
+     OpCase(i_bs_init_fail_heap_sIjId): {
+	 GetArg1(0, BsOp1);
+	 BsOp2 = Arg(1);
+	 I += 2;
 	 goto do_bs_init;
      }
 
      OpCase(i_bs_init_fail_yjId): {
-	 tmp_arg1 = yb(Arg(0));
-	 tmp_arg2 = 0;
+	 BsOp1 = yb(Arg(0));
+	 BsOp2 = 0;
 	 I++;
 	 goto do_bs_init;
      }
 
      OpCase(i_bs_init_fail_xjId): {
-	 tmp_arg1 = xb(Arg(0));
-	 tmp_arg2 = 0;
+	 BsOp1 = xb(Arg(0));
+	 BsOp2 = 0;
 	 I++;
      }
 	 /* FALL THROUGH */
      do_bs_init:
-         if (is_small(tmp_arg1)) {
-	     Sint size = signed_val(tmp_arg1);
+         if (is_small(BsOp1)) {
+	     Sint size = signed_val(BsOp1);
 	     if (size < 0) {
 		 goto badarg;
 	     }
-	     tmp_arg1 = (Eterm) size;
+	     BsOp1 = (Eterm) size;
 	 } else {
 	     Uint bytes;
 
-	     if (!term_to_Uint(tmp_arg1, &bytes)) {
+	     if (!term_to_Uint(BsOp1, &bytes)) {
 		 c_p->freason = bytes;
 		 goto lb_Cl_error;
 	     }
 	     if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
 		 goto system_limit;
 	     }
-	     tmp_arg1 = (Eterm) bytes;
+	     BsOp1 = (Eterm) bytes;
 	 }
-	 if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) {
+	 if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) {
 	     goto do_heap_bin_alloc;
 	 } else {
 	     goto do_proc_bin_alloc;
@@ -3722,15 +3723,15 @@ do {						\
 
 
      OpCase(i_bs_init_heap_IIId): {
-	 tmp_arg1 = Arg(0);
-	 tmp_arg2 = Arg(1);
+	 BsOp1 = Arg(0);
+	 BsOp2 = Arg(1);
 	 I++;
 	 goto do_proc_bin_alloc;
      }
 
      OpCase(i_bs_init_IId): {
-	 tmp_arg1 = Arg(0);
-	 tmp_arg2 = 0;
+	 BsOp1 = Arg(0);
+	 BsOp2 = 0;
      }
      /* FALL THROUGH */
      do_proc_bin_alloc: {
@@ -3739,13 +3740,13 @@ do {						\
 
 	 erts_bin_offset = 0;
 	 erts_writable_bin = 0;
-	 TestBinVHeap(tmp_arg1 / sizeof(Eterm), 
-	 	      tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1));
+	 TestBinVHeap(BsOp1 / sizeof(Eterm),
+		      BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1));
 
 	 /*
 	  * Allocate the binary struct itself.
 	  */
-	 bptr = erts_bin_nrml_alloc(tmp_arg1);
+	 bptr = erts_bin_nrml_alloc(BsOp1);
 	 erts_refc_init(&bptr->refc, 1);
 	 erts_current_bin = (byte *) bptr->orig_bytes;
 
@@ -3755,28 +3756,28 @@ do {						\
 	 pb = (ProcBin *) HTOP;
 	 HTOP += PROC_BIN_SIZE;
 	 pb->thing_word = HEADER_PROC_BIN;
-	 pb->size = tmp_arg1;
+	 pb->size = BsOp1;
 	 pb->next = MSO(c_p).first;
 	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
 	 pb->val = bptr;
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
 	 
-	 OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));
+	 OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm));
 
 	 StoreBifResult(2, make_binary(pb));
      }
 
      OpCase(i_bs_init_heap_bin_heap_IIId): {
-	 tmp_arg1 = Arg(0);
-	 tmp_arg2 = Arg(1);
+	 BsOp1 = Arg(0);
+	 BsOp2 = Arg(1);
 	 I++;
 	 goto do_heap_bin_alloc;
      }
 
      OpCase(i_bs_init_heap_bin_IId): {
-	 tmp_arg1 = Arg(0);
-	 tmp_arg2 = 0;
+	 BsOp1 = Arg(0);
+	 BsOp2 = 0;
      }
      /* Fall through */
      do_heap_bin_alloc:
@@ -3784,33 +3785,36 @@ do {						\
 	     ErlHeapBin* hb;
 	     Uint bin_need;
 
-	     bin_need = heap_bin_size(tmp_arg1);
+	     bin_need = heap_bin_size(BsOp1);
 	     erts_bin_offset = 0;
 	     erts_writable_bin = 0;
-	     TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1));
+	     TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1));
 	     hb = (ErlHeapBin *) HTOP;
 	     HTOP += bin_need;
-	     hb->thing_word = header_heap_bin(tmp_arg1);
-	     hb->size = tmp_arg1;
+	     hb->thing_word = header_heap_bin(BsOp1);
+	     hb->size = BsOp1;
 	     erts_current_bin = (byte *) hb->data;
-	     tmp_arg1 = make_binary(hb);
-	     StoreBifResult(2, tmp_arg1);
+	     BsOp1 = make_binary(hb);
+	     StoreBifResult(2, BsOp1);
 	 }
  }
 
- OpCase(i_bs_add_jId): {
-     Uint Unit = Arg(1);
-     if (is_both_small(tmp_arg1, tmp_arg2)) {
-	 Sint Arg1 = signed_val(tmp_arg1);
-	 Sint Arg2 = signed_val(tmp_arg2);
+ OpCase(bs_add_jssId): {
+     Eterm Op1, Op2;
+     Uint Unit = Arg(3);
+
+     GetArg2(1, Op1, Op2);
+     if (is_both_small(Op1, Op2)) {
+	 Sint Arg1 = signed_val(Op1);
+	 Sint Arg2 = signed_val(Op2);
 
 	 if (Arg1 >= 0 && Arg2 >= 0) {
-	     BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1);
-	     tmp_arg1 += Arg1;
+	     BsSafeMul(Arg2, Unit, goto system_limit, Op1);
+	     Op1 += Arg1;
 
 	 store_bs_add_result:
-	     if (MY_IS_SSMALL((Sint) tmp_arg1)) {
-		 tmp_arg1 = make_small(tmp_arg1);
+	     if (MY_IS_SSMALL((Sint) Op1)) {
+		 Op1 = make_small(Op1);
 	     } else {
 		 /*
 		  * May generate a heap fragment, but in this
@@ -3822,10 +3826,10 @@ do {						\
 		  * references (such as the heap).
 		  */
 		 SWAPOUT;
-		 tmp_arg1 = erts_make_integer(tmp_arg1, c_p);
+		 Op1 = erts_make_integer(Op1, c_p);
 		 HTOP = HEAP_TOP(c_p);
 	     }
-	     StoreBifResult(2, tmp_arg1);
+	     StoreBifResult(4, Op1);
 	 }
 	 goto badarg;
      } else {
@@ -3848,16 +3852,16 @@ do {						\
 	  * an Uint, the reason is SYSTEM_LIMIT.
 	  */
 
-	 if (!term_to_Uint(tmp_arg1, &a)) {
+	 if (!term_to_Uint(Op1, &a)) {
 	     if (a == BADARG) {
 		 goto badarg;
 	     }
-	     if (!term_to_Uint(tmp_arg2, &b)) {
+	     if (!term_to_Uint(Op2, &b)) {
 		 c_p->freason = b;
 		 goto lb_Cl_error;
 	     }
 	     goto system_limit;
-	 } else if (!term_to_Uint(tmp_arg2, &b)) {
+	 } else if (!term_to_Uint(Op2, &b)) {
 	     c_p->freason = b;
 	     goto lb_Cl_error;
 	 }
@@ -3867,8 +3871,8 @@ do {						\
 	  */
 	 
 	 BsSafeMul(b, Unit, goto system_limit, c);
-	 tmp_arg1 = a + c;
-	 if (tmp_arg1 < a) {
+	 Op1 = a + c;
+	 if (Op1 < a) {
 	     /*
 	      * If the result is less than one of the
 	      * arguments, there must have been an overflow.
@@ -3890,46 +3894,43 @@ do {						\
     }
 
  /*
-  * tmp_arg1 = Number of bytes to build
-  * tmp_arg2 = Source binary
-  * Operands: Fail ExtraHeap Live Unit Dst
+  * x(SCRATCH_X_REG);
+  * Operands: Fail ExtraHeap Live Unit Size Dst
   */
 
- OpCase(i_bs_append_jIIId): {
+ OpCase(i_bs_append_jIIIsd): {
      Uint live = Arg(2);
      Uint res;
+     Eterm Size;
 
+     GetArg1(4, Size);
      SWAPOUT;
-     reg[live] = tmp_arg2;
-     res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3));
+     reg[live] = x(SCRATCH_X_REG);
+     res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3));
      SWAPIN;
      if (is_non_value(res)) {
 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
 	 goto lb_Cl_error;
      }
-     StoreBifResult(4, res);
+     StoreBifResult(5, res);
  }
 
  /*
-  * tmp_arg1 = Number of bytes to build
-  * tmp_arg2 = Source binary
-  * Operands: Fail Unit Dst
+  * Operands: Fail Size Src Unit Dst
   */
- OpCase(i_bs_private_append_jId): {
+ OpCase(i_bs_private_append_jIssd): {
      Eterm res;
+     Eterm Size, Src;
 
-     res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1));
+     GetArg2(2, Size, Src);
+     res = erts_bs_private_append(c_p, Src, Size, Arg(1));
      if (is_non_value(res)) {
 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
 	 goto lb_Cl_error;
      }
-     StoreBifResult(2, res);
+     StoreBifResult(4, res);
  }
 
- /*
-  * tmp_arg1 = Initial size of writable binary
-  * Operands: Live Dst
-  */
  OpCase(bs_init_writable): {
      SWAPOUT;
      r(0) = erts_bs_init_writable(c_p, r(0));
@@ -4024,26 +4025,29 @@ do {						\
 
  /*
   * Only used for validating a value matched out. 
-  *
-  * tmp_arg1 = Integer to validate
-  * tmp_arg2 = Match context
   */
- OpCase(i_bs_validate_unicode_retract_j): {
-     /*
-      * There is no need to untag the integer, but it IS necessary
-      * to make sure it is small (a bignum pointer could fall in
-      * the valid range).
-      */
-     if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) ||
-	 (make_small(0xD800UL) <= tmp_arg1 &&
-	  tmp_arg1 <= make_small(0xDFFFUL))) {
-	 ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2);
+ OpCase(i_bs_validate_unicode_retract_jss): {
+	Eterm i; 		/* Integer to validate */
 
-	 mb->offset -= 32;
-	 goto badarg;
-     }
-     Next(1);
- }
+	/*
+	 * There is no need to untag the integer, but it IS necessary
+	 * to make sure it is small (a bignum pointer could fall in
+	 * the valid range).
+	 */
+
+	GetArg1(1, i);
+	if (is_not_small(i) || i > make_small(0x10FFFFUL) ||
+	    (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) {
+	    Eterm ms;		/* Match context */
+	    ErlBinMatchBuffer* mb;
+
+	    GetArg1(2, ms);
+	    mb = ms_matchbuffer(ms);
+	    mb->offset -= 32;
+	    goto badarg;
+	}
+	Next(3);
+    }
 
  /*
   * Matching of binaries.
@@ -4292,35 +4296,36 @@ do {						\
  }
 
  /*
-  * tmp_arg1 = Match context
-  * tmp_arg2 = Size field
-  * Operands: Fail Live FlagsAndUnit Dst
+  * Operands: Fail Live FlagsAndUnit Ms Sz Dst
   */
- OpCase(i_bs_get_integer_fIId): {
+ OpCase(i_bs_get_integer_fIIssd): {
      Uint flags;
      Uint size;
+     Eterm Ms;
+     Eterm Sz;
      ErlBinMatchBuffer* mb;
      Eterm result;
 
      flags = Arg(2);
-     BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size);
+     GetArg2(3, Ms, Sz);
+     BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size);
      if (size >= SMALL_BITS) {
 	 Uint wordsneeded;
-	 /* check bits size before potential gc.
+	 /* Check bits size before potential gc.
 	  * We do not want a gc and then realize we don't need
-	  * the allocated space (i.e. if the op fails)
+	  * the allocated space (i.e. if the op fails).
 	  *
-	  * remember to reacquire the matchbuffer after gc.
+	  * Remember to re-acquire the matchbuffer after gc.
 	  */
 
-	 mb = ms_matchbuffer(tmp_arg1);
+	 mb = ms_matchbuffer(Ms);
 	 if (mb->size - mb->offset < size) {
 	     ClauseFail();
 	 }
 	 wordsneeded = 1+WSIZE(NBYTES((Uint) size));
-	 TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
+	 TestHeapPreserve(wordsneeded, Arg(1), Ms);
      }
-     mb = ms_matchbuffer(tmp_arg1);
+     mb = ms_matchbuffer(Ms);
      LIGHT_SWAPOUT;
      result = erts_bs_get_integer_2(c_p, size, flags, mb);
      LIGHT_SWAPIN;
@@ -4328,7 +4333,7 @@ do {						\
      if (is_non_value(result)) {
 	 ClauseFail();
      }
-     StoreBifResult(3, result);
+     StoreBifResult(5, result);
  }
 
  {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index c90576333a..871c2b1f55 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2854,23 +2854,16 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 	    goto generic;
 	}
     } else {
-	GenOp* op2;
-	NEW_GENOP(stp, op2);
-	
-	op->op = genop_i_fetch_2;
-	op->arity = 2;
-	op->a[0] = Ms;
-	op->a[1] = Size;
-	op->next = op2;
-
-	op2->op = genop_i_bs_get_integer_4;
-	op2->arity = 4;
-	op2->a[0] = Fail;
-	op2->a[1] = Live;
-	op2->a[2].type = TAG_u;
-	op2->a[2].val = (Unit.val << 3) | Flags.val;
-	op2->a[3] = Dst;
-	op2->next = NULL;
+	op->op = genop_i_bs_get_integer_6;
+	op->arity = 6;
+	op->a[0] = Fail;
+	op->a[1] = Live;
+	op->a[2].type = TAG_u;
+	op->a[2].val = (Unit.val << 3) | Flags.val;
+	op->a[3] = Ms;
+	op->a[4] = Size;
+	op->a[5] = Dst;
+	op->next = NULL;
 	return op;
     }
     op->next = NULL;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 492d4d240a..637cef0a22 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1122,7 +1122,7 @@ bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 
 i_bs_get_integer_small_imm x I f I d
 i_bs_get_integer_imm x I I f I d
-i_bs_get_integer f I I d
+i_bs_get_integer f I I s s d
 i_bs_get_integer_8 x f d
 i_bs_get_integer_16 x f d
 i_bs_get_integer_32 x f I d
@@ -1195,14 +1195,12 @@ i_bs_get_utf16 x f I d
 
 bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
-	i_fetch Dst Ms | \
-	i_bs_validate_unicode_retract Fail
+	i_bs_validate_unicode_retract Fail Dst Ms
 bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \
-	i_fetch x Ms | \
-	i_bs_validate_unicode_retract Fail
+	i_bs_validate_unicode_retract Fail x Ms
 
-i_bs_validate_unicode_retract j
+i_bs_validate_unicode_retract j s s
 %hot
 
 #
@@ -1224,12 +1222,12 @@ bs_init2 Fail Sz=u Words Regs Flags Dst => \
 bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
   i_bs_init_fail Sz Fail Regs Dst
 bs_init2 Fail Sz Words Regs Flags Dst => \
-  i_fetch Sz x=0 | i_bs_init_fail_heap Words Fail Regs Dst
+  i_bs_init_fail_heap Sz Words Fail Regs Dst
 
 i_bs_init_fail x j I d
 i_bs_init_fail y j I d
 
-i_bs_init_fail_heap I j I d
+i_bs_init_fail_heap s I j I d
 
 i_bs_init I I d
 i_bs_init_heap_bin I I d
@@ -1246,31 +1244,30 @@ bs_init_bits Fail Sz=u Words Regs Flags Dst =>  i_bs_init_bits_heap Sz Words Reg
 bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
   i_bs_init_bits_fail Sz Fail Regs Dst
 bs_init_bits Fail Sz Words Regs Flags Dst => \
-  i_fetch Sz x=0 | i_bs_init_bits_fail_heap Words Fail Regs Dst
+  i_bs_init_bits_fail_heap Sz Words Fail Regs Dst
 
 i_bs_init_bits_fail x j I d
 i_bs_init_bits_fail y j I d
 
-i_bs_init_bits_fail_heap I j I d
+i_bs_init_bits_fail_heap s I j I d
 
 i_bs_init_bits I I d
 i_bs_init_bits_heap I I I d
 
 bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
-bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D
 
-i_bs_add j I d
+bs_add j s s I d
 
 bs_append Fail Size Extra Live Unit Bin Flags Dst => \
-  i_fetch Size Bin | i_bs_append Fail Extra Live Unit Dst
+  move Bin x | i_bs_append Fail Extra Live Unit Size Dst
 
 bs_private_append Fail Size Unit Bin Flags Dst => \
-  i_fetch Size Bin | i_bs_private_append Fail Unit Dst
+  i_bs_private_append Fail Unit Size Bin Dst
 
 bs_init_writable
 
-i_bs_append j I I I d
-i_bs_private_append j I d
+i_bs_append j I I I s d
+i_bs_private_append j I s s d
 
 #
 # Storing integers into binaries.
-- 
cgit v1.2.3


From bf2b06d3edd2c855071529be17c0c35e778a88fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 Jun 2015 16:33:19 +0200
Subject: Remove the i_fetch instruction

---
 erts/emulator/beam/beam_emu.c |  2 --
 erts/emulator/beam/ops.tab    | 15 ---------------
 2 files changed, 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 26e28832cf..fe3d8aea28 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -148,8 +148,6 @@ do {                                     \
    ASSERT(VALID_INSTR(* (Eterm *)(ip))); \
    I = (ip)
 
-#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2)
-
 /*
  * Store a result into a register given a destination descriptor.
  */
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 637cef0a22..631e10215c 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -486,21 +486,6 @@ put_list c y y
 put_list s s d
 %hot
 
-%macro: i_fetch FetchArgs -pack
-i_fetch c x
-i_fetch c y
-i_fetch x c
-i_fetch x x
-i_fetch x y
-i_fetch y c
-i_fetch y x
-i_fetch y y
-
-%cold
-i_fetch c c
-i_fetch s s
-%hot
-
 #
 # Some more only used by the emulator
 #
-- 
cgit v1.2.3


From 227269c6b37b88fe106d27a5c0ba148c31472822 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 Jun 2015 19:30:20 +0200
Subject: Eliminate use of tmp_arg1 and tmp_arg2 in bit syntax

---
 erts/emulator/beam/beam_emu.c | 81 +++++++++++++++++++------------------------
 1 file changed, 36 insertions(+), 45 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index fe3d8aea28..89234a9a80 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -989,7 +989,6 @@ init_emulator(void)
 #  define REG_I asm("%l4")
 #  define REG_fcalls asm("%l5")
 #  define REG_tmp_arg1 asm("%l6")
-#  define REG_tmp_arg2 asm("%l7")
 #else
 #  define REG_xregs
 #  define REG_htop
@@ -997,7 +996,6 @@ init_emulator(void)
 #  define REG_I
 #  define REG_fcalls
 #  define REG_tmp_arg1
-#  define REG_tmp_arg2
 #endif
 
 #ifdef USE_VM_PROBES
@@ -1145,7 +1143,6 @@ void process_main(void)
      * Temporaries used for picking up arguments for instructions.
      */
     register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
-    register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
 
     /*
      * X registers and floating point registers are located in
@@ -4242,55 +4239,49 @@ do {						\
      }
  }
 
- /* Operands: x(Reg) Size Live Fail Flags Dst */
- OpCase(i_bs_get_integer_imm_xIIfId): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
-     /* Operands: Size Live Fail Flags Dst */
-     goto do_bs_get_integer_imm_test_heap;
- }
+ {
+     Eterm Ms, Sz;
 
- /*
-  * tmp_arg1 = match context
-  * Operands: Size Live Fail Flags Dst
-  */
- do_bs_get_integer_imm_test_heap: {
-     Uint wordsneeded;
-     tmp_arg2 = Arg(0);
-     wordsneeded = 1+WSIZE(NBYTES(tmp_arg2));
-     TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
-     I += 2;
-     /* Operands: Fail Flags Dst */
-     goto do_bs_get_integer_imm;
- }
+     /* Operands: x(Reg) Size Live Fail Flags Dst */
+ OpCase(i_bs_get_integer_imm_xIIfId): {
+	 Uint wordsneeded;
+	 Ms = xb(Arg(0));
+	 Sz = Arg(1);
+	 wordsneeded = 1+WSIZE(NBYTES(Sz));
+	 TestHeapPreserve(wordsneeded, Arg(2), Ms);
+	 I += 3;
+	 /* Operands: Fail Flags Dst */
+	 goto do_bs_get_integer_imm;
+     }
 
- /* Operands: x(Reg) Size Fail Flags Dst */
+     /* Operands: x(Reg) Size Fail Flags Dst */
  OpCase(i_bs_get_integer_small_imm_xIfId): {
-     tmp_arg1 = xb(Arg(0));
-     tmp_arg2 = Arg(1);
-     I += 2;
-     /* Operands: Fail Flags Dst */
-     goto do_bs_get_integer_imm;
- }
+	 Ms = xb(Arg(0));
+	 Sz = Arg(1);
+	 I += 2;
+	 /* Operands: Fail Flags Dst */
+	 goto do_bs_get_integer_imm;
+     }
  
- /*
-  * tmp_arg1 = match context
-  * tmp_arg2 = size of field
-  * Operands: Fail Flags Dst
-  */
+     /*
+      * Ms = match context
+      * Sz = size of field
+      * Operands: Fail Flags Dst
+      */
  do_bs_get_integer_imm: {
-     ErlBinMatchBuffer* mb;
-     Eterm result;
+	 ErlBinMatchBuffer* mb;
+	 Eterm result;
 
-     mb = ms_matchbuffer(tmp_arg1);
-     LIGHT_SWAPOUT;
-     result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb);
-     LIGHT_SWAPIN;
-     HEAP_SPACE_VERIFIED(0);
-     if (is_non_value(result)) {
-	 ClauseFail();
+	 mb = ms_matchbuffer(Ms);
+	 LIGHT_SWAPOUT;
+	 result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb);
+	 LIGHT_SWAPIN;
+	 HEAP_SPACE_VERIFIED(0);
+	 if (is_non_value(result)) {
+	     ClauseFail();
+	 }
+	 StoreBifResult(2, result);
      }
-     StoreBifResult(2, result);
  }
 
  /*
-- 
cgit v1.2.3


From dddd225934cfe79c052e9013a8a2c1d6a29c8ae0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 23 Jun 2015 15:24:07 +0200
Subject: Remove the last use of tmp_arg1

---
 erts/emulator/beam/beam_emu.c | 102 +++++++++++++++++++-----------------------
 erts/emulator/beam/ops.tab    |  53 ++++++----------------
 2 files changed, 59 insertions(+), 96 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 89234a9a80..7dbf14de02 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -603,47 +603,39 @@ void** beam_ops;
    H = CAR(tmp_ptr);				\
    T = CDR(tmp_ptr); } while (0)
 
-#define GetTupleElement(Src, Element, Dest)				\
-  do {									\
-    tmp_arg1 = (Eterm) (((unsigned char *) tuple_val(Src)) + (Element));\
-    (Dest) = (*(Eterm *) tmp_arg1);	                		\
-  } while (0)
-
-#define ExtractNextElement(Dest)					  \
-    tmp_arg1 += sizeof(Eterm);						  \
-    (Dest) = (* (Eterm *) (((unsigned char *) tmp_arg1)))
-
-#define ExtractNextElement2(Dest)				\
-  do {								\
-    Eterm* ene_dstp = &(Dest);					\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);			\
-  } while (0)
-
-#define ExtractNextElement3(Dest)		\
-  do {						\
-    Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
-    tmp_arg1 += 3*sizeof(Eterm);		\
+#define GetTupleElement(Src, Element, Dest)		\
+  do {							\
+    Eterm* src;						\
+    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
+    (Dest) = *src;					\
   } while (0)
 
-#define ExtractNextElement4(Dest)		\
-  do {						\
-    Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
-    ene_dstp[3] = ((Eterm *) tmp_arg1)[4];	\
-    tmp_arg1 += 4*sizeof(Eterm);		\
+#define GetTupleElement2(Src, Element, Dest)		\
+  do {							\
+    Eterm* src;						\
+    Eterm* dst;						\
+    Eterm E1, E2;					\
+    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
+    dst = &(Dest);					\
+    E1 = src[0];					\
+    E2 = src[1];					\
+    dst[0] = E1;					\
+    dst[1] = E2;					\
   } while (0)
 
-#define ExtractElement(Element, Dest)		\
-  do {						\
-     tmp_arg1 += (Element);			\
-     (Dest) = (* (Eterm *) tmp_arg1);		\
+#define GetTupleElement3(Src, Element, Dest)		\
+  do {							\
+    Eterm* src;						\
+    Eterm* dst;						\
+    Eterm E1, E2, E3;					\
+    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
+    dst = &(Dest);					\
+    E1 = src[0];					\
+    E2 = src[1];					\
+    E3 = src[2];					\
+    dst[0] = E1;					\
+    dst[1] = E2;					\
+    dst[2] = E3;					\
   } while (0)
 
 #define EqualImmed(X, Y, Action) if (X != Y) { Action; }
@@ -683,11 +675,9 @@ void** beam_ops;
 
 #define IsTuple(X, Action) if (is_not_tuple(X)) Action
 
-#define IsArity(Pointer, Arity, Fail)					  \
-    if (*(Eterm *)							  \
-	(tmp_arg1 = (Eterm) (tuple_val(Pointer))) != (Arity))             \
-    { 									  \
-        Fail; 								  \
+#define IsArity(Pointer, Arity, Fail)		\
+    if (*tuple_val(Pointer) != (Arity)) {	\
+        Fail;					\
     }
 
 #define IsMap(Src, Fail) if (!is_map(Src)) { Fail; }
@@ -724,14 +714,21 @@ void** beam_ops;
      }							\
   } while (0)
 
-#define IsTupleOfArity(Src, Arity, Fail)				      \
-  do {									      \
-    if (is_not_tuple(Src) || 						      \
-	*(Eterm *)							      \
-	(tmp_arg1 = (Eterm) (tuple_val(Src))) != Arity) {                     \
-        Fail;								      \
-    }									      \
+#ifdef DEBUG
+#define IsTupleOfArity(Src, Arityval, Fail)			\
+  do {								\
+    if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) {	\
+        Fail;							\
+    }								\
   } while (0)
+#else
+#define IsTupleOfArity(Src, Arityval, Fail)			\
+  do {								\
+    if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) {	\
+        Fail;							\
+    }								\
+  } while (0)
+#endif
 
 #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
 
@@ -988,14 +985,12 @@ init_emulator(void)
 #  define REG_stop asm("%l3")
 #  define REG_I asm("%l4")
 #  define REG_fcalls asm("%l5")
-#  define REG_tmp_arg1 asm("%l6")
 #else
 #  define REG_xregs
 #  define REG_htop
 #  define REG_stop
 #  define REG_I
 #  define REG_fcalls
-#  define REG_tmp_arg1
 #endif
 
 #ifdef USE_VM_PROBES
@@ -1139,11 +1134,6 @@ void process_main(void)
      */
     register Sint FCALLS REG_fcalls = 0;
 
-    /*
-     * Temporaries used for picking up arguments for instructions.
-     */
-    register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
-
     /*
      * X registers and floating point registers are located in
      * scheduler specific data.
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 631e10215c..853ba82f83 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -230,6 +230,12 @@ i_get_tuple_element x P y
 i_get_tuple_element y P y
 %hot
 
+%macro: i_get_tuple_element2 GetTupleElement2 -pack
+i_get_tuple_element2 x P x
+
+%macro: i_get_tuple_element3 GetTupleElement3 -pack
+i_get_tuple_element3 x P x
+
 %macro: is_number IsNumber -fail_action
 %cold
 is_number f x
@@ -552,48 +558,15 @@ test_arity Fail=f c Arity => jump Fail
 test_arity f x A
 test_arity f y A
 
-is_tuple_of_arity Fail=f Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \
-  is_tuple_of_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P
-
-test_arity Fail Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \
-  test_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P
-
-original_reg Reg P1 | get_tuple_element Reg P2 Dst=xy | succ(P1, P2) => \
-  extract_next_element Dst | original_reg Reg P2
-
-get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg P
-
-original_reg Reg Pos =>
-
-original_reg/2
-
-extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \
-succ(P1, P2) | succ(D1, D2) => \
-  extract_next_element2 D1 | original_reg Reg P2
-
-extract_next_element2 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \
-succ(P1, P2) | succ2(D1, D2) => \
-  extract_next_element3 D1 | original_reg Reg P2
-
-#extract_next_element3 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \
-#succ(P1, P2) | succ3(D1, D2) => \
-#  extract_next_element4 D1 | original_reg Reg P2
-
-%macro: extract_next_element ExtractNextElement -pack
-extract_next_element x
-extract_next_element y
-
-%macro: extract_next_element2 ExtractNextElement2 -pack
-extract_next_element2 x
-extract_next_element2 y
+get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \
+   get_tuple_element Reg=x P3 D3=x | \
+   succ(P1, P2) | succ(P2, P3) | \
+   succ(D1, D2) | succ(D2, D3) => i_get_tuple_element3 Reg P1 D1
 
-%macro: extract_next_element3 ExtractNextElement3 -pack
-extract_next_element3 x
-extract_next_element3 y
+get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \
+   succ(P1, P2) | succ(D1, D2) => i_get_tuple_element2 Reg P1 D1
 
-#%macro: extract_next_element4 ExtractNextElement4 -pack
-#extract_next_element4 x
-#extract_next_element4 y
+get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst
 
 is_integer Fail=f i =>
 is_integer Fail=f an => jump Fail
-- 
cgit v1.2.3


From d252130b8e880f2f7f826217fe806da76fbcbb7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 10 Apr 2012 12:28:11 +0200
Subject: Rewrite the hipe_mode_switch instructions

The 'cmd' variable that were shared by several hipe_mode_switch
instructions would cause clang to produce sub-optimal code,
probably because it considered the instructions as part of of
loop that needed to be optimized.

What would was that 'cmd' would be assigned to the ESI register
(lower 32 bits of the RSI register). It would use ESI for other
purposes in instructions, but at the end of every instruction
it would set ESI to 1 just in case the next instruction happened
to be hipe_trap_return. This can be seen clearly if this commit
is omitted and the define HIPE_MODE_SWITCH_CMD_RETURN in
hipe/hipe_mode_switch.h is changed from 1 to some other number
such as 42. You will see that 42 is assigned to ESI at the end
of every instruction.

Eliminate this problem by elimininating the shared 'cmd' variable.
---
 erts/emulator/beam/beam_emu.c | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 7dbf14de02..362c6e4826 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4681,7 +4681,12 @@ do {						\
 
 #ifdef HIPE
  {
-     unsigned cmd;
+#define HIPE_MODE_SWITCH(Cmd)			\
+     SWAPOUT;					\
+     c_p->fcalls = FCALLS;			\
+     c_p->def_arg_reg[4] = -neg_o_reds;		\
+     c_p = hipe_mode_switch(c_p, Cmd, reg);     \
+     goto L_post_hipe_mode_switch
 
      OpCase(hipe_trap_call): {
 	 /*
@@ -4695,38 +4700,31 @@ do {						\
 	  */
 	 ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
 	 c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
-	 cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8);
 	 ++hipe_trap_count;
-	 goto L_hipe_mode_switch;
+	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8));
      }
      OpCase(hipe_trap_call_closure): {
        ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
        c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
-       cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8);
        ++hipe_trap_count;
-       goto L_hipe_mode_switch;
+       HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8));
      }
      OpCase(hipe_trap_return): {
-	 cmd = HIPE_MODE_SWITCH_CMD_RETURN;
-	 goto L_hipe_mode_switch;
+	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN);
      }
      OpCase(hipe_trap_throw): {
-	 cmd = HIPE_MODE_SWITCH_CMD_THROW;
-	 goto L_hipe_mode_switch;
+	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW);
      }
      OpCase(hipe_trap_resume): {
-	 cmd = HIPE_MODE_SWITCH_CMD_RESUME;
-	 goto L_hipe_mode_switch;
+	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME);
      }
- L_hipe_mode_switch:
-     /* XXX: this abuse of def_arg_reg[] is horrid! */
-     SWAPOUT;
-     c_p->fcalls = FCALLS;
-     c_p->def_arg_reg[4] = -neg_o_reds;
-     c_p = hipe_mode_switch(c_p, cmd, reg);
+#undef HIPE_MODE_SWITCH
+
+ L_post_hipe_mode_switch:
      reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
      freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
      ERL_BITS_RELOAD_STATEP(c_p);
+     /* XXX: this abuse of def_arg_reg[] is horrid! */
      neg_o_reds = -c_p->def_arg_reg[4];
      FCALLS = c_p->fcalls;
      SWAPIN;
-- 
cgit v1.2.3


From 1f73d45327bb13a615f2f0a8d9d4888ddacb95a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 23 Jun 2015 09:38:23 +0200
Subject: Add back frequently used x(0) instructions

---
 erts/emulator/beam/beam_emu.c | 10 ++++++++++
 erts/emulator/beam/ops.tab    | 37 +++++++++++++++++++++++++++++++++----
 2 files changed, 43 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 362c6e4826..bac7057abe 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1298,6 +1298,11 @@ void process_main(void)
 	    Uint live;
 	    Eterm result;
 
+	OpCase(i_increment_rIId):
+	    increment_reg_val = x(0);
+	    I--;
+	    goto do_increment;
+
 	OpCase(i_increment_xIId):
 	    increment_reg_val = xb(Arg(0));
 	    goto do_increment;
@@ -1357,6 +1362,11 @@ void process_main(void)
      PlusOp2 = xb(Arg(3));
      goto do_plus;
 
+ OpCase(i_plus_jIxyd):
+     PlusOp1 = xb(Arg(2));
+     PlusOp2 = yb(Arg(3));
+     goto do_plus;
+
  OpCase(i_plus_jIssd):
      GetArg2(2, PlusOp1, PlusOp2);
      goto do_plus;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 853ba82f83..4ab08faae3 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -204,6 +204,14 @@ get_list y x y
 get_list y y x
 get_list y y y
 
+# The following get_list instructions using x(0) are frequently used.
+get_list r x x
+get_list r r y
+get_list x r x
+get_list r x y
+get_list r y r
+get_list r x r
+
 # Old-style catch.
 catch y f
 catch_end y
@@ -348,6 +356,14 @@ move c x
 move n x
 move y y
 
+# The following move instructions using x(0) are frequently used.
+
+move x r
+move r x
+move y r
+move c r
+move r y
+
 # Receive operations.
 
 loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail
@@ -390,6 +406,7 @@ is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C
 is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C
 
 %macro: i_is_eq_exact_immed EqualImmed -fail_action
+i_is_eq_exact_immed f r c
 i_is_eq_exact_immed f x c
 i_is_eq_exact_immed f y c
 
@@ -478,15 +495,21 @@ put_list x c x
 put_list x c y
 
 put_list y c x
-put_list y c y
 
 # put_list Constant SrcReg Dst
 
 put_list c x x
-put_list c x y
-
 put_list c y x
-put_list c y y
+
+# The following put_list instructions using x(0) are frequently used.
+
+put_list y r r
+put_list x r r
+put_list r n r
+put_list r n x
+put_list r x x
+put_list r x r
+put_list x x r
 
 %cold
 put_list s s d
@@ -544,10 +567,12 @@ is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S
 
 %macro:is_tuple_of_arity IsTupleOfArity -fail_action
 
+is_tuple_of_arity f r A
 is_tuple_of_arity f x A
 is_tuple_of_arity f y A
 
 %macro: is_tuple IsTuple -fail_action
+is_tuple f r
 is_tuple f x
 is_tuple f y
 
@@ -593,6 +618,7 @@ is_list f y
 is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
 
 %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
+is_nonempty_list_allocate f r I t
 is_nonempty_list_allocate f x I t
 
 is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2
@@ -1440,6 +1466,7 @@ gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \
 # GCing arithmetic instructions.
 #
 
+gen_plus Fail Live Y=y X=x Dst => i_plus Fail Live X Y Dst
 gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst
 
 gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst
@@ -1471,10 +1498,12 @@ gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \
 
 gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 
+i_increment r I I d
 i_increment x I I d
 i_increment y I I d
 
 i_plus j I x x d
+i_plus j I x y d
 i_plus j I s s d
 
 i_minus j I x x d
-- 
cgit v1.2.3


From 3dc2bee53b4d36f41821a6ab512cf01c958c11f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 23 Jun 2015 17:31:49 +0200
Subject: Introduce specialized versions of move2

Currently, move2/2 does the two moves sequentially to ensure
that the instruction will always work correctly.

We can do better than that. If the two move instructions have
any registers in common, we can introduce simpler and slightly
more efficient instructions to handle those cases:

  move_shift/3
  move_dup/3

For the remaining cases when the the move instructions
have no common registers, the move2/4 instruction can perform
the moves in parallel which is probably slightly more efficient.
For clarity's sake, we will remain the instruction to move2_par/4.
---
 erts/emulator/beam/beam_emu.c | 18 ++++++++++-
 erts/emulator/beam/ops.tab    | 71 +++++++++++++++++++++++++++++++------------
 2 files changed, 68 insertions(+), 21 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index bac7057abe..2c10e7ae7c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -551,7 +551,23 @@ void** beam_ops;
        Store(term, Dst);           \
    } while (0)
 
-#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2)
+#define Move2Par(S1, D1, S2, D2)		\
+  do {						\
+      Eterm V1, V2;				\
+      V1 = (S1); V2 = (S2); D1 = V1; D2 = V2;	\
+  } while (0)
+
+#define MoveShift(Src, SD, D)			\
+  do {						\
+    Eterm V;					\
+    V = Src; D = SD; SD = V;			\
+  } while (0)
+
+#define MoveDup(Src, D1, D2)			\
+  do {						\
+    D1 = D2 = (Src);				\
+  } while (0)
+
 #define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)
 
 #define MoveReturn(Src)				\
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4ab08faae3..97525c3b72 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -303,21 +303,40 @@ move_window3 x x x y
 move_window4 x x x x y
 move_window5 x x x x x y
 
-move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
-move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
-move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
+move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2
+move Src=x SD=x | move SD=x D=x   => move_dup Src SD D
+move Src=x D1=x | move Src=x D2=y => move_dup Src D1 D2
+move Src=y SD=x | move SD=x D=y   => move_dup Src SD D
+move Src=x SD=x | move SD=x D=y   => move_dup Src SD D
+move Src=y SD=x | move SD=x D=x   => move_dup Src SD D
 
-move X1=x X2=x | move X3=x Y1=y => move2 X1 X2 X3 Y1
+move SD=x D=x | move Src=xy SD=x => move_shift Src SD D
+move SD=y D=x | move Src=x  SD=y => move_shift Src SD D
+move SD=x D=y | move Src=x  SD=x => move_shift Src SD D
 
-move S1=x S2=x | move X1=x Y1=y => move2 S1 S2 X1 Y1
-move S1=y S2=x | move X1=x Y1=y => move2 S1 S2 X1 Y1
+# The transformations above guarantee that the source for
+# the second move is not the same as the destination for
+# the first move. That means that we can do the moves in
+# parallel (fetch both values, then store them) which could
+# be faster.
 
-move Y1=y X1=x | move S1=x D1=x => move2 Y1 X1 S1 D1
-move S1=x D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1
+move X1=x Y1=y | move X2=x Y2=y => move2_par X1 Y1 X2 Y2
+move Y1=y X1=x | move Y2=y X2=x => move2_par Y1 X1 Y2 X2
 
-move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3
-move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3
-move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6
+move X1=x X2=x | move X3=x X4=x => move2_par X1 X2 X3 X4
+
+move X1=x X2=x | move X3=x Y1=y => move2_par X1 X2 X3 Y1
+
+move S1=x S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1
+
+move S1=y S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1
+
+move Y1=y X1=x | move S1=x D1=x => move2_par Y1 X1 S1 D1
+move S1=x D1=x | move Y1=y X1=x => move2_par S1 D1 Y1 X1
+
+move2_par X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3
+move2_par Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3
+move2_par X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6
 
 move C=aiq X=x==1 => move_x1 C
 move C=aiq X=x==2 => move_x2 C
@@ -325,18 +344,30 @@ move C=aiq X=x==2 => move_x2 C
 move_x1 c
 move_x2 c
 
-%macro: move2 Move2 -pack
-move2 x y x y
-move2 y x y x
-move2 x x x x
+%macro: move_shift MoveShift -pack
+move_shift x x x
+move_shift y x x
+move_shift x y x
+move_shift x x y
+
+%macro: move_dup MoveDup -pack
+move_dup x x x
+move_dup x x y
+move_dup y x x
+move_dup y x y
+
+%macro: move2_par Move2Par -pack
+
+move2_par x y x y
+move2_par y x y x
+move2_par x x x x
 
-move2 x x x y
+move2_par x x x y
 
-move2 x y x y
-move2 y x x y
+move2_par y x x y
 
-move2 x x y x
-move2 y x x x
+move2_par x x y x
+move2_par y x x x
 
 %macro: move3 Move3
 move3 x y x y x y
-- 
cgit v1.2.3


From 62473daf8169a04a07409f344d938bc51a4536c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 25 Jun 2015 13:21:14 +0200
Subject: Introduce swap_temp/3 and swap/2

Sequences of three move instructionst that effectively swap the
contents of two registers are fairly common. We can replace them
with a swap_temp/3 instruction. The third operand is the temporary
register to be used for swapping, since the temporary register
may actually be used.

If swap_temp/3 instruction is followed by a call, the temporary
register will often (but not always) be killed by the call. If
it is killed, we can replace the swap_temp/3 instruction with a
slightly cheaper swap/2 instruction.
---
 erts/emulator/beam/beam_emu.c  | 14 ++++++++++++++
 erts/emulator/beam/beam_load.c |  6 ++++++
 erts/emulator/beam/ops.tab     | 28 ++++++++++++++++++++++++++++
 3 files changed, 48 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2c10e7ae7c..4e6e7aa48a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -545,6 +545,20 @@ void** beam_ops;
    HTOP += 2;                      \
   } while (0)
 
+#define Swap(R1, R2)				\
+  do {						\
+    Eterm V = R1;				\
+    R1 = R2;					\
+    R2 = V;					\
+  } while (0)
+
+#define SwapTemp(R1, R2, Tmp)			\
+  do {						\
+    Eterm V = R1;				\
+    R1 = R2;					\
+    R2 = Tmp = V;				\
+  } while (0)
+
 #define Move(Src, Dst, Store)      \
    do {                            \
        Eterm term = (Src);         \
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 871c2b1f55..1e3690fcd6 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2702,6 +2702,12 @@ same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label)
 	Target.val == Label.val;
 }
 
+static int
+is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live)
+{
+    return Reg.type == TAG_x && Live.type == TAG_u &&
+	Live.val <= Reg.val;
+}
 
 /*
  * Generate an instruction for element/2.
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 97525c3b72..386de0a658 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -303,6 +303,34 @@ move_window3 x x x y
 move_window4 x x x x y
 move_window5 x x x x x y
 
+# Swap registers.
+move R1=x Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp
+
+swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed(Tmp, Live) => \
+  swap R1 R2 | line Loc | apply Live
+
+swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \
+  swap R1 R2 | line Loc | call Live Addr
+swap_temp R1 R2 Tmp | call_only Live Addr | \
+  is_killed(Tmp, Live) => swap R1 R2 | call_only Live Addr
+swap_temp R1 R2 Tmp | call_last Live Addr D | \
+  is_killed(Tmp, Live) => swap R1 R2 | call_last Live Addr D
+
+swap_temp R1 R2 Tmp | line Loc | call_ext Live Addr | is_killed(Tmp, Live) => \
+  swap R1 R2 | line Loc | call_ext Live Addr
+swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \
+  is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_only Live Addr
+swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \
+  is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D
+
+%macro: swap_temp SwapTemp -pack
+swap_temp x x x
+swap_temp x y x
+
+%macro: swap Swap -pack
+swap x x
+swap x y
+
 move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2
 move Src=x SD=x | move SD=x D=x   => move_dup Src SD D
 move Src=x D1=x | move Src=x D2=y => move_dup Src D1 D2
-- 
cgit v1.2.3


From 5e83d2ae89c731bd7e4c32b550150336008d2974 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 25 Jun 2015 16:18:18 +0200
Subject: Use a cheaper tag scheme for 'd' operands

Since 'd' operands can only either an X register or an Y register,
we only need a single bit to distinguish them. Furthermore, we can
pre-multiply the register number with the word size to speed up
address calculation.
---
 erts/emulator/beam/beam_debug.c | 13 ++++------
 erts/emulator/beam/beam_emu.c   | 57 ++++++++++++++++++-----------------------
 erts/emulator/beam/beam_load.c  |  4 +--
 erts/emulator/beam/beam_load.h  |  1 +
 4 files changed, 33 insertions(+), 42 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 30e3765502..ea1f2cd012 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -475,15 +475,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    ap++;
 	    break;
 	case 'd':		/* Destination (x(0), x(N), y(N)) */
-	    switch (loader_tag(*ap)) {
-	    case LOADER_X_REG:
-		erts_print(to, to_arg, "x(%d)",
-			   loader_x_reg_index(*ap));
-		break;
-	    case LOADER_Y_REG:
+	    if (*ap & 1) {
 		erts_print(to, to_arg, "y(%d)",
-			   loader_y_reg_index(*ap) - CP_SIZE);
-		break;
+			   *ap / sizeof(Eterm) - CP_SIZE);
+	    } else {
+		erts_print(to, to_arg, "x(%d)",
+			   *ap / sizeof(Eterm));
 	    }
 	    ap++;
 	    break;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4e6e7aa48a..60a399a5ac 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -148,23 +148,21 @@ do {                                     \
    ASSERT(VALID_INSTR(* (Eterm *)(ip))); \
    I = (ip)
 
+/*
+ * Register target (X or Y register).
+ */
+#define REG_TARGET(Target) (*(((Target) & 1) ? &yb(Target-1) : &xb(Target)))
+
 /*
  * Store a result into a register given a destination descriptor.
  */
 
-#define StoreResult(Result, DestDesc)			\
-  do {							\
-    Eterm stb_reg;					\
-    stb_reg = (DestDesc);				\
-    CHECK_TERM(Result);					\
-    switch (loader_tag(stb_reg)) {			\
-    case LOADER_X_REG:					\
-      x(loader_x_reg_index(stb_reg)) = (Result);	\
-      break;						\
-    default:						\
-      y(loader_y_reg_index(stb_reg)) = (Result);	\
-      break;						\
-    }							\
+#define StoreResult(Result, DestDesc)		\
+  do {						\
+    Eterm stb_reg;				\
+    stb_reg = (DestDesc);			\
+    CHECK_TERM(Result);				\
+    REG_TARGET(stb_reg) = (Result);		\
   } while (0)
 
 #define StoreSimpleDest(Src, Dest) Dest = (Src)
@@ -175,22 +173,16 @@ do {                                     \
  * be just before the next instruction.
  */
  
-#define StoreBifResult(Dst, Result)			\
-  do {							\
-    BeamInstr* stb_next;				\
-    Eterm stb_reg;					\
-    stb_reg = Arg(Dst);					\
-    I += (Dst) + 2;					\
-    stb_next = (BeamInstr *) *I;			\
-    CHECK_TERM(Result);					\
-    switch (loader_tag(stb_reg)) {			\
-    case LOADER_X_REG:					\
-      x(loader_x_reg_index(stb_reg)) = (Result);	\
-      Goto(stb_next);					\
-    default:						\
-      y(loader_y_reg_index(stb_reg)) = (Result);	\
-      Goto(stb_next);					\
-    }							\
+#define StoreBifResult(Dst, Result)		\
+  do {						\
+    BeamInstr* stb_next;			\
+    Eterm stb_reg;				\
+    stb_reg = Arg(Dst);				\
+    I += (Dst) + 2;				\
+    stb_next = (BeamInstr *) *I;		\
+    CHECK_TERM(Result);				\
+    REG_TARGET(stb_reg) = (Result);		\
+    Goto(stb_next);				\
   } while (0)
 
 #define ClauseFail() goto jump_f
@@ -3277,7 +3269,8 @@ do {						\
      Eterm* p;
      
      PreFetch(3, next);
-     GetArg2(0, element, tuple);
+     GetArg1(0, element);
+     tuple = REG_TARGET(Arg(1));
      ASSERT(is_tuple(tuple));
      p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2));
      *p = element;
@@ -4605,7 +4598,7 @@ do {						\
      BeamInstr *next;
 
      PreFetch(2, next);
-     GetR(0, targ1);
+     targ1 = REG_TARGET(Arg(0));
      /* Arg(0) == HEADER_FLONUM */
      GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
      NextPF(2, next);
@@ -4625,7 +4618,7 @@ do {						\
      Eterm fr = Arg(1);
      BeamInstr *next;
 
-     GetR(0, targ1);
+     targ1 = REG_TARGET(Arg(0));
      PreFetch(2, next);
      if (is_small(targ1)) {
 	 fb(fr) = (double) signed_val(targ1);
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 1e3690fcd6..a9d47eb7b0 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2227,10 +2227,10 @@ load_code(LoaderState* stp)
 	    case 'd':	/* Destination (x(0), x(N), y(N) */
 		switch (tag) {
 		case TAG_x:
-		    code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
+		    code[ci++] = tmp_op->a[arg].val * sizeof(Eterm);
 		    break;
 		case TAG_y:
-		    code[ci++] = make_loader_y_reg(tmp_op->a[arg].val);
+		    code[ci++] = tmp_op->a[arg].val * sizeof(Eterm) + 1;
 		    break;
 		default:
 		    LoadError1(stp, "bad tag %d for destination",
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index d5af634fad..eedb5ee4cd 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -52,6 +52,7 @@ extern BeamInstr* em_call_error_handler;
 extern BeamInstr* em_apply_bif;
 extern BeamInstr* em_call_nif;
 
+
 /*
  * The following variables keep a sorted list of address ranges for
  * each module.  It allows us to quickly find a function given an
-- 
cgit v1.2.3


From a52bd8a9ebece53c67024e3ad17a899c22cea2ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 29 Jun 2015 15:56:44 +0200
Subject: Ensure that the move_call_ext_{last,only} instructions are used

Update transformations to ensure that the move_call_ext_last
and move_call_ext_last are used.
---
 erts/emulator/beam/ops.tab | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 386de0a658..18202a4200 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -76,17 +76,6 @@ return
 # with the following call instruction, we need to make sure that
 # there is no line/1 instruction between the move and the call.
 #
-
-move S X0=x==0 | line Loc | call_ext Ar Func => \
-     line Loc | move S X0 | call_ext Ar Func
-move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_bif D => \
-     line Loc | move S X0 | call_ext_last Ar Func D
-move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_bif => \
-     line Loc | move S X0 | call_ext_only Ar Func
-move S X0=x==0 | line Loc | call Ar Func => \
-     line Loc | move S X0 | call Ar Func
-
-#
 # A tail-recursive call to an external function (non-BIF) will
 # never be saved on the stack, so there is no reason to keep
 # the line instruction. (The compiler did not remove the line
@@ -94,10 +83,14 @@ move S X0=x==0 | line Loc | call Ar Func => \
 # BIFs and ordinary Erlang functions.)
 #
 
-line Loc | call_ext_last Ar Func=u$is_not_bif D => \
-     call_ext_last Ar Func D
-line Loc | call_ext_only Ar Func=u$is_not_bif => \
-     call_ext_only Ar Func
+move S X0=x==0 | line Loc | call_ext Ar Func => \
+     line Loc | move S X0 | call_ext Ar Func
+move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \
+     move S X0 | call_ext_last Ar Func D
+move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \
+     move S X0 | call_ext_only Ar Func
+move S X0=x==0 | line Loc | call Ar Func => \
+     line Loc | move S X0 | call Ar Func
 
 line Loc | func_info M F A => func_info M F A | line Loc
 
-- 
cgit v1.2.3


From fad052472def54fff1268b21313b15bd666437c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sun, 21 Jun 2015 18:09:52 +0200
Subject: Teach beam_makeops to pack operands for move3 and move_window

It is currently only possible to pack up to 4 operands. However,
the move_window4 instrucion has 5 operands and move_window5 and
move3 instrucations have 6 operands.

Teach beam_makeops to pack instructions with 5 or 6 operands.
Also rewrite the move_window instructions in beam_emu.c to macros
to allow their operands to get packed.
---
 erts/emulator/beam/beam_emu.c | 88 +++++++++++++++++++++----------------------
 erts/emulator/beam/ops.tab    |  6 ++-
 2 files changed, 47 insertions(+), 47 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 60a399a5ac..a3b38adc4b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -576,6 +576,48 @@ void** beam_ops;
 
 #define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)
 
+#define MoveWindow3(S1, S2, S3, D)		\
+  do {						\
+      Eterm xt0, xt1, xt2;			\
+      Eterm *y = &D;				\
+      xt0  = S1;				\
+      xt1  = S2;				\
+      xt2  = S3;				\
+      y[0] = xt0;				\
+      y[1] = xt1;				\
+      y[2] = xt2;				\
+ } while (0)
+
+#define MoveWindow4(S1, S2, S3, S4, D)		\
+  do {						\
+      Eterm xt0, xt1, xt2, xt3;			\
+      Eterm *y = &D;				\
+      xt0  = S1;				\
+      xt1  = S2;				\
+      xt2  = S3;				\
+      xt3  = S4;				\
+      y[0] = xt0;				\
+      y[1] = xt1;				\
+      y[2] = xt2;				\
+      y[3] = xt3;				\
+ } while (0)
+
+#define MoveWindow5(S1, S2, S3, S4, S5, D)	\
+  do {						\
+      Eterm xt0, xt1, xt2, xt3, xt4;		\
+      Eterm *y = &D;				\
+      xt0  = S1;				\
+      xt1  = S2;				\
+      xt2  = S3;				\
+      xt3  = S4;				\
+      xt4  = S5;				\
+      y[0] = xt0;				\
+      y[1] = xt1;				\
+      y[2] = xt2;				\
+      y[3] = xt3;				\
+      y[4] = xt4;				\
+ } while (0)
+
 #define MoveReturn(Src)				\
     x(0) = (Src);				\
     I = c_p->cp;				\
@@ -1466,52 +1508,6 @@ void process_main(void)
 	Next(3);
     }
 
- OpCase(move_window3_xxxy): {
-     BeamInstr *next;
-     Eterm xt0, xt1, xt2;
-     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3)));
-     PreFetch(4, next);
-     xt0  = xb(Arg(0));
-     xt1  = xb(Arg(1));
-     xt2  = xb(Arg(2));
-     y[0] = xt0;
-     y[1] = xt1;
-     y[2] = xt2;
-     NextPF(4, next);
- }
- OpCase(move_window4_xxxxy): {
-     BeamInstr *next;
-     Eterm xt0, xt1, xt2, xt3;
-     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4)));
-     PreFetch(5, next);
-     xt0  = xb(Arg(0));
-     xt1  = xb(Arg(1));
-     xt2  = xb(Arg(2));
-     xt3  = xb(Arg(3));
-     y[0] = xt0;
-     y[1] = xt1;
-     y[2] = xt2;
-     y[3] = xt3;
-     NextPF(5, next);
- }
- OpCase(move_window5_xxxxxy): {
-     BeamInstr *next;
-     Eterm xt0, xt1, xt2, xt3, xt4;
-     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5)));
-     PreFetch(6, next);
-     xt0  = xb(Arg(0));
-     xt1  = xb(Arg(1));
-     xt2  = xb(Arg(2));
-     xt3  = xb(Arg(3));
-     xt4  = xb(Arg(4));
-     y[0] = xt0;
-     y[1] = xt1;
-     y[2] = xt2;
-     y[3] = xt3;
-     y[4] = xt4;
-     NextPF(6, next);
- }
-
  OpCase(i_move_call_only_fc): {
      r(0) = Arg(1);
  }
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 18202a4200..bce6bacd5c 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -292,6 +292,10 @@ move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \
 move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1
 move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1
 
+%macro: move_window3 MoveWindow3 -pack
+%macro: move_window4 MoveWindow4 -pack
+%macro: move_window5 MoveWindow5 -pack
+
 move_window3 x x x y
 move_window4 x x x x y
 move_window5 x x x x x y
@@ -390,7 +394,7 @@ move2_par y x x y
 move2_par x x y x
 move2_par y x x x
 
-%macro: move3 Move3
+%macro: move3 Move3 -pack
 move3 x y x y x y
 move3 y x y x y x
 move3 x x x x x x
-- 
cgit v1.2.3


From 6d6fce257cf666113e982c0887121739dff5bf1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sun, 1 Apr 2012 09:28:34 +0200
Subject: Eliminate prefetch for conditional instructions

Not pre-fetching in conditional instructions (instructions that use
-fail_action) seems to improve performance slightly.

The reason for that is that conditional instructions may jump to the
failure label, wasting the pre-fetched instruction. Another reason
is that that many conditional instructions do a function call, and
the register pressure is therefore high. Avoiding the pre-fetch
may reduce the register pressure and pontentially result in more
efficient code.
---
 erts/emulator/beam/ops.tab | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index bce6bacd5c..3d52da77d3 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -477,7 +477,7 @@ i_is_ne_exact_literal f x c
 i_is_ne_exact_literal f y c
 
 is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y
-%macro: is_eq_exact EqualExact -fail_action
+%macro: is_eq_exact EqualExact -fail_action -pack
 is_eq_exact f x x
 is_eq_exact f x y
 is_eq_exact f s s
-- 
cgit v1.2.3


From 7bfff20c48cdcabf1eae4b193a374a72977b9d3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 2 Jul 2015 14:50:47 +0200
Subject: Eliminate the variable temp_bits at the top scope of process_main()

---
 erts/emulator/beam/beam_emu.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a3b38adc4b..1ab8c71685 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -819,6 +819,7 @@ void** beam_ops;
 #define BsGetFieldSize(Bits, Unit, Fail, Target)	\
    do {							\
       Sint _signed_size; Uint _uint_size;		\
+      Uint temp_bits;					\
       if (is_small(Bits)) {				\
         _signed_size = signed_val(Bits);		\
          if (_signed_size < 0) { Fail; }		\
@@ -833,6 +834,7 @@ void** beam_ops;
 #define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target)	\
    do {								\
       Sint _signed_size; Uint _uint_size;			\
+      Uint temp_bits;						\
       if (is_small(Bits)) {					\
         _signed_size = signed_val(Bits);			\
          if (_signed_size < 0) { Fail; }			\
@@ -1219,8 +1221,6 @@ void process_main(void)
 #endif
 #endif
 
-    Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
-
     Eterm pt_arity;		/* Used by do_put_tuple */
 
     Uint64 start_time = 0;          /* Monitor long schedule */
-- 
cgit v1.2.3


From e820b8534fd440aaf4991f3c57b8990ce836c8b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 4 Jul 2015 08:58:33 +0200
Subject: Speed up list matching

The combination is_non_empty_list followed by get_list is extremly
common (but not in estone_SUITE, which is why it has not been noticed
before). Therefore it is worthwile to introduce a combined
instruction.
---
 erts/emulator/beam/beam_emu.c | 11 +++++++++++
 erts/emulator/beam/ops.tab    |  7 +++++++
 2 files changed, 18 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1ab8c71685..989066a1ca 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -737,6 +737,17 @@ void** beam_ops;
     if (is_not_list(x(0))) { Fail; }			\
     TestHeap(Need, Alive)
 
+#define IsNonemptyListGetList(Src, H, T, Fail)	\
+    if (is_not_list(Src)) {			\
+         Fail;					\
+    } else {					\
+       Eterm* tmp_ptr = list_val(Src);		\
+       Eterm hd, tl;				\
+       hd = CAR(tmp_ptr);			\
+       tl = CDR(tmp_ptr);			\
+       H = hd; T = tl;				\
+    }
+
 #define IsTuple(X, Action) if (is_not_tuple(X)) Action
 
 #define IsArity(Pointer, Arity, Fail)		\
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 3d52da77d3..46fefb88af 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -682,6 +682,13 @@ is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1
 %macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack
 is_non_empty_list_test_heap f I t
 
+is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \
+  is_nonempty_list_get_list Fail S D1 D2
+
+%macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack
+is_nonempty_list_get_list f r x x
+is_nonempty_list_get_list f x x x
+
 %macro: is_nonempty_list IsNonemptyList -fail_action
 is_nonempty_list f x
 is_nonempty_list f y
-- 
cgit v1.2.3


From c5755ae5ded3ba27dc5d884303ead232abc77a40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 4 Jul 2015 17:10:14 +0200
Subject: Slightly tweak the peformance for get_list

Fetch the head and tail parts to temporary variables before
writing them to their destinations. That should allow the CPU to
perform the moves in parallel, which might improve performance.
---
 erts/emulator/beam/beam_emu.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 989066a1ca..a296713a0a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -662,10 +662,14 @@ void** beam_ops;
      SET_I((BeamInstr *) Arg(0));		\
      Goto(*I);
 
-#define GetList(Src, H, T) do {			\
-   Eterm* tmp_ptr = list_val(Src);		\
-   H = CAR(tmp_ptr);				\
-   T = CDR(tmp_ptr); } while (0)
+#define GetList(Src, H, T)			\
+  do {						\
+    Eterm* tmp_ptr = list_val(Src);		\
+    Eterm hd, tl;				\
+    hd = CAR(tmp_ptr);				\
+    tl = CDR(tmp_ptr);				\
+    H = hd; T = tl;				\
+  } while (0)
 
 #define GetTupleElement(Src, Element, Dest)		\
   do {							\
-- 
cgit v1.2.3


From e98fb1920ad053d2db4594c5d33cdfcfcdc6d771 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 6 Jul 2015 20:25:28 +0200
Subject: Teach non-smp VM how to deal with trace port crash

---
 erts/emulator/beam/beam_emu.c    | 44 +++++++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_process.c | 16 +++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8bfb7d2ad2..a4e9fe1cba 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2082,6 +2082,22 @@ void process_main(void)
 	 OpCase(wait_f):
 
 	 wait2: {
+#ifndef ERTS_SMP
+	     if (ERTS_PROC_IS_EXITING(c_p)) {
+		 /*
+		  * I non smp case:
+		  *
+		  * Currently executing process might be sent an exit
+		  * signal if it is traced by a port that it also is
+		  * linked to, and the port terminates during the
+		  * trace. In this case we do *not* want to clear
+		  * the active flag, which will make the process hang
+		  * in limbo forever.
+		  */
+		 SWAPOUT;
+		 goto do_schedule;
+	     }
+#endif
 	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
 	     SWAPOUT;
 	     c_p->arity = 0;
@@ -6110,6 +6126,23 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
     int arity;
     Eterm tmp;
 
+#ifndef ERTS_SMP
+    if (ERTS_PROC_IS_EXITING(c_p)) {
+	/*
+	 * I non smp case:
+	 *
+	 * Currently executing process might be sent an exit
+	 * signal if it is traced by a port that it also is
+	 * linked to, and the port terminates during the
+	 * trace. In this case we do *not* want to clear
+	 * the active flag, which will make the process hang
+	 * in limbo forever. Get out of here and terminate
+	 * the process...
+	 */
+	return -1;
+    }
+#endif
+
     if (is_not_atom(module) || is_not_atom(function)) {
 	/*
 	 * No need to test args here -- done below.
@@ -6186,7 +6219,16 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-#ifdef ERTS_SMP
+#ifndef ERTS_SMP
+	if (ERTS_PROC_IS_EXITING(c_p)) {
+	    /*
+	     * See comment in the begining of the function...
+	     *
+	     * This second test is needed since gc might be traced.
+	     */
+	    return -1;
+	}
+#else /* ERTS_SMP */
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (!c_p->msg.len)
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ea63d20dfa..b6ad8575cf 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11082,6 +11082,22 @@ set_proc_exiting(Process *p,
     cancel_timer(p);
     p->i = (BeamInstr *) beam_exit;
 
+#ifndef ERTS_SMP
+    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) {
+	/*
+	 * I non smp case:
+	 *
+	 * Currently executing process might be sent an exit
+	 * signal if it is traced by a port that it also is
+	 * linked to, and the port terminates during the
+	 * trace. In this case we want schedule out the
+	 * process as quickly as possible in order to detect
+	 * the event as fast as possible.
+	 */
+	ERTS_VBUMP_ALL_REDS(p);
+    }
+#endif
+
     if (enqueue)
 	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
 		 state,
-- 
cgit v1.2.3


From b3b9d321ce0622e1d774ad54e9700e22edde8abd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 8 Jul 2015 16:13:30 +0200
Subject: Fix crash when disassembling modules with BIFs

In a debug-compiled emulator, running erts_debug:df(io) would
trigger an assertion failure:

1> erts_debug:df(io).
beam/beam_debug.c:301:erts_debug_disassemble_1() Assertion failed: (((funcinfo[0]) & 0x3F) == ((0x0 << 4) | ((0x2 << 2) | 0x3)))
                          Aborted (core dumped)

It turns out that the assertion is wrong. It should have been
updated in 64ccd8c9b7a7 which made it possible to have stubs for
BIFs in the BEAM code for a module. The faulty assertion was only
found when when 16317f73f79265 added a smoke test of the BEAM
disassembler.
---
 erts/emulator/beam/beam_debug.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8a35ad17c6..c774a70d4c 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -298,8 +298,8 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr);
     hp = HAlloc(p, hsz);
     addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr);
-    ASSERT(is_atom(funcinfo[0]));
-    ASSERT(is_atom(funcinfo[1]));
+    ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL);
+    ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL);
     mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2]));
     hp += 4;
     return TUPLE3(hp, addr, bin, mfa);
-- 
cgit v1.2.3


From 9b44550e9f1b8bf49d447152625f5b3999649034 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 7 Jul 2015 21:27:51 +0200
Subject: Avoid unnecessary copying of data when retrieving corrected monotonic
 time

---
 erts/emulator/beam/erl_time_sup.c | 85 +++++++++++++++++++--------------------
 1 file changed, 41 insertions(+), 44 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 7f8f560681..7327e0b48c 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -124,7 +124,11 @@ typedef struct {
 
 typedef struct {
     ErtsMonotonicCorrectionInstance prev;
-    ErtsMonotonicCorrectionInstance curr;    
+    ErtsMonotonicCorrectionInstance curr;
+} ErtsMonotonicCorrectionInstances;
+
+typedef struct {
+    ErtsMonotonicCorrectionInstances insts;
     ErtsMonotonicDriftData drift;
     ErtsMonotonicTime last_check;
     int short_check_interval;
@@ -272,27 +276,24 @@ static ERTS_INLINE ErtsMonotonicTime
 read_corrected_time(int os_drift_corrected)
 {
     ErtsMonotonicTime os_mtime;
-    ErtsMonotonicCorrectionData cdata;
-    ErtsMonotonicCorrectionInstance *cip;
+    ErtsMonotonicCorrectionInstance ci;
 
     erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
     os_mtime = erts_os_monotonic_time();
 
-    cdata = time_sup.inf.c.parmon.cdata;
-
-    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
-
-    if (os_mtime >= cdata.curr.os_mtime)
-	cip = &cdata.curr;
+    if (os_mtime >= time_sup.inf.c.parmon.cdata.insts.curr.os_mtime)
+	ci = time_sup.inf.c.parmon.cdata.insts.curr;
     else {
-	if (os_mtime < cdata.prev.os_mtime)
+	if (os_mtime < time_sup.inf.c.parmon.cdata.insts.prev.os_mtime)
 	    erl_exit(ERTS_ABORT_EXIT,
 		     "OS monotonic time stepped backwards\n");
-	cip = &cdata.prev;
+	ci = time_sup.inf.c.parmon.cdata.insts.prev;
     }
 
-    return calc_corrected_erl_mtime(os_mtime, cip, NULL,
+    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
+
+    return calc_corrected_erl_mtime(os_mtime, &ci, NULL,
 				    os_drift_corrected);
 }
 
@@ -360,9 +361,8 @@ check_time_correction(void *vesdp)
 {
     int init_drift_adj = !vesdp;
     ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
-    ErtsMonotonicCorrectionData cdata;
     ErtsMonotonicCorrection new_correction;
-    ErtsMonotonicCorrectionInstance *cip;
+    ErtsMonotonicCorrectionInstance ci;
     ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime,
 	erl_stime, time_offset, timeout_pos;
     Uint timeout;
@@ -373,16 +373,15 @@ check_time_correction(void *vesdp)
 
     erts_os_times(&os_mtime, &os_stime);
 
-    cdata = time_sup.inf.c.parmon.cdata;
+    ci = time_sup.inf.c.parmon.cdata.insts.curr;
 
     erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
 
-    if (os_mtime < cdata.curr.os_mtime)
+    if (os_mtime < ci.os_mtime)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "OS monotonic time stepped backwards\n");
-    cip = &cdata.curr;
 
-    erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, &mdiff,
+    erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, &mdiff,
 					 os_drift_corrected);
     time_offset = get_time_offset();
     erl_stime = erl_mtime + time_offset;
@@ -397,7 +396,7 @@ check_time_correction(void *vesdp)
 	    time_sup.inf.c.shadow_offset = 0;
     }
 
-    new_correction = cip->correction;
+    new_correction = ci.correction;
 
     if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE
 	&& (sdiff < -2*time_sup.r.o.adj.small_diff
@@ -408,7 +407,7 @@ check_time_correction(void *vesdp)
 	set_time_offset(time_offset);
 	schedule_send_time_offset_changed_notifications(time_offset);
 	begin_short_intervals = 1;
-	if (cdata.curr.correction.error != 0) {
+	if (ci.correction.error != 0) {
 	    set_new_correction = 1;
 	    new_correction.error = 0;
 	}
@@ -425,12 +424,12 @@ check_time_correction(void *vesdp)
 	time_sup.inf.c.shadow_offset -= sdiff;
 	sdiff = 0;
 	begin_short_intervals = 1;
-	if (cdata.curr.correction.error != 0) {
+	if (ci.correction.error != 0) {
 	    set_new_correction = 1;
 	    new_correction.error = 0;
 	}
     }
-    else if (cdata.curr.correction.error == 0) {
+    else if (ci.correction.error == 0) {
 	if (sdiff < -time_sup.r.o.adj.small_diff) {
 	    set_new_correction = 1;
 	    if (sdiff < -time_sup.r.o.adj.large_diff)
@@ -446,9 +445,9 @@ check_time_correction(void *vesdp)
 		new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ;
 	}
     }
-    else if (cdata.curr.correction.error > 0) {
+    else if (ci.correction.error > 0) {
 	if (sdiff < 0) {
-	    if (cdata.curr.correction.error != ERTS_TCORR_ERR_LARGE_ADJ
+	    if (ci.correction.error != ERTS_TCORR_ERR_LARGE_ADJ
 		&& sdiff < -time_sup.r.o.adj.large_diff) {
 		new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ;
 		set_new_correction = 1;
@@ -466,9 +465,9 @@ check_time_correction(void *vesdp)
 	    new_correction.error = 0;
 	}
     }
-    else /* if (cdata.curr.correction.error < 0) */ { 
+    else /* if (ci.correction.error < 0) */ { 
 	if (0 < sdiff) {
-	    if (cdata.curr.correction.error != -ERTS_TCORR_ERR_LARGE_ADJ
+	    if (ci.correction.error != -ERTS_TCORR_ERR_LARGE_ADJ
 		&& time_sup.r.o.adj.large_diff < sdiff) {
 		new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ;
 		set_new_correction = 1;
@@ -631,8 +630,8 @@ check_time_correction(void *vesdp)
 #ifdef ERTS_TIME_CORRECTION_PRINT
     print_correction(set_new_correction,
 		     sdiff,
-		     cip->correction.error,
-		     cip->correction.drift,
+		     ci.correction.error,
+		     ci.correction.drift,
 		     new_correction.error,
 		     new_correction.drift,
 		     timeout);
@@ -644,7 +643,7 @@ check_time_correction(void *vesdp)
 	os_mtime = erts_os_monotonic_time();
 
 	/* Save previous correction instance */
-	time_sup.inf.c.parmon.cdata.prev = *cip;
+	time_sup.inf.c.parmon.cdata.insts.prev = ci;
 
 	/*
 	 * Current correction instance begin when
@@ -657,15 +656,15 @@ check_time_correction(void *vesdp)
 	 * next OS monotonic time using previous
 	 * correction.
 	 */
-	erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, NULL,
+	erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, NULL,
 					     os_drift_corrected);
 
 	/*
 	 * Save new current correction instance.
 	 */
-	time_sup.inf.c.parmon.cdata.curr.erl_mtime = erl_mtime;
-	time_sup.inf.c.parmon.cdata.curr.os_mtime = os_mtime;
-	time_sup.inf.c.parmon.cdata.curr.correction = new_correction;
+	time_sup.inf.c.parmon.cdata.insts.curr.erl_mtime = erl_mtime;
+	time_sup.inf.c.parmon.cdata.insts.curr.os_mtime = os_mtime;
+	time_sup.inf.c.parmon.cdata.insts.curr.correction = new_correction;
 
 	erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx);
     }
@@ -784,24 +783,22 @@ static ErtsMonotonicTime
 finalize_corrected_time_offset(ErtsSystemTime *stimep)
 {
     ErtsMonotonicTime os_mtime;
-    ErtsMonotonicCorrectionData cdata;
-    ErtsMonotonicCorrectionInstance *cip;
+    ErtsMonotonicCorrectionInstance ci;
     int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time;
 
     erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
     erts_os_times(&os_mtime, stimep);
 
-    cdata = time_sup.inf.c.parmon.cdata;
+    ci = time_sup.inf.c.parmon.cdata.insts.curr;
 
     erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
 
-    if (os_mtime < cdata.curr.os_mtime)
+    if (os_mtime < ci.os_mtime)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "OS monotonic time stepped backwards\n");
-    cip = &cdata.curr;
 
-    return calc_corrected_erl_mtime(os_mtime, cip, NULL,
+    return calc_corrected_erl_mtime(os_mtime, &ci, NULL,
 				    os_drift_corrected);
 }
 
@@ -1128,13 +1125,13 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     
 	cdatap->drift.intervals[0].time.sys = time_sup.inf.c.sinit;
 	cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit;
-	cdatap->curr.correction.drift = 0;
-	cdatap->curr.correction.error = 0;
-	cdatap->curr.erl_mtime = ERTS_MONOTONIC_BEGIN;
-	cdatap->curr.os_mtime = time_sup.inf.c.minit;
+	cdatap->insts.curr.correction.drift = 0;
+	cdatap->insts.curr.correction.error = 0;
+	cdatap->insts.curr.erl_mtime = ERTS_MONOTONIC_BEGIN;
+	cdatap->insts.curr.os_mtime = time_sup.inf.c.minit;
 	cdatap->last_check = time_sup.inf.c.minit;
 	cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER;
-	cdatap->prev = cdatap->curr;
+	cdatap->insts.prev = cdatap->insts.curr;
 
 	if (!time_sup.r.o.os_corrected_monotonic_time)
 	    time_sup.r.o.get_time = get_corrected_time;
-- 
cgit v1.2.3


From a782ed0ca14952dd83560ad7c5a43888bef19cdb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 8 Jul 2015 20:05:20 +0200
Subject: Fix calculation of end time

---
 erts/emulator/beam/erl_time.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 36a3d52264..43e543e035 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -345,8 +345,10 @@ erts_time_unit_conversion(Uint64 value,
 #endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
 
 #define ERTS_MONOTONIC_TIME_END_EXTERNAL				\
-    (ERTS_MONOTONIC_TIME_START_EXTERNAL					\
-     + (ERTS_MONOTONIC_END - ERTS_MONOTONIC_BEGIN))
+    (ERTS_MONOTONIC_TIME_START_EXTERNAL < 0				\
+     ? (ERTS_MONOTONIC_TIME_START_EXTERNAL				\
+	+ (ERTS_MONOTONIC_END - ERTS_MONOTONIC_BEGIN))			\
+     : (ERTS_MONOTONIC_END - ERTS_MONOTONIC_TIME_START_EXTERNAL))
 
 #define ERTS_MSEC_TO_CLKTCKS__(MON) \
     ((MON) * (ERTS_CLKTCK_RESOLUTION/1000))
-- 
cgit v1.2.3


From c431a065ba515d27830f01c852f70940efb3003b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 2 Jul 2015 11:13:32 +0200
Subject: ose: Remove all code related to the OSE port

The OSE port is no longer supported and this commit removed it
and any changes related to it. The things that were general
improvements have been left in the code.
---
 erts/emulator/beam/atom.names      |  7 ----
 erts/emulator/beam/erl_alloc.types | 15 ---------
 erts/emulator/beam/erl_async.c     |  1 -
 erts/emulator/beam/erl_driver.h    | 10 ------
 erts/emulator/beam/erl_port_task.c |  2 +-
 erts/emulator/beam/erl_process.c   | 68 ++++----------------------------------
 erts/emulator/beam/erl_process.h   |  7 ----
 erts/emulator/beam/erl_trace.c     |  6 ----
 erts/emulator/beam/io.c            |  4 ---
 erts/emulator/beam/sys.h           |  4 +--
 10 files changed, 8 insertions(+), 116 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index f9a2f3e33e..6328b3d18f 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -437,13 +437,6 @@ atom orelse
 atom os_pid
 atom os_type
 atom os_version
-atom ose_bg_proc
-atom ose_int_proc
-atom ose_phantom
-atom ose_pri_proc
-atom ose_process_prio
-atom ose_process_type
-atom ose_ti_proc
 atom out
 atom out_exited
 atom out_exiting
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 1d5dec4807..4804fb407d 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -394,21 +394,6 @@ type	SYS_WRITE_BUF	BINARY		SYSTEM		sys_write_buf
 
 +endif
 
-+if ose
-
-type	SYS_READ_BUF	TEMPORARY	SYSTEM		sys_read_buf
-type	FD_TAB		LONG_LIVED	SYSTEM		fd_tab
-type	FD_ENTRY_BUF	STANDARD	SYSTEM		fd_entry_buf
-type	FD_SIG_LIST	SHORT_LIVED	SYSTEM		fd_sig_list
-type    DRV_EV		STANDARD	SYSTEM		driver_event
-type	CS_PROG_PATH	LONG_LIVED	SYSTEM		cs_prog_path
-type	ENVIRONMENT	TEMPORARY	SYSTEM		environment
-type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
-type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
-
-+endif
-
-
 +if win32
 
 type	DRV_DATA_BUF	SYSTEM		SYSTEM		drv_data_buf
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index f071898046..be0bc0cfec 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -167,7 +167,6 @@ async_ready_q(Uint sched_id)
 
 #endif
 
-
 void
 erts_init_async(void)
 {
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index ef2d41e7a2..e71b87803b 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -690,16 +690,6 @@ EXTERN char *driver_dl_error(void);
 EXTERN int erl_drv_putenv(char *key, char *value);
 EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 
-#ifdef __OSE__
-typedef ErlDrvUInt ErlDrvOseEventId;
-EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev);
-EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig, ErlDrvOseEventId handle,
-					   ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig), void *extra);
-EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev);
-EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig,
-                  ErlDrvOseEventId *handle, void **extra);
-#endif
-
 #endif /* !ERL_DRIVER_TYPES_ONLY */
 
 #ifdef WIN32_DYNAMIC_ERL_DRIVER
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 5c38db1cbc..2c09834d19 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1734,7 +1734,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    reds = ERTS_PORT_REDS_INPUT;
 	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
             DTRACE_DRIVER(driver_ready_input, pp);
-	    /* NOTE some windows/ose drivers use ->ready_input
+	    /* NOTE some windows drivers use ->ready_input
 	       for input and output */
 	    (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data,
 					ptp->u.alive.td.io.event);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 135f09c04d..5b13f74dcb 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -58,11 +58,7 @@
 
 #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10)
 
-#ifndef ERTS_SCHED_MIN_SPIN
 #define ERTS_SCHED_SPIN_UNTIL_YIELD 100
-#else
-#define ERTS_SCHED_SPIN_UNTIL_YIELD 1
-#endif
 
 #define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40
 #define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000
@@ -152,12 +148,7 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
-#ifdef __OSE__
-/* Eager check I/O not supported on OSE yet. */
-int erts_eager_check_io = 0;
-#else
 int erts_eager_check_io = 1;
-#endif
 int erts_sched_compact_load;
 int erts_sched_balance_util = 0;
 Uint erts_no_schedulers;
@@ -2521,19 +2512,10 @@ try_set_sys_scheduling(void)
 #endif
 
 static ERTS_INLINE int
-prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking)
+prepare_for_sys_schedule(int non_blocking)
 {
     if (non_blocking && erts_eager_check_io) {
 #ifdef ERTS_SMP
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-	if (esdp->no != 1) {
-	    /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used
-	       then we make sure to wake scheduler 1 */
-	    ErtsRunQueue *rq = ERTS_RUNQ_IX(0);
-	    wake_scheduler(rq);
-	    return 0;
-	}
-#endif
 	return try_set_sys_scheduling();
 #else
 	return 1;
@@ -2543,16 +2525,6 @@ prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking)
 #ifdef ERTS_SMP
 	while (!erts_port_task_have_outstanding_io_tasks()
 	       && try_set_sys_scheduling()) {
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-	    if (esdp->no != 1) {
-		/* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used
-		   then we make sure to wake scheduler 1 */
-		ErtsRunQueue *rq = ERTS_RUNQ_IX(0);
-		clear_sys_scheduling();
-		wake_scheduler(rq);
-		return 0;
-	    }
-#endif
 	    if (!erts_port_task_have_outstanding_io_tasks())
 		return 1;
 	    clear_sys_scheduling();
@@ -2876,8 +2848,6 @@ aux_thread(void *unused)
 		erts_thr_progress_active(NULL, thr_prgr_active = 0);
 	    erts_thr_progress_prepare_wait(NULL);
 
-	    ERTS_SCHED_FAIR_YIELD();
-
 	    flgs = sched_spin_wait(ssi, 0);
 
 	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
@@ -2945,7 +2915,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
      * be waiting in erl_sys_schedule()
      */
 
-    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp, 0)) {
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) {
 
 	sched_waiting(esdp->no, rq);
 
@@ -3010,8 +2980,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 			erts_thr_progress_prepare_wait(esdp);
 		    }
 
-		    ERTS_SCHED_FAIR_YIELD();
-
 		    flgs = sched_spin_wait(ssi, spincount);
 		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
 			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
@@ -3082,13 +3050,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 #ifdef ERTS_DIRTY_SCHEDULERS
 	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
 #endif
-
-#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1
-	ASSERT(esdp->no == 1);
-#endif
 	sched_waiting_sys(esdp->no, rq);
 
-
 	erts_smp_runq_unlock(rq);
 
 	ASSERT(working);
@@ -3158,7 +3121,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		 * Got to check that we still got I/O tasks; otherwise
 		 * we have to continue checking for I/O...
 		 */
-		if (!prepare_for_sys_schedule(esdp, 0)) {
+		if (!prepare_for_sys_schedule(0)) {
 		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
 		    goto tse_wait;
 		}
@@ -3180,7 +3143,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	     * Got to check that we still got I/O tasks; otherwise
 	     * we have to wait in erl_sys_schedule() after all...
 	     */
-	    if (!prepare_for_sys_schedule(esdp, 0)) {
+	    if (!prepare_for_sys_schedule(0)) {
 		/*
 		 * Not allowed to wait in erl_sys_schedule;
 		 * do tse wait instead...
@@ -5301,17 +5264,11 @@ erts_early_init_scheduling(int no_schedulers)
     wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
     wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
 #endif
-#ifndef ERTS_SCHED_MIN_SPIN
     sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
     sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
 			   * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT);
     sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
 				* ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM);
-#else
-    sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
-    sched_busy_wait.tse = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
-    sched_busy_wait.aux_work = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
-#endif
 }
 
 int
@@ -8187,18 +8144,12 @@ erts_start_schedulers(void)
 
 	erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1);
 
-#ifdef __OSE__
-        /* This should be done in the bind strategy */
-	opts.coreNo = (actual+1) % ose_num_cpus();
-#endif
-
 	res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts);
 
 	if (res != 0) {
            break;
 	}
     }
-
     erts_no_schedulers = actual;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -8227,10 +8178,6 @@ erts_start_schedulers(void)
 
     erts_snprintf(opts.name, 16, "aux");
 
-#ifdef __OSE__
-    opts.coreNo = 0;
-#endif /* __OSE__ */
-
     res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
     if (res != 0)
 	erl_exit(1, "Failed to create aux thread\n");
@@ -8250,7 +8197,6 @@ erts_start_schedulers(void)
 		      actual, actual == 1 ? " was" : "s were");
 	erts_send_error_to_logger_nogl(dsbufp);
     }
-
 }
 
 #endif /* ERTS_SMP */
@@ -9391,12 +9337,10 @@ Process *schedule(Process *p, int calls)
 	    int leader_update = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0
 		: erts_thr_progress_update(esdp);
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
-	    if (aux_work | leader_update | ERTS_SCHED_FAIR) {
+	    if (aux_work | leader_update) {
 		erts_smp_runq_unlock(rq);
 		if (leader_update)
 		    erts_thr_progress_leader_update(esdp);
-		else if (ERTS_SCHED_FAIR)
-		  ERTS_SCHED_FAIR_YIELD();
 		if (aux_work)
 		    handle_aux_work(&esdp->aux_work_data, aux_work, 0);
 		erts_smp_runq_lock(rq);
@@ -9472,7 +9416,7 @@ Process *schedule(Process *p, int calls)
 	}
 	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
 		 (fcalls > input_reductions &&
-		  prepare_for_sys_schedule(esdp, !0))) {
+		  prepare_for_sys_schedule(!0))) {
 	    ErtsMonotonicTime current_time;
 	    /*
 	     * Schedule system-level activities.
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 20ffe7ea7c..caac89c087 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -691,13 +691,6 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
 extern ErtsSchedulerData *erts_scheduler_data;
 #endif
 
-#ifdef ERTS_SCHED_FAIR
-#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
-#else
-#define ERTS_SCHED_FAIR 0
-#define ERTS_SCHED_FAIR_YIELD()
-#endif
-
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
 #endif
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index e1b03a057f..3a74d752fe 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3297,8 +3297,6 @@ sys_msg_dispatcher_func(void *unused)
 	    if (erts_thr_progress_update(NULL))
 		erts_thr_progress_leader_update(NULL);
 
-	    ERTS_SCHED_FAIR_YIELD();
-
 #ifdef DEBUG_PRINTOUTS
 	    print_msg_type(smqp);
 #endif
@@ -3453,9 +3451,6 @@ static void
 init_sys_msg_dispatcher(void)
 {
     erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
-#ifdef __OSE__
-    thr_opts.coreNo   = 0;
-#endif
     thr_opts.detached = 1;
     thr_opts.name = "sys_msg_dispatcher";
     init_smq_element_alloc();
@@ -3463,7 +3458,6 @@ init_sys_msg_dispatcher(void)
     sys_message_queue_end = NULL;
     erts_smp_cnd_init(&smq_cnd);
     erts_smp_mtx_init(&smq_mtx, "sys_msg_q");
-
     erts_smp_thr_create(&sys_msg_dispatcher_tid,
 			sys_msg_dispatcher_func,
 			NULL,
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 6b4b90cb06..d754fc634e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -52,9 +52,7 @@
 #include "erl_hl_timer.h"
 
 extern ErlDrvEntry fd_driver_entry;
-#ifndef __OSE__
 extern ErlDrvEntry vanilla_driver_entry;
-#endif
 extern ErlDrvEntry spawn_driver_entry;
 extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */
 
@@ -2794,9 +2792,7 @@ void erts_init_io(int port_tab_size,
     erts_smp_rwmtx_rwlock(&erts_driver_list_lock);
 
     init_driver(&fd_driver, &fd_driver_entry, NULL);
-#ifndef __OSE__
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
-#endif
     init_driver(&spawn_driver, &spawn_driver_entry, NULL);
     erts_init_static_drivers();
     for (dp = driver_tab; *dp != NULL; dp++)
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 14eeb0ee8f..34011147d9 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -74,9 +74,7 @@
 
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
-#elif defined (__OSE__)
-#  include "erl_ose_sys.h"
-#else 
+#else
 #  include "erl_unix_sys.h"
 #ifndef UNIX
 #  define UNIX 1
-- 
cgit v1.2.3


From ed830e036f1bb5c2f1f67224897a31d2fa544c91 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 14 Jul 2015 13:05:39 +0200
Subject: erts: Don't abort when a system process is terminated

---
 erts/emulator/beam/erl_process.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ee1dd36d48..db37243321 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12126,7 +12126,8 @@ erts_do_exit_process(Process* p, Eterm reason)
 #endif
 
     if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC)
-	erl_exit(1, "System process %T terminated: %T\n", p->common.id, reason);
+	erl_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n",
+                 p->common.id, reason);
 
 #ifdef ERTS_SMP
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
-- 
cgit v1.2.3


From 2e61f98dd41ce7328aebc27debd78845afdc0dba Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Fri, 31 Jul 2015 21:15:13 +0200
Subject: erts: fix binary_to_integer boundary case

erlang:binary_to_integer/1 and /2 fail to detect invalid
input consisting of a single + or - sign but nothing else.
For an input like <<"+">> they return 0, while list_to_integer/1
correctly signals a badarg for "+".

Fixed by checking if the input is empty after the initial +/-
sign processing.

Added a test case which fails without this fix but passes with it.

Thanks to "niku" for reporting the issue.
---
 erts/emulator/beam/big.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 044bf6a34e..15bcd44fb9 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2618,6 +2618,9 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes,
 	size--;
     }
 
+    if (size == 0)
+	goto bytebuf_to_integer_1_error;
+
     if (size < SMALL_DIGITS && base <= 10) {
 	/* *
 	 * Take shortcut if we know that all chars are '0' < b < '9' and
-- 
cgit v1.2.3


From 79b504ca9134121a63c049292d8636d3cd0d99b4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 10 Jul 2015 16:56:15 +0200
Subject: Teach smp VM how to deal with crash of a linked trace port

Problem: The sys-msg-dispather crashes the VM when trying to send
exit signals from the links of the terminating trace port.
If try-lock of the linked process fails, a pending exit is
scheduled and erts_scheduler_data() is then called to find
"my" run queue. But sys-msg-dispatcher is not a scheduler
and has no scheduler data, hence SEGV.

Fix: If not a scheduler and we cannot get process locks,
schedule process in its previous run-queue.
---
 erts/emulator/beam/erl_process.c | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b6ad8575cf..26ef96ea74 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11195,10 +11195,14 @@ save_pending_exiter(Process *p)
 {
     ErtsProcList *plp;
     ErtsRunQueue *rq;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    rq = erts_get_runq_current(NULL);
+    if (!esdp)
+        rq = RUNQ_READ_RQ(&p->run_queue);
+    else
+        rq = esdp->run_queue;
 
     plp = proclist_create(p);
 
@@ -11215,6 +11219,7 @@ save_pending_exiter(Process *p)
     else
 #endif
 	wake_scheduler(rq);
+
 }
 
 #endif
@@ -11411,23 +11416,21 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		if (need_locks
 		    && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
 		    /* ... but we havn't got all locks on it ... */
-		    save_pending_exiter(rp);
+		    save_pending_exiter(rp); 
 		    /*
 		     * The pending exit will be discovered when next
 		     * process is scheduled in
 		     */
-		    goto set_pending_exit;
-		}
-		else {
-		    /* ...and we have all locks on it... */
-		    *rp_locks = ERTS_PROC_LOCKS_ALL;
-		    set_proc_exiting(rp,
-				     state,
-				     (is_immed(rsn)
-				      ? rsn
-				      : copy_object(rsn, rp)),
-				     NULL);
+                    goto set_pending_exit;
 		}
+                /* ...and we have all locks on it... */
+                *rp_locks = ERTS_PROC_LOCKS_ALL;
+                set_proc_exiting(rp,
+                                 state,
+                                 (is_immed(rsn)
+                                  ? rsn
+                                  : copy_object(rsn, rp)),
+                                 NULL);
 	    }
 	    else { /* Process running... */
 
-- 
cgit v1.2.3


From 1497898b7d680de86d64bdec133003288adee820 Mon Sep 17 00:00:00 2001
From: Daniel Goertzen <daniel.goertzen@gmail.com>
Date: Tue, 11 Aug 2015 12:02:50 -0500
Subject: fix unused parameter warning in enif_make_pid

---
 erts/emulator/beam/erl_nif_api_funcs.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index f93152c921..2f2180e1aa 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -543,7 +543,7 @@ static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env,
 
 #ifndef enif_make_pid
 
-#  define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid))
+#  define enif_make_pid(ENV, PID) ((void)(ENV),(const ERL_NIF_TERM)((PID)->pid))
 
 #if SIZEOF_LONG == 8
 #  define enif_get_int64 enif_get_long
-- 
cgit v1.2.3


From 02380778fd2a9d6af85865a89ef0747351cc0f88 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 13 Aug 2015 14:52:20 +0200
Subject: erts: Make sure to unlock status lock when setting process prio

---
 erts/emulator/beam/erl_process.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7b3d12ce09..98f01bbce9 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9177,6 +9177,10 @@ erts_set_process_priority(Process *p, Eterm value)
 
 	    a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	} while (a != e);
+
+        if (slocked)
+            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+
     }
 
     switch (oprio) {
-- 
cgit v1.2.3


From f76ba62c7869bb7bbda27446d0a9f7214062db4f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 25 Aug 2015 12:40:37 +0200
Subject: erts: bool is a reserved word, use boolean instead

---
 erts/emulator/beam/beam_debug.c | 6 +++---
 erts/emulator/beam/bif.c        | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index c774a70d4c..90985e4f53 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -79,7 +79,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
 {
     Process* p = BIF_P;
     Eterm MFA = BIF_ARG_1;
-    Eterm bool = BIF_ARG_2;
+    Eterm boolean = BIF_ARG_2;
     Eterm* tp;
     Eterm mfa[3];
     int i;
@@ -87,7 +87,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
     Eterm res;
     BpFunctions f;
 
-    if (bool != am_true && bool != am_false)
+    if (boolean != am_true && boolean != am_false)
 	goto error;
 
     if (is_not_tuple(MFA)) {
@@ -124,7 +124,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
     erts_smp_thr_progress_block();
 
     erts_bp_match_functions(&f, mfa, specified);
-    if (bool == am_true) {
+    if (boolean == am_true) {
 	erts_set_debug_break(&f);
 	erts_install_breakpoints(&f);
 	erts_commit_staged_bp();
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 4e3a1cef69..0bd46a2dae 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -616,7 +616,7 @@ erts_queue_monitor_message(Process *p,
 }
 
 static BIF_RETTYPE
-local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool)
+local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean)
 {
     BIF_RETTYPE ret;
     Process *rp;
@@ -634,7 +634,7 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool)
     if (!rp) {
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	p_locks &= ~ERTS_PROC_LOCK_LINK;
-	if (bool)
+	if (boolean)
 	    ret = am_false;
 	else
 	    erts_queue_monitor_message(p, &p_locks,
@@ -643,7 +643,7 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool)
     else {
 	ASSERT(rp != p);
 
-	if (bool)
+	if (boolean)
 	    ret = am_true;
 
 	erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL);
-- 
cgit v1.2.3


From 5a02ed2f3505f5ed3282c6b43963e19e63e49905 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 25 Aug 2015 15:47:29 +0200
Subject: Fix ethread events with timeout

Lots of pthread platforms unnecessarily falled back on the pipe/select
solution. This since we tried to use the same monotonic clock source
for pthread_cond_timedwait() as used by OS monotonic time. This has
been fixed on most platforms by using another clock source.

Darwin can however not use pthread_cond_timedwait() with monotonic
clock source and has to use the pipe/select solution. On darwin we
now use select with _DARWIN_UNLIMITED_SELECT in order to be able to
handle a large amount of file descriptors.
---
 erts/emulator/beam/erl_process.c |  7 ++++++-
 erts/emulator/beam/erl_threads.h | 10 ++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ee1dd36d48..58698173be 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7938,11 +7938,16 @@ sched_thread_func(void *vesdp)
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
     Uint no = esdp->no;
+#ifdef ERTS_SMP
+    erts_tse_t *tse;
+#endif
 
     erts_sched_init_time_sup(esdp);
 
 #ifdef ERTS_SMP
-    ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch();
+    tse = erts_tse_fetch();
+    erts_tse_prepare_timed(tse);
+    ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = tse;
     callbacks.arg = (void *) esdp->ssi;
     callbacks.wakeup = thr_prgr_wakeup;
     callbacks.prepare_wait = thr_prgr_prep_wait;
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 5347979372..34f91e2ec8 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -649,6 +649,7 @@ ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
 ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
 ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
 ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
+ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep);
 ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
 ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
 ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
@@ -3461,6 +3462,15 @@ ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
 #endif
 }
 
+ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep)
+{
+#ifdef USE_THREADS
+    int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event);
+    if (res != 0)
+	erts_thr_fatal_error(res, "prepare timed");
+#endif
+}
+
 ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-- 
cgit v1.2.3


From 90e0bad70affc23228e3d11d1131aba615895dd5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 28 Aug 2015 18:08:04 +0200
Subject: erts: Fix hipe bug for maps:merge/2

Add forgotten HIPE_WRAPPER_BIF_DISABLE_GC which
could lead to stack-heap overrun if unlucky with the
yielding during maps:merge when called by native hipe code.
---
 erts/emulator/beam/erl_map.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index a91e36e3c5..ff2a355309 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -32,7 +32,9 @@
 #include "global.h"
 #include "erl_process.h"
 #include "error.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "erl_binary.h"
 
 #include "erl_map.h"
@@ -952,8 +954,11 @@ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
     BIF_P->fvalue = BIF_ARG_1;
     BIF_ERROR(BIF_P, BADMAP);
 }
+
 /* maps:merge/2 */
 
+HIPE_WRAPPER_BIF_DISABLE_GC(maps_merge, 2)
+
 BIF_RETTYPE maps_merge_2(BIF_ALIST_2) {
     if (is_flatmap(BIF_ARG_1)) {
 	if (is_flatmap(BIF_ARG_2)) {
-- 
cgit v1.2.3


From a22b5ba19193e3f39129fadd20d375f6cc3f8529 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 1 Sep 2015 19:53:40 +0200
Subject: erts: Fix bug when tracing with 'process_dump'

If the process stack contained a match state
the print function would crash the vm as it was not
recognized by tag_val_def().

Add new MATCHSTATE_DEF returned by tag_val_def().
All other callers either ignore it or has a default
clause to handle invalid terms.
---
 erts/emulator/beam/erl_debug.c       | 3 +++
 erts/emulator/beam/erl_printf_term.c | 9 ++++-----
 erts/emulator/beam/erl_term.c        | 1 +
 erts/emulator/beam/erl_term.h        | 3 +++
 erts/emulator/beam/utils.c           | 9 ++++-----
 5 files changed, 15 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 50bdc79506..b6c131a43e 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -188,6 +188,9 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj)
     case BINARY_DEF:
 	erts_print(to, to_arg, "#Bin");
 	break;
+    case MATCHSTATE_DEF:
+        erts_print(to, to_arg, "#Matchstate");
+        break;
     default:
 	erts_print(to, to_arg, "unknown object %x", obj);
     }
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index d18760dc43..ba9a174fdf 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -441,11 +441,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 	    PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd);
 	}
 	    break;
-	case BINARY_DEF:
-	    if (header_is_bin_matchstate(*boxed_val(wobj))) {
-		PRINT_STRING(res, fn, arg, "#MatchState");
-	    }
-	    else {
+	case BINARY_DEF: {
 		ProcBin* pb = (ProcBin *) binary_val(wobj);
 		if (pb->size == 1)
 		    PRINT_STRING(res, fn, arg, "<<1 byte>>");
@@ -519,6 +515,9 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
 		}
 	    }
 	    break;
+        case MATCHSTATE_DEF:
+            PRINT_STRING(res, fn, arg, "#MatchState");
+            break;
 	default:
 	    PRINT_STRING(res, fn, arg, "<unknown:");
 	    PRINT_POINTER(res, fn, arg, wobj);
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 28cbe7004f..07818472ec 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -90,6 +90,7 @@ unsigned tag_val_def(Wterm x)
 	    case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
 	    case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
 	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):	return MAP_DEF;
+	    case (_TAG_HEADER_BIN_MATCHSTATE >> _TAG_PRIMARY_SIZE): return MATCHSTATE_DEF;
 	  }
 	  break;
       }
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 37014ccf94..1b6bb27395 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1095,6 +1095,9 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define FLOAT_DEF		0xe
 #define BIG_DEF			0xf
 #define SMALL_DEF		0x10
+#define MATCHSTATE_DEF          0x11   /* not a "real" term */
+
+#define FIRST_VACANT_TAG_DEF    0x12
 
 #if ET_DEBUG
 extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f810fca9a4..a8bbdb9354 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -788,11 +788,10 @@ Uint32 make_hash(Eterm term_arg)
     Eterm hash = 0;
     unsigned op;
 
-    /* Must not collide with the real tag_val_def's: */
-#define MAKE_HASH_TUPLE_OP 0x11
-#define MAKE_HASH_TERM_ARRAY_OP 0x12
-#define MAKE_HASH_CDR_PRE_OP 0x13
-#define	MAKE_HASH_CDR_POST_OP 0x14
+#define MAKE_HASH_TUPLE_OP      (FIRST_VACANT_TAG_DEF)
+#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1)
+#define MAKE_HASH_CDR_PRE_OP    (FIRST_VACANT_TAG_DEF+2)
+#define MAKE_HASH_CDR_POST_OP   (FIRST_VACANT_TAG_DEF+3)
 
     /* 
     ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit 
-- 
cgit v1.2.3


From e4dedc6cffb5025161d9a6295523b215e5a165c7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 9 Sep 2015 17:41:41 +0200
Subject: Add configure switch --disable-saved-compile-time

---
 erts/emulator/beam/break.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 64c8bc5e58..4ce9d24479 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -536,7 +536,9 @@ do_break(void)
 	    erts_printf("Erlang (%s) emulator version "
 		       ERLANG_VERSION "\n",
 		       EMULATOR);
+#if ERTS_SAVED_COMPILE_TIME
 	    erts_printf("Compiled on " ERLANG_COMPILE_DATE "\n");
+#endif
 	    return;
 	case 'd':
 	    distribution_info(ERTS_PRINT_STDOUT, NULL);
@@ -774,7 +776,9 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     }
     erts_fdprintf(fd, "System version: ");
     erts_print_system_version(fd, NULL, NULL);
+#if ERTS_SAVED_COMPILE_TIME
     erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE);
+#endif
 
     erts_fdprintf(fd, "Taints: ");
     erts_print_nif_taints(fd, NULL);
-- 
cgit v1.2.3


From 34e1b675f95d0837823b794f3440300806f7ef88 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 11 Sep 2015 16:40:56 +0200
Subject: erts: Cleanup main carrier creation

and remove that magic "-40" from default mmbcs for ll_alloc
---
 erts/emulator/beam/erl_alloc.c      |  4 ++--
 erts/emulator/beam/erl_alloc_util.c | 33 ++++++++++++++++++++++++---------
 2 files changed, 26 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d68f22d573..70d5357e27 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -272,9 +272,9 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.util.name_prefix	= "ll_";
     ip->init.util.alloc_no	= ERTS_ALC_A_LONG_LIVED;
 #ifndef SMALL_MEMORY
-    ip->init.util.mmbcs 	= 2*1024*1024 - 40; /* Main carrier size */
+    ip->init.util.mmbcs 	= 2*1024*1024; /* Main carrier size */
 #else
-    ip->init.util.mmbcs 	= 1*1024*1024 - 40; /* Main carrier size */
+    ip->init.util.mmbcs 	= 1*1024*1024; /* Main carrier size */
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_LONG_LIVED;
     ip->init.util.asbcst	= 0;
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index db4c30b9eb..444d7055c8 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3534,7 +3534,21 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	return NULL;
     }
 
-    blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
+    if (flags & CFLG_MAIN_CARRIER) {
+        ASSERT(flags & CFLG_MBC);
+        ASSERT(flags & CFLG_NO_CPOOL);
+        ASSERT(umem_sz == allctr->main_carrier_size);
+        ERTS_UNDEF(blk_sz, 0);
+
+        if (allctr->main_carrier_size < allctr->min_mbc_size)
+            allctr->main_carrier_size = allctr->min_mbc_size;
+        crr_sz = bcrr_sz = allctr->main_carrier_size;
+    }
+    else {
+        ERTS_UNDEF(bcrr_sz, 0);
+	ERTS_UNDEF(crr_sz, 0);
+        blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
+    }
 
 #ifdef ERTS_SMP
     allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON;
@@ -3580,10 +3594,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	mseg_flags = ERTS_MSEG_FLG_NONE;
     }
     else {
-	crr_sz = (*allctr->get_next_mbc_size)(allctr);
-	if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz)
-	    crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz;
-	mseg_flags = ERTS_MSEG_FLG_2POW;
+        if (!(flags & CFLG_MAIN_CARRIER)) {
+            crr_sz = (*allctr->get_next_mbc_size)(allctr);
+            if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz)
+                crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz;
+        }
+        mseg_flags = ERTS_MSEG_FLG_2POW;
     }
 
     crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags);
@@ -3618,11 +3634,10 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
     if (flags & CFLG_SBC) {
 	bcrr_sz = blk_sz + SBC_HEADER_SIZE;
     }
-    else {
+    else if (!(flags & CFLG_MAIN_CARRIER)) {
 	bcrr_sz = MBC_HEADER_SIZE(allctr) + blk_sz;
-	if (!(flags & CFLG_MAIN_CARRIER)
-	    && bcrr_sz < allctr->smallest_mbc_size)
-	    bcrr_sz = allctr->smallest_mbc_size;
+	if (bcrr_sz < allctr->smallest_mbc_size)
+            bcrr_sz = allctr->smallest_mbc_size;
     }
 
     crr_sz = (flags & CFLG_FORCE_SIZE
-- 
cgit v1.2.3


From 1fd4809777931a4dc166fd9f1b552317a385cd62 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 7 Sep 2015 19:58:47 +0200
Subject: erts: Fix strangeness in treatment of MSEG_ALIGN_BITS

And remove old case of using only page alignment (12 bits).
---
 erts/emulator/beam/erl_alloc_util.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 792f8c63ac..f314eab4c2 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -220,7 +220,7 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
 #if ERTS_HAVE_MSEG_SUPER_ALIGNED \
     || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC)
-#  ifndef MSEG_ALIGN_BITS
+#  ifdef MSEG_ALIGN_BITS
 #    define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS
 #  else
 #    define ERTS_SUPER_ALIGN_BITS 18
-- 
cgit v1.2.3


From ada8342bedf8d4b84d4c3c10fcfc7919e532fd8c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 4 Sep 2015 18:45:06 +0200
Subject: erts: Add new allocator LITERAL

---
 erts/emulator/beam/beam_bif_load.c |  6 ++++++
 erts/emulator/beam/beam_load.c     | 24 +++++++++++------------
 erts/emulator/beam/erl_alloc.c     | 39 ++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_alloc.types |  3 +++
 erts/emulator/beam/erl_bif_info.c  |  2 +-
 5 files changed, 61 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 11508a1b39..68689b8e7f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -684,6 +684,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	erts_cleanup_funs_on_purge(code, end);
 	beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length,
 			    erts_active_code_ix());
+        if (code[MI_LITERALS_START]) {
+            erts_free(ERTS_ALC_T_LITERAL, (void *) code[MI_LITERALS_START]);
+        }
 	erts_free(ERTS_ALC_T_CODE, (void *) code);
 	modp->curr.code = NULL;
 	modp->curr.code_length = 0;
@@ -1019,6 +1022,9 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 	    beam_catches_delmod(modp->old.catches, code, modp->old.code_length,
 				code_ix);
 	    decrement_refc(code);
+            if (code[MI_LITERALS_START]) {
+                erts_free(ERTS_ALC_T_LITERAL, (void *) code[MI_LITERALS_START]);
+            }
 	    erts_free(ERTS_ALC_T_CODE, (void *) code);
 	    modp->old.code = NULL;
 	    modp->old.code_length = 0;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index a9d47eb7b0..f0b4be4c3d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -654,6 +654,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
     stp->code[MI_COMPILE_PTR] = 0;
     stp->code[MI_COMPILE_SIZE] = 0;
     stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
+    stp->code[MI_LITERALS_START] = 0;
     stp->code[MI_MD5_PTR] = 0;
 
     /*
@@ -921,6 +922,9 @@ loader_state_dtor(Binary* magic)
 	stp->bin = 0;
     }
     if (stp->code != 0) {
+        if (stp->code[MI_LITERALS_START]) {
+            erts_free(ERTS_ALC_T_LITERAL, (void*) stp->code[MI_LITERALS_START]);
+        }
 	erts_free(ERTS_ALC_T_CODE, stp->code);
 	stp->code = 0;
     }
@@ -4348,7 +4352,6 @@ static int
 freeze_code(LoaderState* stp)
 {
     BeamInstr* code = stp->code;
-    Uint *literal_end = NULL;
     int i;
     byte* str_table;
     unsigned strtab_size = stp->chunks[STR_CHUNK].size;
@@ -4378,7 +4381,6 @@ freeze_code(LoaderState* stp)
 	    sizeof(Eterm) + (stp->current_li+1) * stp->loc_size;
     }
     size = (stp->ci * sizeof(BeamInstr)) +
-	(stp->total_literal_size * sizeof(Eterm)) +
 	strtab_size + attr_size + compile_size + MD5_SIZE + line_size;
 
     /*
@@ -4406,10 +4408,9 @@ freeze_code(LoaderState* stp)
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
 
-    literal_end = (Uint *) (code+stp->ci);
     /*
-     * Place the literal heap directly after the code and fix up all
-     * instructions that refer to it.
+     * Place the literals in their own allocated heap (for fast range check)
+     * and fix up all instructions that refer to it.
      */
     {
 	Uint* ptr;
@@ -4420,7 +4421,8 @@ freeze_code(LoaderState* stp)
 
         ERTS_INIT_OFF_HEAP(&code_off_heap);
 
-	low = (Uint *) (code+stp->ci);
+        low = erts_alloc(ERTS_ALC_T_LITERAL,
+                         stp->total_literal_size*sizeof(Eterm));
 	high = low + stp->total_literal_size;
 	code[MI_LITERALS_START] = (BeamInstr) low;
 	code[MI_LITERALS_END] = (BeamInstr) high;
@@ -4443,7 +4445,6 @@ freeze_code(LoaderState* stp)
 	    op_ptr[0] = lit->term;
 	    lp = lp->next;
 	}
-	literal_end += stp->total_literal_size;
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
 
@@ -4452,9 +4453,9 @@ freeze_code(LoaderState* stp)
      */
     if (stp->line_instr == 0) {
 	code[MI_LINE_TABLE] = (BeamInstr) 0;
-	str_table = (byte *) literal_end;
+	str_table = (byte *) (code+stp->ci);
     } else {
-	Eterm* line_tab = (Eterm *) literal_end;
+	Eterm* line_tab = (Eterm *) (code+stp->ci);
 	Eterm* p;
 	int ftab_size = stp->num_functions;
 	int num_instrs = stp->current_li;
@@ -4486,7 +4487,7 @@ freeze_code(LoaderState* stp)
 		*locp++ = (Uint16) stp->line_instr[i].loc;
 	    }
 	    *locp++ = LINE_INVALID_LOCATION;
-	    str_table = (byte *) locp;
+            str_table = (byte *) locp;
 	} else {
 	    Uint32* locp = (Uint32 *) p;
 	    ASSERT(stp->loc_size == 4);
@@ -4494,9 +4495,8 @@ freeze_code(LoaderState* stp)
 		*locp++ = stp->line_instr[i].loc;
 	    }
 	    *locp++ = LINE_INVALID_LOCATION;
-	    str_table = (byte *) locp;
+            str_table = (byte *) locp;
 	}
-
 	CHKBLK(ERTS_ALC_T_CODE,code);
     }
 
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 70d5357e27..d83a2930d6 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -130,6 +130,7 @@ static ErtsAllocatorState_t binary_alloc_state;
 static ErtsAllocatorState_t ets_alloc_state;
 static ErtsAllocatorState_t driver_alloc_state;
 static ErtsAllocatorState_t fix_alloc_state;
+static ErtsAllocatorState_t literal_alloc_state;
 
 typedef struct {
     erts_smp_atomic32_t refc;
@@ -211,6 +212,7 @@ typedef struct {
     struct au_init ets_alloc;
     struct au_init driver_alloc;
     struct au_init fix_alloc;
+    struct au_init literal_alloc;
 } erts_alc_hndl_args_init_t;
 
 #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
@@ -284,6 +286,32 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL_LL_ALLOC;
 }
 
+static void
+set_default_literal_alloc_opts(struct au_init *ip)
+{
+    SET_DEFAULT_ALLOC_OPTS(ip);
+    ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
+    ip->thr_spec		= 0;
+    ip->atype			= BESTFIT;
+    ip->init.bf.ao		= 1;
+    ip->init.util.ramv		= 0;
+    ip->init.util.mmsbc		= 0;
+    ip->init.util.sbct		= ~((UWord) 0);
+    ip->init.util.name_prefix	= "literal_";
+    ip->init.util.alloc_no	= ERTS_ALC_A_LITERAL;
+#ifndef SMALL_MEMORY
+    ip->init.util.mmbcs 	= 2*1024*1024; /* Main carrier size */
+#else
+    ip->init.util.mmbcs 	= 1*1024*1024; /* Main carrier size */
+#endif
+    ip->init.util.ts 		= ERTS_ALC_MTA_LITERAL;
+    ip->init.util.asbcst	= 0;
+    ip->init.util.rsbcst	= 0;
+    ip->init.util.rsbcmt	= 0;
+    ip->init.util.rmbcmt	= 0;
+    ip->init.util.acul		= 0;
+}
+
 static void
 set_default_temp_alloc_opts(struct au_init *ip)
 {
@@ -577,6 +605,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_default_driver_alloc_opts(&init.driver_alloc);
     set_default_fix_alloc_opts(&init.fix_alloc,
 			       fix_type_sizes);
+    set_default_literal_alloc_opts(&init.literal_alloc);
 
     if (argc && argv)
 	handle_args(argc, argv, &init);
@@ -604,6 +633,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     init.ets_alloc.thr_spec = 0;
     init.driver_alloc.thr_spec = 0;
     init.fix_alloc.thr_spec = 0;
+    init.literal_alloc.thr_spec = 0;
 #endif
 
     /* Make adjustments for carrier migration support */
@@ -616,6 +646,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_carrier_migration_support(&init.ets_alloc);
     adjust_carrier_migration_support(&init.driver_alloc);
     adjust_carrier_migration_support(&init.fix_alloc);
+    adjust_carrier_migration_support(&init.literal_alloc);
 
     if (init.erts_alloc_config) {
 	/* Adjust flags that erts_alloc_config won't like */
@@ -630,6 +661,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.ets_alloc.thr_spec = 0;
 	init.driver_alloc.thr_spec = 0;
 	init.fix_alloc.thr_spec = 0;
+        init.literal_alloc.thr_spec = 0;
 
 	/* No carrier migration */
 	init.temp_alloc.init.util.acul = 0;
@@ -641,6 +673,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.ets_alloc.init.util.acul = 0;
 	init.driver_alloc.init.util.acul = 0;
 	init.fix_alloc.init.util.acul = 0;
+        init.literal_alloc.init.util.acul = 0;
     }
 
 #ifdef ERTS_SMP
@@ -657,6 +690,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_tpref(&init.ets_alloc, erts_no_schedulers);
     adjust_tpref(&init.driver_alloc, erts_no_schedulers);
     adjust_tpref(&init.fix_alloc, erts_no_schedulers);
+    adjust_tpref(&init.literal_alloc, erts_no_schedulers);
 
 #else
     /* No thread specific if not smp */
@@ -675,6 +709,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     refuse_af_strategy(&init.ets_alloc);
     refuse_af_strategy(&init.driver_alloc);
     refuse_af_strategy(&init.fix_alloc);
+    refuse_af_strategy(&init.literal_alloc);
 
 #ifdef ERTS_SMP 
     if (!init.temp_alloc.thr_spec)
@@ -718,6 +753,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu);
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	if (!erts_allctrs[i].alloc)
@@ -770,6 +806,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
 		       &init.fix_alloc,
 		       &fix_alloc_state);
+    start_au_allocator(ERTS_ALC_A_LITERAL,
+                       &init.literal_alloc,
+                       &literal_alloc_state);
 
     erts_mtrace_install_wrapper_functions();
     extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 4804fb407d..7d519c1be4 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -86,6 +86,7 @@ allocator	LONG_LIVED	true		ll_alloc
 allocator	EHEAP		true		eheap_alloc
 allocator	ETS		true		ets_alloc
 allocator	FIXED_SIZE	true		fix_alloc
+allocator	LITERAL 	true		literal_alloc
 
 +else # Non smp build
 
@@ -96,6 +97,7 @@ allocator	LONG_LIVED	false		ll_alloc
 allocator	EHEAP		false		eheap_alloc
 allocator	ETS		false		ets_alloc
 allocator	FIXED_SIZE	false		fix_alloc
+allocator	LITERAL 	false		literal_alloc
 
 +endif
 
@@ -343,6 +345,7 @@ type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
 type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
 type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
 type	CODE		LONG_LIVED	CODE		code
+type	LITERAL		LITERAL 	CODE		literal
 type	DB_HEIR_DATA	STANDARD	ETS		db_heir_data
 type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 9a132ee007..79ccf8db64 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4321,7 +4321,7 @@ static void os_info_init(void)
     os_flavor(buf, 1024);
     flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1);
     erts_free(ERTS_ALC_T_TMP, (void *) buf);
-    hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm));
+    hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm));
     os_type_tuple = TUPLE2(hp, type, flav);
     hp += 3;
     os_version(&major, &minor, &build);
-- 
cgit v1.2.3


From 3f75988488197f7cf3846f9b7705376779728f4c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 28 Sep 2015 15:42:22 +0200
Subject: erts: Remove assertion that ptab is empty

Another process may already have been placed in this slot
since the free och the process struct can be scheduled for
later.
---
 erts/emulator/beam/erl_process_lock.h | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 788348e613..a64c993e8f 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -854,9 +854,6 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
 #endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
-	ASSERT(ERTS_AINT_NULL
-	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
-					    internal_pid_index(p->common.id)));
 	erts_free_proc(p);
     }
 }
@@ -872,9 +869,6 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 #endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
-	ASSERT(ERTS_AINT_NULL
-	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
-					    internal_pid_index(p->common.id)));
 	erts_free_proc(p);
     }
 }
-- 
cgit v1.2.3


From 5e229f3d7591b0df7d6d475065e1dc2d5273a148 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Sep 2015 15:28:46 +0200
Subject: erts: Fix resurrection of carriers from dc_list

Problem #1: Seems the dc_list check did end up as dead code by mistake.

Solution: goto check_dc_list

Problem #2: crr->cpool.max_size was set to zero for all carriers in
dc_list, which meant no carriers were ever resurrected by cpool_fetch.

Solution: Do not use callback 'largest_fblk_in_mbc' to set max_size
as it will always return 0 (due to problem #3).

Problem #3: Resurrected carriers were broken as their one free block
was not linked.

Solution: Link free block when fetching carrier from dc_list.
---
 erts/emulator/beam/erl_alloc_util.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 236ee35d18..3861196cfc 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3149,7 +3149,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	cpool_entrance = sentinel;
 	cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev));
 	if (cpdp == sentinel)
-	    return NULL;
+	    goto check_dc_list;
     }
 
     has_passed_sentinel = 0;
@@ -3160,18 +3160,18 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	    if (cpool_entrance == sentinel) {
 		cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
 		if (cpdp == sentinel)
-		    return NULL;
+		    break;
 	    }
 	    i = 0; /* Last one to inspect */
 	}
 	else if (cpdp == sentinel) {
 	    if (has_passed_sentinel) {
 		/* We been here before. cpool_entrance must have been removed */
-		return NULL;
+		break;
 	    }
 	    cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
 	    if (cpdp == sentinel)
-		return NULL;
+                break;
 	    has_passed_sentinel = 1;
 	}
 	crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool));
@@ -3195,10 +3195,12 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	    return NULL;
     }
 
+check_dc_list:
     /* Last; check our own pending dealloc carrier list... */
     crr = allctr->cpool.dc_list.last;
     while (crr) {
 	if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
+	    Block_t* blk;
 	    unlink_carrier(&allctr->cpool.dc_list, crr);
 #ifdef ERTS_ALC_CPOOL_DEBUG
 	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
@@ -3207,6 +3209,9 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 #else
 	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
 #endif
+	    blk = MBC_TO_FIRST_BLK(allctr, crr);
+	    ASSERT(FBLK_TO_MBC(blk) == crr);
+	    allctr->link_free_block(allctr, blk);
 	    return crr;
 	}
 	crr = crr->prev;
@@ -3279,7 +3284,6 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
     orig_allctr = crr->cpool.orig_allctr;
 
     if (allctr != orig_allctr) {
-	Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr);
 	int cinit = orig_allctr->dd.ix - allctr->dd.ix;
 
 	/*
@@ -3296,6 +3300,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 	 * since the block is an mbc block that is free and last
 	 * in the carrier.
 	 */
+	blk = MBC_TO_FIRST_BLK(allctr, crr);
 	ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk));
 
 	ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk));
@@ -3319,7 +3324,9 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 	return;
     }
 
-    max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
+    blk = MBC_TO_FIRST_BLK(allctr, crr);
+    ASSERT(IS_FREE_LAST_MBC_BLK(blk));
+    max_size = (erts_aint_t) MBC_FBLK_SZ(blk);
     erts_atomic_set_nob(&crr->cpool.max_size, max_size);
 
     crr->next = NULL;
-- 
cgit v1.2.3


From 933bb51d40bcd665602fe4ece951b3111c301e74 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 22 Sep 2015 14:01:54 +0200
Subject: erts: Fix confusion of callbacks destroying_mbc() vs remove_mbc()

Problem #1 Goodfit was crippled by the fact that destroying_mbc()
was called _before_ the carriers was unlinked from mbc_list.

Problem #2 destroying_mbc() was called for carriers that later could be
resurrected from dc_list without a matching call to creating_mbc().
This was mostly a practical problem for the new test case
alloc_SUITE:migration that use the callbacks to create/destroy a mutex.

Solution:
destroying_mbc() is now only called just before a carrier is
destroyed (deallocated or put in mseg cache).

remove_mbc() is called both (like before) when inserted into cpool
but now also when last block is freed and mbc is scheduled for
destruction but may later be resurrected from dc_list.
---
 erts/emulator/beam/erl_alloc_util.c        | 27 ++++++++++++++++++++-------
 erts/emulator/beam/erl_alloc_util.h        |  2 +-
 erts/emulator/beam/erl_ao_firstfit_alloc.c | 22 +++++++++++++++++-----
 3 files changed, 38 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 3861196cfc..173cb091b6 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1437,6 +1437,16 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
 
 static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned);
 
+static ERTS_INLINE void
+dealloc_mbc(Allctr_t *allctr, Carrier_t *crr)
+{
+    ASSERT(IS_MB_CARRIER(crr));
+    if (allctr->destroying_mbc)
+        allctr->destroying_mbc(allctr, crr);
+
+    dealloc_carrier(allctr, crr, 1);
+}
+
 #ifdef ERTS_SMP
 
 static ERTS_INLINE Allctr_t*
@@ -3242,7 +3252,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr,
 
 	    dcrr = crr;
 	    crr = crr->next;
-	    dealloc_carrier(allctr, dcrr, 1);
+	    dealloc_mbc(allctr, dcrr);
 	    i++;
 	} while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER);
 
@@ -3273,11 +3283,14 @@ static void
 schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 {
     Allctr_t *orig_allctr;
+    Block_t *blk;
     int check_pending_dealloc;
     erts_aint_t max_size;
 
+    ASSERT(IS_MB_CARRIER(crr));
+
     if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
-	dealloc_carrier(allctr, crr, 1);
+	dealloc_mbc(allctr, crr);
 	return;
     }
 
@@ -3320,7 +3333,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 
     if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID
 	|| erts_thr_progress_has_reached(crr->cpool.thr_prgr)) {
-	dealloc_carrier(allctr, crr, 1);
+	dealloc_mbc(allctr, crr);
 	return;
     }
 
@@ -3901,9 +3914,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	}
 #endif
 
-	if (allctr->destroying_mbc)
-	    (*allctr->destroying_mbc)(allctr, crr);
-
 #ifdef ERTS_SMP
 	if (busy_pcrr_pp && *busy_pcrr_pp) {
 	    ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr);
@@ -3927,12 +3937,15 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	    else
 #endif
 		STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz);
+
+            if (allctr->remove_mbc)
+                allctr->remove_mbc(allctr, crr);
 	}
 
 #ifdef ERTS_SMP
 	schedule_dealloc_carrier(allctr, crr);
 #else
-	dealloc_carrier(allctr, crr, 1);
+	dealloc_mbc(allctr, crr);
 #endif
     }
 }
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index df1f0aa65a..f4a2ae7ff3 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -277,7 +277,7 @@ typedef struct ErtsDoubleLink_t_ {
 typedef struct {
     erts_atomic_t next;
     erts_atomic_t prev;
-    Allctr_t *orig_allctr;
+    Allctr_t *orig_allctr;      /* read-only while carrier is alive */
     ErtsThrPrgrVal thr_prgr;
     erts_atomic_t max_size;
     UWord abandon_limit;
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 7c2a5c3323..19420af8ab 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -209,7 +209,9 @@ static Block_t*	aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint);
 static void aoff_link_free_block(Allctr_t *, Block_t*);
 static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del);
 static void aoff_creating_mbc(Allctr_t*, Carrier_t*);
+#ifdef DEBUG
 static void aoff_destroying_mbc(Allctr_t*, Carrier_t*);
+#endif
 static void aoff_add_mbc(Allctr_t*, Carrier_t*);
 static void aoff_remove_mbc(Allctr_t*, Carrier_t*);
 static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*);
@@ -271,7 +273,11 @@ erts_aoffalc_start(AOFFAllctr_t *alc,
 
     allctr->get_next_mbc_size		= NULL;
     allctr->creating_mbc		= aoff_creating_mbc;
+#ifdef DEBUG
     allctr->destroying_mbc		= aoff_destroying_mbc;
+#else
+    allctr->destroying_mbc		= NULL;
+#endif
     allctr->add_mbc                     = aoff_add_mbc;
     allctr->remove_mbc                  = aoff_remove_mbc;
     allctr->largest_fblk_in_mbc         = aoff_largest_fblk_in_mbc;
@@ -885,17 +891,18 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier)
     HARD_CHECK_TREE(NULL, 0, *root, 0);
 }
 
+#define IS_CRR_IN_TREE(CRR,ROOT) \
+    ((CRR)->rbt_node.parent || (ROOT) == &(CRR)->rbt_node)
+
+#ifdef DEBUG
 static void aoff_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier)
 {
     AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
     AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier;
-    AOFF_RBTree_t *root = alc->mbc_root;
 
-    if (crr->rbt_node.parent || &crr->rbt_node == root) {
-	aoff_remove_mbc(allctr, carrier);
-    }
-    /*else already removed */
+    ASSERT(!IS_CRR_IN_TREE(crr, alc->mbc_root));
 }
+#endif
 
 static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier)
 {
@@ -903,6 +910,7 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier)
     AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier;
     AOFF_RBTree_t **root = &alc->mbc_root;
 
+    ASSERT(!IS_CRR_IN_TREE(crr, *root));
     HARD_CHECK_TREE(NULL, 0, *root, 0);   
 
     /* Link carrier in address order tree
@@ -919,6 +927,10 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier)
     AOFF_RBTree_t **root = &alc->mbc_root;
 
     ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier));
+
+    if (!IS_CRR_IN_TREE(crr,*root))
+        return;
+
     HARD_CHECK_TREE(NULL, 0, *root, 0);
 
     rbt_delete(root, &crr->rbt_node);
-- 
cgit v1.2.3


From 4759886bf9fc2e8ab8821a36fa8061a18e69a08d Mon Sep 17 00:00:00 2001
From: Luis Rascao <luis.rascao@gmail.com>
Date: Sat, 26 Sep 2015 12:06:29 +0100
Subject: Fix build fail when enabling distribution debug messages

---
 erts/emulator/beam/dist.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 23897a49ae..0bbcc5f966 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -45,6 +45,8 @@
 #include "erl_thr_progress.h"
 #include "dtrace-wrapper.h"
 
+#define DIST_CTL_DEFAULT_SIZE 64
+
 /* Turn this on to get printouts of all distribution messages
  * which go on the line
  */
@@ -66,9 +68,13 @@ static void bw(byte *buf, ErlDrvSizeT sz)
 static void
 dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz)
 {
+    ErtsHeapFactory factory;
+    DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE);
+    Eterm* ctl = ctl_default;
     byte *extp = edep->extp;
     Eterm msg;
-    Sint size = erts_decode_dist_ext_size(edep);
+    Sint ctl_len;
+    Sint size = ctl_len = erts_decode_dist_ext_size(edep);
     if (size < 0) {
 	erts_fprintf(stderr,
 		     "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n",
@@ -76,10 +82,9 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz)
 	bw(buf, sz);
     }
     else {
-	Eterm *hp;
 	ErlHeapFragment *mbuf = new_message_buffer(size);
-	hp = mbuf->mem;
-	msg = erts_decode_dist_ext(&hp, &mbuf->off_heap, edep);
+	erts_factory_static_init(&factory, ctl, ctl_len, &mbuf->off_heap);
+	msg = erts_decode_dist_ext(&factory, edep);
 	if (is_value(msg))
 	    erts_fprintf(stderr, "    %s: %T\n", what, msg);
 	else {
@@ -1136,7 +1141,6 @@ int erts_net_message(Port *prt,
 		     byte *buf,
 		     ErlDrvSizeT len)
 {
-#define DIST_CTL_DEFAULT_SIZE 64
     ErtsDistExternal ede;
     byte *t;
     Sint ctl_len;
@@ -1790,8 +1794,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 
     #ifdef ERTS_DIST_MSG_DBG
 	    erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl);
-	    if (is_value(msg))
-		erts_fprintf(stderr, "    MSG: %T\n", msg);
+        if (is_value(ctx->msg))
+            erts_fprintf(stderr, "    MSG: %T\n", ctx->msg);
     #endif
 
 	    ctx->data_size = ctx->pass_through_size;
-- 
cgit v1.2.3


From 93f5844185a459cbf9efc3843919c463a2eef0fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 8 Oct 2015 13:04:40 +0200
Subject: Teach erlang:is_builtin/3 that erlang:apply/3 is built-in

erlang:is_builtin(erlang, apply, 3) returns 'false'. That seems to be
an oversight in the implementation of erlang:is_builtin/3 rather than
a conscious design decision. Part of apply/3 is implemented in C (as a
special instruction), and part of it in Erlang (only used if apply/3
is used recursively). That makes apply/3 special compared to all other
BIFs.

From the viewpoint of the user, apply/3 is a built-in function,
since it cannot possibly be implemented in pure Erlang.

Noticed-by: Stavros Aronis
---
 erts/emulator/beam/beam_emu.c | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a42ce1c9f1..faf496a030 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6752,6 +6752,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     Export e;
     Export* ep;
 
+    if (Mod == am_erlang && Name == am_apply && arity == 3) {
+	/*
+	 * Special case. apply/3 is built-in (implemented in C),
+	 * but implemented in a different way than all other
+	 * BIFs.
+	 */
+	return 1;
+    }
+
     e.code[0] = Mod;
     e.code[1] = Name;
     e.code[2] = arity;
-- 
cgit v1.2.3


From 76777477d8722b785afa8da0166b4577fb7e047c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 9 Oct 2015 17:18:31 +0200
Subject: erts: Remove vheap mature from process control block

Binary vheap mature is not necessary for binary gc.
---
 erts/emulator/beam/erl_gc.c      | 2 --
 erts/emulator/beam/erl_process.c | 2 --
 erts/emulator/beam/erl_process.h | 2 --
 3 files changed, 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 734f120e09..29579f74e6 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -848,7 +848,6 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 
     if (OLD_HEAP(p) &&
 	    ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
-	    ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
 	    ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
 	ErlMessage *msgp;
 	Uint size_after;
@@ -2373,7 +2372,6 @@ sweep_off_heap(Process *p, int fullsweep)
     }
     BIN_VHEAP_SZ(p)     = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
     MSO(p).overhead     = bin_vheap;
-    BIN_VHEAP_MATURE(p) = bin_vheap;
 
     /*
      * If we got any shrink candidates, check them out.
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 879b523c38..15a6d5d651 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10834,7 +10834,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
     p->bin_old_vheap    = 0;
-    p->bin_vheap_mature = 0;
 
     p->sys_task_qs = NULL;
 
@@ -11054,7 +11053,6 @@ void erts_init_empty_process(Process *p)
     p->bin_old_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap = 0;
     p->sys_task_qs = NULL;
-    p->bin_vheap_mature = 0;
     ERTS_PTMR_INIT(p);
     p->next = NULL;
     p->off_heap.first = NULL;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 65422b8c15..e7c5614b9c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -898,7 +898,6 @@ struct ErtsPendingSuspend_ {
 
 #  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
 #  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
-#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
 #  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
 #  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap
 
@@ -1017,7 +1016,6 @@ struct process {
     ErtsPSD *psd;		/* Rarely used process specific data */
 
     Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
-    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
     Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
     Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
-- 
cgit v1.2.3


From 285b4fa13914dc4748c1020213a9c65cad3d2738 Mon Sep 17 00:00:00 2001
From: Serge Aleynikov <saleyn@gmail.com>
Date: Tue, 2 Jun 2015 08:44:23 -0400
Subject: erts: Add {line_delimiter, byte()} option to inet:setopts/2

A new {line_delimiter, byte()} option allows line-oriented TCP-based protocols
to use a custom line delimiting character. It is to be used in conjunction
with {packet, line}.

This option also works with erlang:decode_packet/3 when its first argument
is 'line'.
---
 erts/emulator/beam/atom.names      |  1 +
 erts/emulator/beam/erl_bif_port.c  | 10 ++++++++--
 erts/emulator/beam/packet_parser.c |  5 +++--
 erts/emulator/beam/packet_parser.h |  3 ++-
 4 files changed, 14 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index f9a2f3e33e..190e7817dc 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -321,6 +321,7 @@ atom ldflags
 atom Le='=<'
 atom lf
 atom line
+atom line_delimiter
 atom line_length
 atom linked_in_driver
 atom links
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 3ff54c7a60..e47d7bcbbb 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1329,7 +1329,8 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
     ErlSubBin* rest;
     Eterm res;
     Eterm options;
-    int code;
+    int   code;
+    char  delimiter = '\n';
 
     if (!is_binary(BIF_ARG_2) || 
         (!is_list(BIF_ARG_3) && !is_nil(BIF_ARG_3))) {
@@ -1370,6 +1371,11 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
                 case am_line_length:
                     trunc_len = val;
                     goto next_option;
+                case am_line_delimiter:
+                    if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) {
+                        delimiter = (char)val;
+                        goto next_option;
+                    }
                 }
             }
         }
@@ -1390,7 +1396,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
         pca.aligned_ptr = bin_ptr;
     }
     packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz,
-                                  max_plen, trunc_len, &http_state);
+                                  max_plen, trunc_len, delimiter, &http_state);
     if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) {
         if (packet_sz < 0) {
 	    goto error;
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index 2dd421a9e9..a737a86f14 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -256,6 +256,7 @@ int packet_get_length(enum PacketParseType htype,
                       const char* ptr, unsigned n, /* Bytes read so far */
                       unsigned max_plen,     /* Max packet length, 0=no limit */
                       unsigned trunc_len,    /* Truncate (lines) if longer, 0=no limit */
+                      char     delimiter,    /* Line delimiting character */
                       int*     statep)       /* Protocol specific state */
 {
     unsigned hlen, plen;
@@ -299,9 +300,9 @@ int packet_get_length(enum PacketParseType htype,
         goto remain;
 
     case TCP_PB_LINE_LF: {
-        /* TCP_PB_LINE_LF:  [Data ... \n]  */
+        /* TCP_PB_LINE_LF:  [Data ... Delimiter]  */
         const char* ptr2;
-        if ((ptr2 = memchr(ptr, '\n', n)) == NULL) {
+        if ((ptr2 = memchr(ptr, delimiter, n)) == NULL) {
             if (n > max_plen && max_plen != 0) { /* packet full */
                 DEBUGF((" => packet full (no NL)=%d\r\n", n));
                 goto error;
diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h
index ff158ff8b8..717d905fad 100644
--- a/erts/emulator/beam/packet_parser.h
+++ b/erts/emulator/beam/packet_parser.h
@@ -105,7 +105,8 @@ int packet_get_length(enum PacketParseType htype,
 		      const char* ptr, unsigned n,  /* Bytes read so far */
 		      unsigned max_plen,      /* Packet max length, 0=no limit */
 		      unsigned trunc_len,     /* Truncate (lines) if longer, 0=no limit */
-		      int* statep);           /* Internal protocol state */
+		      char     delimiter,     /* Line delimiting character */
+		      int*     statep);       /* Internal protocol state */
 
 ERTS_GLB_INLINE
 void packet_get_body(enum PacketParseType htype,
-- 
cgit v1.2.3


From afac3c7136c48d8630bd400c5454e146915e634f Mon Sep 17 00:00:00 2001
From: Serge Aleynikov <saleyn@gmail.com>
Date: Mon, 26 Oct 2015 14:46:54 +0100
Subject: erts: Add {line_delimiter, byte()} option to inet:setopts/2

A new {line_delimiter, byte()} option allows line-oriented TCP-based
protocols to use a custom line delimiting character. It is to be
used in conjunction with {packet, line}.

This option also works with erlang:decode_packet/3 when its first argument
is 'line'.
---
 erts/emulator/beam/atom.names      |  1 +
 erts/emulator/beam/erl_bif_port.c  | 10 ++++++++--
 erts/emulator/beam/packet_parser.c |  5 +++--
 erts/emulator/beam/packet_parser.h |  3 ++-
 4 files changed, 14 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index f9a2f3e33e..190e7817dc 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -321,6 +321,7 @@ atom ldflags
 atom Le='=<'
 atom lf
 atom line
+atom line_delimiter
 atom line_length
 atom linked_in_driver
 atom links
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 3ff54c7a60..e47d7bcbbb 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1329,7 +1329,8 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
     ErlSubBin* rest;
     Eterm res;
     Eterm options;
-    int code;
+    int   code;
+    char  delimiter = '\n';
 
     if (!is_binary(BIF_ARG_2) || 
         (!is_list(BIF_ARG_3) && !is_nil(BIF_ARG_3))) {
@@ -1370,6 +1371,11 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
                 case am_line_length:
                     trunc_len = val;
                     goto next_option;
+                case am_line_delimiter:
+                    if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) {
+                        delimiter = (char)val;
+                        goto next_option;
+                    }
                 }
             }
         }
@@ -1390,7 +1396,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
         pca.aligned_ptr = bin_ptr;
     }
     packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz,
-                                  max_plen, trunc_len, &http_state);
+                                  max_plen, trunc_len, delimiter, &http_state);
     if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) {
         if (packet_sz < 0) {
 	    goto error;
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index 2dd421a9e9..a737a86f14 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -256,6 +256,7 @@ int packet_get_length(enum PacketParseType htype,
                       const char* ptr, unsigned n, /* Bytes read so far */
                       unsigned max_plen,     /* Max packet length, 0=no limit */
                       unsigned trunc_len,    /* Truncate (lines) if longer, 0=no limit */
+                      char     delimiter,    /* Line delimiting character */
                       int*     statep)       /* Protocol specific state */
 {
     unsigned hlen, plen;
@@ -299,9 +300,9 @@ int packet_get_length(enum PacketParseType htype,
         goto remain;
 
     case TCP_PB_LINE_LF: {
-        /* TCP_PB_LINE_LF:  [Data ... \n]  */
+        /* TCP_PB_LINE_LF:  [Data ... Delimiter]  */
         const char* ptr2;
-        if ((ptr2 = memchr(ptr, '\n', n)) == NULL) {
+        if ((ptr2 = memchr(ptr, delimiter, n)) == NULL) {
             if (n > max_plen && max_plen != 0) { /* packet full */
                 DEBUGF((" => packet full (no NL)=%d\r\n", n));
                 goto error;
diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h
index ff158ff8b8..717d905fad 100644
--- a/erts/emulator/beam/packet_parser.h
+++ b/erts/emulator/beam/packet_parser.h
@@ -105,7 +105,8 @@ int packet_get_length(enum PacketParseType htype,
 		      const char* ptr, unsigned n,  /* Bytes read so far */
 		      unsigned max_plen,      /* Packet max length, 0=no limit */
 		      unsigned trunc_len,     /* Truncate (lines) if longer, 0=no limit */
-		      int* statep);           /* Internal protocol state */
+		      char     delimiter,     /* Line delimiting character */
+		      int*     statep);       /* Internal protocol state */
 
 ERTS_GLB_INLINE
 void packet_get_body(enum PacketParseType htype,
-- 
cgit v1.2.3


From 15af6b28beded3b253860b4ea5a7debe2c662674 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 2 Nov 2015 17:29:54 +0100
Subject: erts: Remove faulty ASSERT in erts_proc_*_refc

There is no guarantee that the ptab-slot in not reused
when we finally deallocates the process struct.
---
 erts/emulator/beam/erl_process_lock.h | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 788348e613..a64c993e8f 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -854,9 +854,6 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
 #endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
-	ASSERT(ERTS_AINT_NULL
-	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
-					    internal_pid_index(p->common.id)));
 	erts_free_proc(p);
     }
 }
@@ -872,9 +869,6 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 #endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
-	ASSERT(ERTS_AINT_NULL
-	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
-					    internal_pid_index(p->common.id)));
 	erts_free_proc(p);
     }
 }
-- 
cgit v1.2.3


From 1eeb980a3591b51f12363543b2ac9b0260ebd870 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 30 Oct 2015 19:09:14 +0100
Subject: erts: Remove ERTS_PSD_DIST_ENTRY

Not needed as it is always set to erts_this_dist_entry (on net_kernel).
---
 erts/emulator/beam/dist.c            |  9 ++-------
 erts/emulator/beam/erl_node_tables.c |  6 ------
 erts/emulator/beam/erl_process.c     | 11 +----------
 erts/emulator/beam/erl_process.h     | 17 ++++-------------
 4 files changed, 7 insertions(+), 36 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 23897a49ae..5a0f8388f8 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2650,13 +2650,8 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
     if (!net_kernel)
 	goto error;
 
-    /* By setting dist_entry==erts_this_dist_entry and DISTRIBUTION on
-       net_kernel do_net_exist will be called when net_kernel
-       is terminated !! */
-    (void) ERTS_PROC_SET_DIST_ENTRY(net_kernel,
-				    ERTS_PROC_LOCK_MAIN,
-				    erts_this_dist_entry);
-    erts_refc_inc(&erts_this_dist_entry->refc, 2);
+    /* By setting F_DISTRIBUTION on net_kernel,
+     * do_net_exist will be called when net_kernel is terminated !! */
     net_kernel->flags |= F_DISTRIBUTION;
 
     if (net_kernel != BIF_P)
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 2fb790b953..865de1726d 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1472,12 +1472,6 @@ setup_reference_table(void)
 		insert_links(ERTS_P_LINKS(proc), proc->common.id);
 	    if (ERTS_P_MONITORS(proc))
 		insert_monitors(ERTS_P_MONITORS(proc), proc->common.id);
-	    /* Insert controller */
-	    {
-		DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc);
-		if (dep)
-		    insert_dist_entry(dep, CTRL_REF, proc->common.id, 0);
-	    }
 	}
     }
     
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 3b1b593d1c..d583118e7b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -625,11 +625,6 @@ erts_pre_init_process(void)
      erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks
 	 = ERTS_PSD_SCHED_ID_SET_LOCKS;
 
-     erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks
-	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks
-	 = ERTS_PSD_DIST_ENTRY_SET_LOCKS;
-
      erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks
 	 = ERTS_PSD_CALL_TIME_BP_GET_LOCKS;
      erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
@@ -12373,9 +12368,7 @@ erts_continue_exit_process(Process *p)
 	    erts_proc_dec_refc(p);
     }
     
-    dep = ((p->flags & F_DISTRIBUTION)
-	   ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL)
-	   : NULL);
+    dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
     scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL);
     pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL);
     nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, ERTS_PROC_LOCKS_ALL, NULL);
@@ -12387,8 +12380,6 @@ erts_continue_exit_process(Process *p)
 
     if (dep) {
 	erts_do_net_exits(dep, reason);
-	if(dep)
-	    erts_deref_dist_entry(dep);
     }
 
     /*
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 20ffe7ea7c..10c6fa4a67 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -801,12 +801,11 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_ERROR_HANDLER			0
 #define ERTS_PSD_SAVED_CALLS_BUF		1
 #define ERTS_PSD_SCHED_ID			2
-#define ERTS_PSD_DIST_ENTRY			3
-#define ERTS_PSD_CALL_TIME_BP			4
-#define ERTS_PSD_DELAYED_GC_TASK_QS		5
-#define ERTS_PSD_NIF_TRAP_EXPORT		6
+#define ERTS_PSD_CALL_TIME_BP			3
+#define ERTS_PSD_DELAYED_GC_TASK_QS		4
+#define ERTS_PSD_NIF_TRAP_EXPORT		5
 
-#define ERTS_PSD_SIZE				7
+#define ERTS_PSD_SIZE				6
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -824,9 +823,6 @@ typedef struct {
 #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
 #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS
 
-#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN
-#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN
-
 #define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
@@ -1887,11 +1883,6 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #define ERTS_PROC_SCHED_ID(P, L, ID) \
   ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))
 
-#define ERTS_PROC_GET_DIST_ENTRY(P) \
-  ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY))
-#define ERTS_PROC_SET_DIST_ENTRY(P, L, D) \
-  ((DistEntry *) erts_psd_set((P), (L), ERTS_PSD_DIST_ENTRY, (void *) (D)))
-
 #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
   ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
 #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
-- 
cgit v1.2.3


From b93e9b611056828ac2c82f225960aa29348ebe97 Mon Sep 17 00:00:00 2001
From: Andrew Bennett <potatosaladx@gmail.com>
Date: Wed, 10 Jun 2015 13:48:30 -0600
Subject: stdlib: Add BIF binary:split/2 and binary:split/3

---
 erts/emulator/beam/atom.names       |   2 +
 erts/emulator/beam/bif.tab          |   7 +
 erts/emulator/beam/erl_bif_binary.c | 697 ++++++++++++++++++++++++++++++++++++
 3 files changed, 706 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index f9a2f3e33e..3d357886ee 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -121,6 +121,7 @@ atom binary_longest_prefix_trap
 atom binary_longest_suffix_trap
 atom binary_match_trap
 atom binary_matches_trap
+atom binary_split_trap
 atom binary_to_list_continue
 atom binary_to_term_trap
 atom block
@@ -584,6 +585,7 @@ atom trace trace_ts traced
 atom trace_control_word
 atom tracer
 atom trap_exit
+atom trim
 atom try_clause
 atom true
 atom tuple
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 4f0656d174..65f8d6f1f5 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -643,3 +643,10 @@ bif erts_debug:map_info/1
 #
 
 bif erlang:hash/2
+
+#
+# New in 19.0
+#
+
+bif binary:split/2
+bif binary:split/3
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 134aa2d396..68e5fe23c7 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -67,12 +67,16 @@ static Export binary_bin_to_list_trap_export;
 static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3);
 static Export binary_copy_trap_export;
 static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2);
+static Export binary_split_trap_export;
+static BIF_RETTYPE binary_split_trap(BIF_ALIST_3);
 static Uint max_loop_limit;
 
 static BIF_RETTYPE
 binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
 static BIF_RETTYPE
 binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
+static BIF_RETTYPE
+binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
 
 void erts_init_bif_binary(void)
 {
@@ -100,6 +104,10 @@ void erts_init_bif_binary(void)
 			  am_erlang, am_binary_copy_trap, 2,
 			  &binary_copy_trap);
 
+    erts_init_trap_export(&binary_split_trap_export,
+			  am_erlang, am_binary_split_trap, 3,
+			  &binary_split_trap);
+
     max_loop_limit = 0;
     return;
 }
@@ -2534,6 +2542,695 @@ BIF_RETTYPE binary_copy_2(BIF_ALIST_2)
     return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2);
 }
 
+#define BINARY_SPLIT_GLOBAL	0x01
+#define BINARY_SPLIT_TRIM	0x02
+
+static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
+			   Uint hsend, Uint hsflags, Eterm type, Binary *bin,
+			   Eterm state_term, Eterm *res_term)
+{
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    byte *temp_alloc = NULL;
+
+    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	goto badarg;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
+    }
+    if (state_term != NIL) {
+	Eterm *ptr = big_val(state_term);
+	type = ptr[1];
+	hsflags = (Uint)(ptr[2]);
+    }
+
+    if (hsflags & BINARY_SPLIT_GLOBAL) {
+	if (type == am_bm) {
+	    BMData *bm;
+	    Sint pos;
+	    Eterm ret;
+	    Eterm *hp;
+	    BMFindAllState state;
+	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_bm_data(bm);
+#endif
+	    if (state_term == NIL) {
+		bm_init_find_all(&state, hsstart, hsend);
+	    } else {
+		Eterm *ptr = big_val(state_term);
+		bm_restore_find_all(&state, (char *)(ptr+3));
+	    }
+
+	    pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
+	    if (pos == BM_NOT_FOUND) {
+		hp = HAlloc(p, 2);
+		ret = NIL;
+		ret = CONS(hp, subject, ret);
+	    } else if (pos == BM_RESTART) {
+		int x =
+		    (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		    !!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap bm!\n");
+#endif
+		hp = HAlloc(p, x+3);
+		hp[0] = make_pos_bignum_header(x+2);
+		hp[1] = type;
+		hp[2] = (Eterm)(hsflags);
+		bm_serialize_find_all(&state, (char *)(hp+3));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		bm_clean_find_all(&state);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		size_t orig_size;
+		Eterm orig;
+		Uint offset;
+		Uint bit_offset;
+		Uint bit_size;
+		ErlSubBin *sb;
+		FindallData *fad = state.out;
+		int i, j, k;
+		orig_size = binary_size(subject);
+		j = state.m - 1;
+		k = (int)(orig_size);
+		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - fad[j].pos - fad[j].len) == 0) {
+		    for (i = (j - 1); i >= 0; --i) {
+			if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
+			    break;
+			}
+		    }
+		    if (i == -1) {
+			if (fad[0].pos == 0) {
+			    ret = NIL;
+			} else {
+			    hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+			    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
+			    sb = (ErlSubBin *)(hp);
+			    sb->thing_word = HEADER_SUB_BIN;
+			    sb->size = fad[0].pos;
+			    sb->offs = offset;
+			    sb->orig = orig;
+			    sb->bitoffs = bit_offset;
+			    sb->bitsize = bit_size;
+			    sb->is_writable = 0;
+			    fad[0].epos = make_binary(sb);
+			    hp += ERL_SUB_BIN_SIZE;
+
+			    ret = NIL;
+			    ret = CONS(hp, make_binary(sb), ret);
+			    hp += 2;
+			}
+			erts_free_aligned_binary_bytes(temp_alloc);
+			bm_clean_find_all(&state);
+			BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+			*res_term = ret;
+			return DO_BIN_MATCH_OK;
+		    }
+		    j = i;
+		    k = fad[j+1].pos;
+		}
+		hp = HAlloc(p, (j + 2) * (ERL_SUB_BIN_SIZE + 2));
+		ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
+		sb = (ErlSubBin *)(hp);
+		sb->thing_word = HEADER_SUB_BIN;
+		sb->size = fad[0].pos;
+		sb->offs = offset;
+		sb->orig = orig;
+		sb->bitoffs = bit_offset;
+		sb->bitsize = 0;
+		sb->is_writable = 0;
+		fad[0].epos = make_binary(sb);
+		hp += ERL_SUB_BIN_SIZE;
+
+		for (i = 1; i <= j; ++i) {
+		    sb = (ErlSubBin *)(hp);
+		    sb->thing_word = HEADER_SUB_BIN;
+		    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
+		    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
+		    sb->orig = orig;
+		    sb->bitoffs = bit_offset;
+		    sb->bitsize = 0;
+		    sb->is_writable = 0;
+		    fad[i].epos = make_binary(sb);
+		    hp += ERL_SUB_BIN_SIZE;
+		}
+		ret = NIL;
+		sb = (ErlSubBin *)(hp);
+		sb->thing_word = HEADER_SUB_BIN;
+		sb->size = k - fad[j].pos - fad[j].len;
+		sb->offs = offset + fad[j].pos + fad[j].len;
+		sb->orig = orig;
+		sb->bitoffs = bit_offset;
+		sb->bitsize = bit_size;
+		sb->is_writable = 0;
+		hp += ERL_SUB_BIN_SIZE;
+		ret = CONS(hp, make_binary(sb), ret);
+		hp += 2;
+		for (i = j; i >= 0; --i) {
+		    ret = CONS(hp, fad[i].epos, ret);
+		    hp += 2;
+		}
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    bm_clean_find_all(&state);
+	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	    *res_term = ret;
+	    return DO_BIN_MATCH_OK;
+	} else if (type == am_ac) {
+	    ACTrie *act;
+	    int acr;
+	    ACFindAllState state;
+	    Eterm ret;
+	    Eterm *hp;
+	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_ac_trie(act);
+#endif
+	    if (state_term == NIL) {
+		ac_init_find_all(&state, act, hsstart, hsend);
+	    } else {
+		Eterm *ptr = big_val(state_term);
+		ac_restore_find_all(&state, (char *)(ptr+3));
+	    }
+	    acr = ac_find_all_non_overlapping(&state, bytes, &reds);
+	    if (acr == AC_NOT_FOUND) {
+		hp = HAlloc(p, 2);
+		ret = NIL;
+		ret = CONS(hp, subject, ret);
+	    } else if (acr == AC_RESTART) {
+		int x = (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		    !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap ac!\n");
+#endif
+		hp = HAlloc(p, x+3);
+		hp[0] = make_pos_bignum_header(x+2);
+		hp[1] = type;
+		hp[2] = (Eterm)(hsflags);
+		ac_serialize_find_all(&state, (char *)(hp+3));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		ac_clean_find_all(&state);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		size_t orig_size;
+		Eterm orig;
+		Uint offset;
+		Uint bit_offset;
+		Uint bit_size;
+		ErlSubBin *sb;
+		FindallData *fad = state.out;
+		int i, j, k;
+		orig_size = binary_size(subject);
+		j = state.m - 1;
+		k = (int)(orig_size);
+		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - fad[j].pos - fad[j].len) == 0) {
+		    for (i = (j - 1); i >= 0; --i) {
+			if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
+			    break;
+			}
+		    }
+		    if (i == -1) {
+			if (fad[0].pos == 0) {
+			    ret = NIL;
+			} else {
+			    hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+			    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
+			    sb = (ErlSubBin *)(hp);
+			    sb->thing_word = HEADER_SUB_BIN;
+			    sb->size = fad[0].pos;
+			    sb->offs = offset;
+			    sb->orig = orig;
+			    sb->bitoffs = bit_offset;
+			    sb->bitsize = bit_size;
+			    sb->is_writable = 0;
+			    fad[0].epos = make_binary(sb);
+			    hp += ERL_SUB_BIN_SIZE;
+
+			    ret = NIL;
+			    ret = CONS(hp, make_binary(sb), ret);
+			    hp += 2;
+			}
+			erts_free_aligned_binary_bytes(temp_alloc);
+			ac_clean_find_all(&state);
+			BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+			*res_term = ret;
+			return DO_BIN_MATCH_OK;
+		    }
+		    j = i;
+		    k = fad[j+1].pos;
+		}
+		hp = HAlloc(p, (j + 2) * (ERL_SUB_BIN_SIZE + 2));
+		ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
+		sb = (ErlSubBin *)(hp);
+		sb->thing_word = HEADER_SUB_BIN;
+		sb->size = fad[0].pos;
+		sb->offs = offset;
+		sb->orig = orig;
+		sb->bitoffs = bit_offset;
+		sb->bitsize = 0;
+		sb->is_writable = 0;
+		fad[0].epos = make_binary(sb);
+		hp += ERL_SUB_BIN_SIZE;
+
+		for (i = 1; i <= j; ++i) {
+		    sb = (ErlSubBin *)(hp);
+		    sb->thing_word = HEADER_SUB_BIN;
+		    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
+		    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
+		    sb->orig = orig;
+		    sb->bitoffs = bit_offset;
+		    sb->bitsize = 0;
+		    sb->is_writable = 0;
+		    fad[i].epos = make_binary(sb);
+		    hp += ERL_SUB_BIN_SIZE;
+		}
+		ret = NIL;
+		sb = (ErlSubBin *)(hp);
+		sb->thing_word = HEADER_SUB_BIN;
+		sb->size = k - fad[j].pos - fad[j].len;
+		sb->offs = offset + fad[j].pos + fad[j].len;
+		sb->orig = orig;
+		sb->bitoffs = bit_offset;
+		sb->bitsize = bit_size;
+		sb->is_writable = 0;
+		hp += ERL_SUB_BIN_SIZE;
+		ret = CONS(hp, make_binary(sb), ret);
+		hp += 2;
+		for (i = j; i >= 0; --i) {
+		    ret = CONS(hp, fad[i].epos, ret);
+		    hp += 2;
+		}
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    ac_clean_find_all(&state);
+	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	    *res_term = ret;
+	    return DO_BIN_MATCH_OK;
+	}
+    } else {
+	if (type == am_bm) {
+	    BMData *bm;
+	    Sint pos;
+	    Eterm ret;
+	    Eterm *hp;
+	    BMFindFirstState state;
+	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_bm_data(bm);
+#endif
+	    if (state_term == NIL) {
+		bm_init_find_first_match(&state, hsstart, hsend);
+	    } else {
+		Eterm *ptr = big_val(state_term);
+		memcpy((void *)(&state), (const void *)(ptr+3), sizeof(BMFindFirstState));
+	    }
+
+#ifdef HARDDEBUG
+	    erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos,
+			state.len);
+#endif
+	    pos = bm_find_first_match(&state, bm, bytes, &reds);
+	    if (pos == BM_NOT_FOUND) {
+		hp = HAlloc(p, 2);
+		ret = NIL;
+		ret = CONS(hp, subject, ret);
+	    } else if (pos == BM_RESTART) {
+		int x =
+		    (sizeof(state) / sizeof(Eterm)) +
+		    !!(sizeof(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap bm!\n");
+#endif
+		hp = HAlloc(p, x+3);
+		hp[0] = make_pos_bignum_header(x+2);
+		hp[1] = type;
+		hp[2] = (Eterm)(hsflags);
+		memcpy((void *)(hp+3), (const void *)(&state), sizeof(state));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		size_t orig_size;
+		Eterm orig;
+		Uint offset;
+		Uint bit_offset;
+		Uint bit_size;
+		ErlSubBin *sb1;
+		ErlSubBin *sb2;
+
+		orig_size = binary_size(subject);
+
+		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - pos - bm->len) == 0) {
+		    if (pos == 0) {
+			ret = NIL;
+		    } else {
+			hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+			sb1 = (ErlSubBin *) hp;
+			sb1->thing_word = HEADER_SUB_BIN;
+			sb1->size = pos;
+			sb1->offs = offset;
+			sb1->orig = orig;
+			sb1->bitoffs = bit_offset;
+			sb1->bitsize = bit_size;
+			sb1->is_writable = 0;
+			hp += ERL_SUB_BIN_SIZE;
+
+			ret = NIL;
+			ret = CONS(hp, make_binary(sb1), ret);
+			hp += 2;
+		    }
+		} else {
+		    hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
+		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+		    sb1 = (ErlSubBin *) hp;
+		    sb1->thing_word = HEADER_SUB_BIN;
+		    sb1->size = pos;
+		    sb1->offs = offset;
+		    sb1->orig = orig;
+		    sb1->bitoffs = bit_offset;
+		    sb1->bitsize = 0;
+		    sb1->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    sb2 = (ErlSubBin *) hp;
+		    sb2->thing_word = HEADER_SUB_BIN;
+		    sb2->size = orig_size - pos - bm->len;
+		    sb2->offs = offset + pos + bm->len;
+		    sb2->orig = orig;
+		    sb2->bitoffs = bit_offset;
+		    sb2->bitsize = bit_size;
+		    sb2->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    ret = NIL;
+		    ret = CONS(hp, make_binary(sb2), ret);
+		    hp += 2;
+		    ret = CONS(hp, make_binary(sb1), ret);
+		    hp += 2;
+		}
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	    *res_term = ret;
+	    return DO_BIN_MATCH_OK;
+	} else if (type == am_ac) {
+	    ACTrie *act;
+	    Uint pos, rlen;
+	    int acr;
+	    ACFindFirstState state;
+	    Eterm ret;
+	    Eterm *hp;
+	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_ac_trie(act);
+#endif
+	    if (state_term == NIL) {
+		ac_init_find_first_match(&state, act, hsstart, hsend);
+	    } else {
+		Eterm *ptr = big_val(state_term);
+		memcpy((void *)(&state), (const void *)(ptr+3), sizeof(ACFindFirstState));
+	    }
+	    acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
+	    if (acr == AC_NOT_FOUND) {
+		hp = HAlloc(p, 2);
+		ret = NIL;
+		ret = CONS(hp, subject, ret);
+	    } else if (acr == AC_RESTART) {
+		int x =
+		    (sizeof(state) / sizeof(Eterm)) +
+		    !!(sizeof(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap ac!\n");
+#endif
+		hp = HAlloc(p, x+3);
+		hp[0] = make_pos_bignum_header(x+2);
+		hp[1] = type;
+		hp[2] = (Eterm)(hsflags);
+		memcpy((void *)(hp+3), (const void *)(&state), sizeof(state));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		size_t orig_size;
+		Eterm orig;
+		Uint offset;
+		Uint bit_offset;
+		Uint bit_size;
+		ErlSubBin *sb1;
+		ErlSubBin *sb2;
+
+		orig_size = binary_size(subject);
+
+		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - pos - rlen) == 0) {
+		    if (pos == 0) {
+			ret = NIL;
+		    } else {
+			hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+			sb1 = (ErlSubBin *) hp;
+			sb1->thing_word = HEADER_SUB_BIN;
+			sb1->size = pos;
+			sb1->offs = offset;
+			sb1->orig = orig;
+			sb1->bitoffs = bit_offset;
+			sb1->bitsize = bit_size;
+			sb1->is_writable = 0;
+			hp += ERL_SUB_BIN_SIZE;
+
+			ret = NIL;
+			ret = CONS(hp, make_binary(sb1), ret);
+			hp += 2;
+		    }
+		} else {
+		    hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
+		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+		    sb1 = (ErlSubBin *) hp;
+		    sb1->thing_word = HEADER_SUB_BIN;
+		    sb1->size = pos;
+		    sb1->offs = offset;
+		    sb1->orig = orig;
+		    sb1->bitoffs = bit_offset;
+		    sb1->bitsize = 0;
+		    sb1->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    sb2 = (ErlSubBin *) hp;
+		    sb2->thing_word = HEADER_SUB_BIN;
+		    sb2->size = orig_size - pos - rlen;
+		    sb2->offs = offset + pos + rlen;
+		    sb2->orig = orig;
+		    sb2->bitoffs = bit_offset;
+		    sb2->bitsize = bit_size;
+		    sb2->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    ret = NIL;
+		    ret = CONS(hp, make_binary(sb2), ret);
+		    hp += 2;
+		    ret = CONS(hp, make_binary(sb1), ret);
+		    hp += 2;
+		}
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	    *res_term = ret;
+	    return DO_BIN_MATCH_OK;
+	}
+    }
+ badarg:
+    return DO_BIN_MATCH_BADARG;
+}
+
+static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uint *optp)
+{
+    Eterm *tp;
+    Uint pos;
+    Sint len;
+    *optp = 0;
+    *posp = 0;
+    *endp = binary_size(bin);
+    if (l == ((Eterm) 0) || l == NIL) {
+	return 0;
+    } else if (is_list(l)) {
+	while(is_list(l)) {
+	    Eterm t = CAR(list_val(l));
+	    Uint orig_size;
+	    if (is_atom(t)) {
+		if (t == am_global) {
+		    *optp |= BINARY_SPLIT_GLOBAL;
+		    l = CDR(list_val(l));
+		    continue;
+		}
+		if (t == am_trim) {
+		    *optp |= BINARY_SPLIT_TRIM;
+		    l = CDR(list_val(l));
+		    continue;
+		}
+	    }
+	    if (!is_tuple(t)) {
+		goto badarg;
+	    }
+	    tp = tuple_val(t);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (tp[1] != am_scope || is_not_tuple(tp[2])) {
+		goto badarg;
+	    }
+	    tp = tuple_val(tp[2]);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (!term_to_Uint(tp[1], &pos)) {
+		goto badarg;
+	    }
+	    if (!term_to_Sint(tp[2], &len)) {
+		goto badarg;
+	    }
+	    if (len < 0) {
+		Uint lentmp = -(Uint)len;
+		/* overflow */
+		if ((Sint)lentmp < 0) {
+		    goto badarg;
+		}
+		len = lentmp;
+		pos -= len;
+	    }
+	    /* overflow */
+	    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+		goto badarg;
+	    }
+	    *endp = len + pos;
+	    *posp = pos;
+	    if ((orig_size = binary_size(bin)) < pos ||
+		orig_size < (*endp)) {
+		goto badarg;
+	    }
+	    l = CDR(list_val(l));
+	}
+	return 0;
+    } else {
+    badarg:
+	return 1;
+    }
+}
+
+static BIF_RETTYPE binary_split_trap(BIF_ALIST_3)
+{
+    int runres;
+    Eterm result;
+    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    runres = do_binary_split(BIF_P,BIF_ARG_1,0,0,0,NIL,bin,BIF_ARG_2,&result);
+    if (runres == DO_BIN_MATCH_OK) {
+	BIF_RET(result);
+    } else {
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_split_trap_export, BIF_P, BIF_ARG_1, result,
+		  BIF_ARG_3);
+    }
+}
+
+BIF_RETTYPE binary_split_3(BIF_ALIST_3)
+{
+    return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
+
+static BIF_RETTYPE
+binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
+{
+    Uint hsflags;
+    Uint hsstart;
+    Uint hsend;
+    Eterm *tp;
+    Eterm type;
+    Binary *bin;
+    Eterm bin_term = NIL;
+    int runres;
+    Eterm result;
+
+    if (is_not_binary(arg1)) {
+	goto badarg;
+    }
+    if (parse_split_opts_list(arg3, arg1, &hsstart, &hsend, &hsflags)) {
+	goto badarg;
+    }
+    if (hsend == 0) {
+	tp = HAlloc(p, 2);
+	result = NIL;
+	result = CONS(tp, arg1, result);
+	BIF_RET(result);
+    }
+    if (is_tuple(arg2)) {
+	tp = tuple_val(arg2);
+	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
+	    goto badarg;
+	}
+	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
+	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
+	    goto badarg;
+	}
+	type = tp[1];
+	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	if (type == am_bm &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
+	    goto badarg;
+	}
+	if (type == am_ac &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
+	    goto badarg;
+	}
+	bin_term = tp[2];
+    } else if (do_binary_match_compile(arg2, &type, &bin)) {
+	goto badarg;
+    }
+    runres = do_binary_split(p, arg1, hsstart, hsend, hsflags, type, bin, NIL, &result);
+    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
+	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
+    } else if (bin_term == NIL) {
+	erts_bin_free(bin);
+    }
+    switch(runres) {
+    case DO_BIN_MATCH_OK:
+	BIF_RET(result);
+    case DO_BIN_MATCH_RESTART:
+	BIF_TRAP3(&binary_split_trap_export, p, arg1, result, bin_term);
+    default:
+	goto badarg;
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+
+BIF_RETTYPE binary_split_2(BIF_ALIST_2)
+{
+    return binary_split(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+}
+
+
 BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1)
 {
     ErlSubBin *sb;
-- 
cgit v1.2.3


From d1283d1f826bdbd584bc87572ab46001164939e1 Mon Sep 17 00:00:00 2001
From: Andrew Bennett <potatosaladx@gmail.com>
Date: Thu, 25 Jun 2015 11:26:48 -0600
Subject: stdlib: Add BIF option 'trim_all' to binary:split/3

---
 erts/emulator/beam/atom.names       |   1 +
 erts/emulator/beam/erl_bif_binary.c | 496 +++++++++++++++++++++++++-----------
 2 files changed, 349 insertions(+), 148 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 3d357886ee..5f27aaa14b 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -586,6 +586,7 @@ atom trace_control_word
 atom tracer
 atom trap_exit
 atom trim
+atom trim_all
 atom try_clause
 atom true
 atom tuple
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 68e5fe23c7..1709f7671d 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2544,6 +2544,7 @@ BIF_RETTYPE binary_copy_2(BIF_ALIST_2)
 
 #define BINARY_SPLIT_GLOBAL	0x01
 #define BINARY_SPLIT_TRIM	0x02
+#define BINARY_SPLIT_TRIM_ALL	0x04
 
 static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 			   Uint hsend, Uint hsflags, Eterm type, Binary *bin,
@@ -2616,88 +2617,177 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		Uint bit_size;
 		ErlSubBin *sb;
 		FindallData *fad = state.out;
-		int i, j, k;
+		Sint i, j;
+		Sint drop = 0;
+		Sint head = 0;
+		Sint tail;
+		Uint list_size;
+		Uint tail_pos;
+
+		tail = state.m - 1;
+		list_size = state.m + 1;
 		orig_size = binary_size(subject);
-		j = state.m - 1;
-		k = (int)(orig_size);
-		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - fad[j].pos - fad[j].len) == 0) {
-		    for (i = (j - 1); i >= 0; --i) {
-			if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
-			    break;
+		tail_pos = (Uint)(orig_size);
+
+		if (hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) {
+		    if ((orig_size - fad[tail].pos - fad[tail].len) == 0) {
+			list_size--;
+			for (i = (tail - 1); i >= 0; --i) {
+			    if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
+				break;
+			    }
+			    list_size--;
 			}
+			if (i == -1) {
+			    if (fad[head].pos == 0) {
+				ret = NIL;
+			    } else {
+				hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+				ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
+				sb = (ErlSubBin *)(hp);
+				sb->thing_word = HEADER_SUB_BIN;
+				sb->size = fad[head].pos;
+				sb->offs = offset;
+				sb->orig = orig;
+				sb->bitoffs = bit_offset;
+				sb->bitsize = bit_size;
+				sb->is_writable = 0;
+				fad[head].epos = make_binary(sb);
+				hp += ERL_SUB_BIN_SIZE;
+
+				ret = NIL;
+				ret = CONS(hp, make_binary(sb), ret);
+				hp += 2;
+			    }
+			    erts_free_aligned_binary_bytes(temp_alloc);
+			    bm_clean_find_all(&state);
+			    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+			    *res_term = ret;
+			    return DO_BIN_MATCH_OK;
+			}
+			tail = i;
+			tail_pos = fad[tail+1].pos;
 		    }
-		    if (i == -1) {
-			if (fad[0].pos == 0) {
-			    ret = NIL;
-			} else {
-			    hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
-			    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+		}
+		if (hsflags & BINARY_SPLIT_TRIM_ALL) {
+		    if (fad[head].pos == 0) {
+			drop++;
+			list_size--;
+			for (i = drop, j = tail; i <= j; ++i) {
+			    if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
+				break;
+			    }
+			    drop++;
+			    list_size--;
+			}
+			head = drop - 1;
+		    }
+		    for (i = (head+1), j = tail; i <= j; ++i) {
+			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) == 0) {
+			    list_size--;
+			}
+		    }
+
+		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
+		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
 
+		    if (drop == 0) {
+			sb = (ErlSubBin *)(hp);
+			sb->thing_word = HEADER_SUB_BIN;
+			sb->size = fad[head].pos;
+			sb->offs = offset;
+			sb->orig = orig;
+			sb->bitoffs = bit_offset;
+			sb->bitsize = 0;
+			sb->is_writable = 0;
+			fad[head].epos = make_binary(sb);
+			hp += ERL_SUB_BIN_SIZE;
+		    }
+
+		    for (i = (head+1), j = tail; i <= j; ++i) {
+			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
 			    sb = (ErlSubBin *)(hp);
 			    sb->thing_word = HEADER_SUB_BIN;
-			    sb->size = fad[0].pos;
-			    sb->offs = offset;
+			    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
+			    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
 			    sb->orig = orig;
 			    sb->bitoffs = bit_offset;
-			    sb->bitsize = bit_size;
+			    sb->bitsize = 0;
 			    sb->is_writable = 0;
-			    fad[0].epos = make_binary(sb);
+			    fad[i].epos = make_binary(sb);
 			    hp += ERL_SUB_BIN_SIZE;
+			}
+		    }
 
-			    ret = NIL;
-			    ret = CONS(hp, make_binary(sb), ret);
+		    sb = (ErlSubBin *)(hp);
+		    sb->thing_word = HEADER_SUB_BIN;
+		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
+		    sb->offs = offset + fad[tail].pos + fad[tail].len;
+		    sb->orig = orig;
+		    sb->bitoffs = bit_offset;
+		    sb->bitsize = bit_size;
+		    sb->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    ret = NIL;
+		    ret = CONS(hp, make_binary(sb), ret);
+		    hp += 2;
+		    for (i = tail, j = head; i > j; --i) {
+			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
+			    ret = CONS(hp, fad[i].epos, ret);
 			    hp += 2;
 			}
-			erts_free_aligned_binary_bytes(temp_alloc);
-			bm_clean_find_all(&state);
-			BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-			*res_term = ret;
-			return DO_BIN_MATCH_OK;
 		    }
-		    j = i;
-		    k = fad[j+1].pos;
-		}
-		hp = HAlloc(p, (j + 2) * (ERL_SUB_BIN_SIZE + 2));
-		ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-		sb = (ErlSubBin *)(hp);
-		sb->thing_word = HEADER_SUB_BIN;
-		sb->size = fad[0].pos;
-		sb->offs = offset;
-		sb->orig = orig;
-		sb->bitoffs = bit_offset;
-		sb->bitsize = 0;
-		sb->is_writable = 0;
-		fad[0].epos = make_binary(sb);
-		hp += ERL_SUB_BIN_SIZE;
-
-		for (i = 1; i <= j; ++i) {
+		    if (drop == 0) {
+			ret = CONS(hp, fad[head].epos, ret);
+			hp += 2;
+		    }
+		} else {
+		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
+		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
 		    sb = (ErlSubBin *)(hp);
 		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
-		    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
+		    sb->size = fad[head].pos;
+		    sb->offs = offset;
 		    sb->orig = orig;
 		    sb->bitoffs = bit_offset;
 		    sb->bitsize = 0;
 		    sb->is_writable = 0;
-		    fad[i].epos = make_binary(sb);
+		    fad[head].epos = make_binary(sb);
 		    hp += ERL_SUB_BIN_SIZE;
-		}
-		ret = NIL;
-		sb = (ErlSubBin *)(hp);
-		sb->thing_word = HEADER_SUB_BIN;
-		sb->size = k - fad[j].pos - fad[j].len;
-		sb->offs = offset + fad[j].pos + fad[j].len;
-		sb->orig = orig;
-		sb->bitoffs = bit_offset;
-		sb->bitsize = bit_size;
-		sb->is_writable = 0;
-		hp += ERL_SUB_BIN_SIZE;
-		ret = CONS(hp, make_binary(sb), ret);
-		hp += 2;
-		for (i = j; i >= 0; --i) {
-		    ret = CONS(hp, fad[i].epos, ret);
+
+		    for (i = (head+1), j = tail; i <= j; ++i) {
+			sb = (ErlSubBin *)(hp);
+			sb->thing_word = HEADER_SUB_BIN;
+			sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
+			sb->offs = offset + fad[i-1].pos + fad[i-1].len;
+			sb->orig = orig;
+			sb->bitoffs = bit_offset;
+			sb->bitsize = 0;
+			sb->is_writable = 0;
+			fad[i].epos = make_binary(sb);
+			hp += ERL_SUB_BIN_SIZE;
+		    }
+
+		    sb = (ErlSubBin *)(hp);
+		    sb->thing_word = HEADER_SUB_BIN;
+		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
+		    sb->offs = offset + fad[tail].pos + fad[tail].len;
+		    sb->orig = orig;
+		    sb->bitoffs = bit_offset;
+		    sb->bitsize = bit_size;
+		    sb->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    ret = NIL;
+		    ret = CONS(hp, make_binary(sb), ret);
 		    hp += 2;
+		    for (i = tail, j = head; i >= j; --i) {
+			ret = CONS(hp, fad[i].epos, ret);
+			hp += 2;
+		    }
 		}
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
@@ -2752,88 +2842,177 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		Uint bit_size;
 		ErlSubBin *sb;
 		FindallData *fad = state.out;
-		int i, j, k;
+		Sint i, j;
+		Sint drop = 0;
+		Sint head = 0;
+		Sint tail;
+		Uint list_size;
+		Uint tail_pos;
+
+		tail = state.m - 1;
+		list_size = state.m + 1;
 		orig_size = binary_size(subject);
-		j = state.m - 1;
-		k = (int)(orig_size);
-		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - fad[j].pos - fad[j].len) == 0) {
-		    for (i = (j - 1); i >= 0; --i) {
-			if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
-			    break;
+		tail_pos = (Uint)(orig_size);
+
+		if (hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) {
+		    if ((orig_size - fad[tail].pos - fad[tail].len) == 0) {
+			list_size--;
+			for (i = (tail - 1); i >= 0; --i) {
+			    if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
+				break;
+			    }
+			    list_size--;
 			}
+			if (i == -1) {
+			    if (fad[head].pos == 0) {
+				ret = NIL;
+			    } else {
+				hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+				ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
+				sb = (ErlSubBin *)(hp);
+				sb->thing_word = HEADER_SUB_BIN;
+				sb->size = fad[head].pos;
+				sb->offs = offset;
+				sb->orig = orig;
+				sb->bitoffs = bit_offset;
+				sb->bitsize = bit_size;
+				sb->is_writable = 0;
+				fad[head].epos = make_binary(sb);
+				hp += ERL_SUB_BIN_SIZE;
+
+				ret = NIL;
+				ret = CONS(hp, make_binary(sb), ret);
+				hp += 2;
+			    }
+			    erts_free_aligned_binary_bytes(temp_alloc);
+			    ac_clean_find_all(&state);
+			    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+			    *res_term = ret;
+			    return DO_BIN_MATCH_OK;
+			}
+			tail = i;
+			tail_pos = fad[tail+1].pos;
+		    }
+		}
+		if (hsflags & BINARY_SPLIT_TRIM_ALL) {
+		    if (fad[head].pos == 0) {
+			drop++;
+			list_size--;
+			for (i = drop, j = tail; i <= j; ++i) {
+			    if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
+				break;
+			    }
+			    drop++;
+			    list_size--;
+			}
+			head = drop - 1;
+		    }
+		    for (i = (head+1), j = tail; i <= j; ++i) {
+			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) == 0) {
+			    list_size--;
+			}
+		    }
+
+		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
+		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
+		    if (drop == 0) {
+			sb = (ErlSubBin *)(hp);
+			sb->thing_word = HEADER_SUB_BIN;
+			sb->size = fad[head].pos;
+			sb->offs = offset;
+			sb->orig = orig;
+			sb->bitoffs = bit_offset;
+			sb->bitsize = 0;
+			sb->is_writable = 0;
+			fad[head].epos = make_binary(sb);
+			hp += ERL_SUB_BIN_SIZE;
 		    }
-		    if (i == -1) {
-			if (fad[0].pos == 0) {
-			    ret = NIL;
-			} else {
-			    hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
-			    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
 
+		    for (i = (head+1), j = tail; i <= j; ++i) {
+			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
 			    sb = (ErlSubBin *)(hp);
 			    sb->thing_word = HEADER_SUB_BIN;
-			    sb->size = fad[0].pos;
-			    sb->offs = offset;
+			    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
+			    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
 			    sb->orig = orig;
 			    sb->bitoffs = bit_offset;
-			    sb->bitsize = bit_size;
+			    sb->bitsize = 0;
 			    sb->is_writable = 0;
-			    fad[0].epos = make_binary(sb);
+			    fad[i].epos = make_binary(sb);
 			    hp += ERL_SUB_BIN_SIZE;
+			}
+		    }
 
-			    ret = NIL;
-			    ret = CONS(hp, make_binary(sb), ret);
+		    sb = (ErlSubBin *)(hp);
+		    sb->thing_word = HEADER_SUB_BIN;
+		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
+		    sb->offs = offset + fad[tail].pos + fad[tail].len;
+		    sb->orig = orig;
+		    sb->bitoffs = bit_offset;
+		    sb->bitsize = bit_size;
+		    sb->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    ret = NIL;
+		    ret = CONS(hp, make_binary(sb), ret);
+		    hp += 2;
+		    for (i = tail, j = head; i > j; --i) {
+			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
+			    ret = CONS(hp, fad[i].epos, ret);
 			    hp += 2;
 			}
-			erts_free_aligned_binary_bytes(temp_alloc);
-			ac_clean_find_all(&state);
-			BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-			*res_term = ret;
-			return DO_BIN_MATCH_OK;
 		    }
-		    j = i;
-		    k = fad[j+1].pos;
-		}
-		hp = HAlloc(p, (j + 2) * (ERL_SUB_BIN_SIZE + 2));
-		ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-		sb = (ErlSubBin *)(hp);
-		sb->thing_word = HEADER_SUB_BIN;
-		sb->size = fad[0].pos;
-		sb->offs = offset;
-		sb->orig = orig;
-		sb->bitoffs = bit_offset;
-		sb->bitsize = 0;
-		sb->is_writable = 0;
-		fad[0].epos = make_binary(sb);
-		hp += ERL_SUB_BIN_SIZE;
-
-		for (i = 1; i <= j; ++i) {
+		    if (drop == 0) {
+			ret = CONS(hp, fad[head].epos, ret);
+			hp += 2;
+		    }
+		} else {
+		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
+		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+
 		    sb = (ErlSubBin *)(hp);
 		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
-		    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
+		    sb->size = fad[head].pos;
+		    sb->offs = offset;
 		    sb->orig = orig;
 		    sb->bitoffs = bit_offset;
 		    sb->bitsize = 0;
 		    sb->is_writable = 0;
-		    fad[i].epos = make_binary(sb);
+		    fad[head].epos = make_binary(sb);
 		    hp += ERL_SUB_BIN_SIZE;
-		}
-		ret = NIL;
-		sb = (ErlSubBin *)(hp);
-		sb->thing_word = HEADER_SUB_BIN;
-		sb->size = k - fad[j].pos - fad[j].len;
-		sb->offs = offset + fad[j].pos + fad[j].len;
-		sb->orig = orig;
-		sb->bitoffs = bit_offset;
-		sb->bitsize = bit_size;
-		sb->is_writable = 0;
-		hp += ERL_SUB_BIN_SIZE;
-		ret = CONS(hp, make_binary(sb), ret);
-		hp += 2;
-		for (i = j; i >= 0; --i) {
-		    ret = CONS(hp, fad[i].epos, ret);
+
+		    for (i = (head+1), j = tail; i <= j; ++i) {
+			sb = (ErlSubBin *)(hp);
+			sb->thing_word = HEADER_SUB_BIN;
+			sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
+			sb->offs = offset + fad[i-1].pos + fad[i-1].len;
+			sb->orig = orig;
+			sb->bitoffs = bit_offset;
+			sb->bitsize = 0;
+			sb->is_writable = 0;
+			fad[i].epos = make_binary(sb);
+			hp += ERL_SUB_BIN_SIZE;
+		    }
+
+		    sb = (ErlSubBin *)(hp);
+		    sb->thing_word = HEADER_SUB_BIN;
+		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
+		    sb->offs = offset + fad[tail].pos + fad[tail].len;
+		    sb->orig = orig;
+		    sb->bitoffs = bit_offset;
+		    sb->bitsize = bit_size;
+		    sb->is_writable = 0;
+		    hp += ERL_SUB_BIN_SIZE;
+
+		    ret = NIL;
+		    ret = CONS(hp, make_binary(sb), ret);
 		    hp += 2;
+		    for (i = tail, j = head; i >= j; --i) {
+			ret = CONS(hp, fad[i].epos, ret);
+			hp += 2;
+		    }
 		}
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
@@ -2898,7 +3077,7 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 
 		orig_size = binary_size(subject);
 
-		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - pos - bm->len) == 0) {
+		if ((hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && (orig_size - pos - bm->len) == 0) {
 		    if (pos == 0) {
 			ret = NIL;
 		    } else {
@@ -2919,17 +3098,23 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 			hp += 2;
 		    }
 		} else {
-		    hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
-		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-		    sb1 = (ErlSubBin *) hp;
-		    sb1->thing_word = HEADER_SUB_BIN;
-		    sb1->size = pos;
-		    sb1->offs = offset;
-		    sb1->orig = orig;
-		    sb1->bitoffs = bit_offset;
-		    sb1->bitsize = 0;
-		    sb1->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
+		    if ((hsflags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
+			hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2));
+			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+			sb1 = NULL;
+		    } else {
+			hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
+			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+			sb1 = (ErlSubBin *) hp;
+			sb1->thing_word = HEADER_SUB_BIN;
+			sb1->size = pos;
+			sb1->offs = offset;
+			sb1->orig = orig;
+			sb1->bitoffs = bit_offset;
+			sb1->bitsize = 0;
+			sb1->is_writable = 0;
+			hp += ERL_SUB_BIN_SIZE;
+		    }
 
 		    sb2 = (ErlSubBin *) hp;
 		    sb2->thing_word = HEADER_SUB_BIN;
@@ -2944,8 +3129,10 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		    ret = NIL;
 		    ret = CONS(hp, make_binary(sb2), ret);
 		    hp += 2;
-		    ret = CONS(hp, make_binary(sb1), ret);
-		    hp += 2;
+		    if (sb1 != NULL) {
+			ret = CONS(hp, make_binary(sb1), ret);
+			hp += 2;
+		    }
 		}
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
@@ -3003,7 +3190,7 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 
 		orig_size = binary_size(subject);
 
-		if ((hsflags & BINARY_SPLIT_TRIM) && (orig_size - pos - rlen) == 0) {
+		if ((hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && (orig_size - pos - rlen) == 0) {
 		    if (pos == 0) {
 			ret = NIL;
 		    } else {
@@ -3024,17 +3211,23 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 			hp += 2;
 		    }
 		} else {
-		    hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
-		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-		    sb1 = (ErlSubBin *) hp;
-		    sb1->thing_word = HEADER_SUB_BIN;
-		    sb1->size = pos;
-		    sb1->offs = offset;
-		    sb1->orig = orig;
-		    sb1->bitoffs = bit_offset;
-		    sb1->bitsize = 0;
-		    sb1->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
+		    if ((hsflags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
+			hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
+			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+			sb1 = NULL;
+		    } else {
+			hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
+			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+			sb1 = (ErlSubBin *) hp;
+			sb1->thing_word = HEADER_SUB_BIN;
+			sb1->size = pos;
+			sb1->offs = offset;
+			sb1->orig = orig;
+			sb1->bitoffs = bit_offset;
+			sb1->bitsize = 0;
+			sb1->is_writable = 0;
+			hp += ERL_SUB_BIN_SIZE;
+		    }
 
 		    sb2 = (ErlSubBin *) hp;
 		    sb2->thing_word = HEADER_SUB_BIN;
@@ -3049,8 +3242,10 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		    ret = NIL;
 		    ret = CONS(hp, make_binary(sb2), ret);
 		    hp += 2;
-		    ret = CONS(hp, make_binary(sb1), ret);
-		    hp += 2;
+		    if (sb1 != NULL) {
+			ret = CONS(hp, make_binary(sb1), ret);
+			hp += 2;
+		    }
 		}
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
@@ -3088,6 +3283,11 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin
 		    l = CDR(list_val(l));
 		    continue;
 		}
+		if (t == am_trim_all) {
+		    *optp |= BINARY_SPLIT_TRIM_ALL;
+		    l = CDR(list_val(l));
+		    continue;
+		}
 	    }
 	    if (!is_tuple(t)) {
 		goto badarg;
-- 
cgit v1.2.3


From 20855f1819f91eeeb1fa746186477d3824024500 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 26 Aug 2015 15:20:45 +0200
Subject: erts: Replace 0 with THE_NON_VALUE

---
 erts/emulator/beam/erl_bif_binary.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 1709f7671d..860c9a9779 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1305,7 +1305,7 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
     Eterm *tp;
     Uint pos;
     Sint len;
-    if (l == ((Eterm) 0) || l == NIL) {
+    if (l == THE_NON_VALUE || l == NIL) {
 	/* Invalid term or NIL, we're called from binary_match(es)_2 or
 	   have no options*/
 	*posp = 0;
@@ -1535,13 +1535,13 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 
 BIF_RETTYPE binary_match_2(BIF_ALIST_2)
 {
-    return binary_match(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE);
 }
 
 
 BIF_RETTYPE binary_matches_2(BIF_ALIST_2)
 {
-    return binary_matches(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+    return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE);
 }
 
 
@@ -3266,7 +3266,7 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin
     *optp = 0;
     *posp = 0;
     *endp = binary_size(bin);
-    if (l == ((Eterm) 0) || l == NIL) {
+    if (l == THE_NON_VALUE || l == NIL) {
 	return 0;
     } else if (is_list(l)) {
 	while(is_list(l)) {
@@ -3427,7 +3427,7 @@ binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 
 BIF_RETTYPE binary_split_2(BIF_ALIST_2)
 {
-    return binary_split(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+    return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE);
 }
 
 
-- 
cgit v1.2.3


From 5b600aac42fdc4d08fabba682d7803351c9bfbdb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 26 Aug 2015 15:43:00 +0200
Subject: erts: Refactor backend of binary:split

to reduce code volume.
---
 erts/emulator/beam/erl_bif_binary.c | 659 +++++++++---------------------------
 1 file changed, 151 insertions(+), 508 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 860c9a9779..6adc61df19 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2542,6 +2542,11 @@ BIF_RETTYPE binary_copy_2(BIF_ALIST_2)
     return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2);
 }
 
+static Eterm do_split_single_result(Process*, Eterm subject,
+				    Sint pos, Sint len, Uint hsflags);
+static Eterm do_split_global_result(Process*, FindallData *fad, Uint fad_sz,
+				    Eterm subject, Uint hsflags);
+
 #define BINARY_SPLIT_GLOBAL	0x01
 #define BINARY_SPLIT_TRIM	0x02
 #define BINARY_SPLIT_TRIM_ALL	0x04
@@ -2571,7 +2576,6 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 	if (type == am_bm) {
 	    BMData *bm;
 	    Sint pos;
-	    Eterm ret;
 	    Eterm *hp;
 	    BMFindAllState state;
 	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
@@ -2591,8 +2595,7 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 	    pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
 	    if (pos == BM_NOT_FOUND) {
 		hp = HAlloc(p, 2);
-		ret = NIL;
-		ret = CONS(hp, subject, ret);
+		*res_term = CONS(hp, subject, NIL);
 	    } else if (pos == BM_RESTART) {
 		int x =
 		    (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
@@ -2610,196 +2613,16 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		bm_clean_find_all(&state);
 		return DO_BIN_MATCH_RESTART;
 	    } else {
-		size_t orig_size;
-		Eterm orig;
-		Uint offset;
-		Uint bit_offset;
-		Uint bit_size;
-		ErlSubBin *sb;
-		FindallData *fad = state.out;
-		Sint i, j;
-		Sint drop = 0;
-		Sint head = 0;
-		Sint tail;
-		Uint list_size;
-		Uint tail_pos;
-
-		tail = state.m - 1;
-		list_size = state.m + 1;
-		orig_size = binary_size(subject);
-		tail_pos = (Uint)(orig_size);
-
-		if (hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) {
-		    if ((orig_size - fad[tail].pos - fad[tail].len) == 0) {
-			list_size--;
-			for (i = (tail - 1); i >= 0; --i) {
-			    if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
-				break;
-			    }
-			    list_size--;
-			}
-			if (i == -1) {
-			    if (fad[head].pos == 0) {
-				ret = NIL;
-			    } else {
-				hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
-				ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-				sb = (ErlSubBin *)(hp);
-				sb->thing_word = HEADER_SUB_BIN;
-				sb->size = fad[head].pos;
-				sb->offs = offset;
-				sb->orig = orig;
-				sb->bitoffs = bit_offset;
-				sb->bitsize = bit_size;
-				sb->is_writable = 0;
-				fad[head].epos = make_binary(sb);
-				hp += ERL_SUB_BIN_SIZE;
-
-				ret = NIL;
-				ret = CONS(hp, make_binary(sb), ret);
-				hp += 2;
-			    }
-			    erts_free_aligned_binary_bytes(temp_alloc);
-			    bm_clean_find_all(&state);
-			    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-			    *res_term = ret;
-			    return DO_BIN_MATCH_OK;
-			}
-			tail = i;
-			tail_pos = fad[tail+1].pos;
-		    }
-		}
-		if (hsflags & BINARY_SPLIT_TRIM_ALL) {
-		    if (fad[head].pos == 0) {
-			drop++;
-			list_size--;
-			for (i = drop, j = tail; i <= j; ++i) {
-			    if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
-				break;
-			    }
-			    drop++;
-			    list_size--;
-			}
-			head = drop - 1;
-		    }
-		    for (i = (head+1), j = tail; i <= j; ++i) {
-			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) == 0) {
-			    list_size--;
-			}
-		    }
-
-		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
-		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-		    if (drop == 0) {
-			sb = (ErlSubBin *)(hp);
-			sb->thing_word = HEADER_SUB_BIN;
-			sb->size = fad[head].pos;
-			sb->offs = offset;
-			sb->orig = orig;
-			sb->bitoffs = bit_offset;
-			sb->bitsize = 0;
-			sb->is_writable = 0;
-			fad[head].epos = make_binary(sb);
-			hp += ERL_SUB_BIN_SIZE;
-		    }
-
-		    for (i = (head+1), j = tail; i <= j; ++i) {
-			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
-			    sb = (ErlSubBin *)(hp);
-			    sb->thing_word = HEADER_SUB_BIN;
-			    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
-			    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
-			    sb->orig = orig;
-			    sb->bitoffs = bit_offset;
-			    sb->bitsize = 0;
-			    sb->is_writable = 0;
-			    fad[i].epos = make_binary(sb);
-			    hp += ERL_SUB_BIN_SIZE;
-			}
-		    }
-
-		    sb = (ErlSubBin *)(hp);
-		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
-		    sb->offs = offset + fad[tail].pos + fad[tail].len;
-		    sb->orig = orig;
-		    sb->bitoffs = bit_offset;
-		    sb->bitsize = bit_size;
-		    sb->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    ret = NIL;
-		    ret = CONS(hp, make_binary(sb), ret);
-		    hp += 2;
-		    for (i = tail, j = head; i > j; --i) {
-			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
-			    ret = CONS(hp, fad[i].epos, ret);
-			    hp += 2;
-			}
-		    }
-		    if (drop == 0) {
-			ret = CONS(hp, fad[head].epos, ret);
-			hp += 2;
-		    }
-		} else {
-		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
-		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-		    sb = (ErlSubBin *)(hp);
-		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = fad[head].pos;
-		    sb->offs = offset;
-		    sb->orig = orig;
-		    sb->bitoffs = bit_offset;
-		    sb->bitsize = 0;
-		    sb->is_writable = 0;
-		    fad[head].epos = make_binary(sb);
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    for (i = (head+1), j = tail; i <= j; ++i) {
-			sb = (ErlSubBin *)(hp);
-			sb->thing_word = HEADER_SUB_BIN;
-			sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
-			sb->offs = offset + fad[i-1].pos + fad[i-1].len;
-			sb->orig = orig;
-			sb->bitoffs = bit_offset;
-			sb->bitsize = 0;
-			sb->is_writable = 0;
-			fad[i].epos = make_binary(sb);
-			hp += ERL_SUB_BIN_SIZE;
-		    }
-
-		    sb = (ErlSubBin *)(hp);
-		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
-		    sb->offs = offset + fad[tail].pos + fad[tail].len;
-		    sb->orig = orig;
-		    sb->bitoffs = bit_offset;
-		    sb->bitsize = bit_size;
-		    sb->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    ret = NIL;
-		    ret = CONS(hp, make_binary(sb), ret);
-		    hp += 2;
-		    for (i = tail, j = head; i >= j; --i) {
-			ret = CONS(hp, fad[i].epos, ret);
-			hp += 2;
-		    }
-		}
+                *res_term = do_split_global_result(p, state.out, state.m, subject, hsflags);
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    bm_clean_find_all(&state);
 	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	    *res_term = ret;
 	    return DO_BIN_MATCH_OK;
 	} else if (type == am_ac) {
 	    ACTrie *act;
 	    int acr;
 	    ACFindAllState state;
-	    Eterm ret;
 	    Eterm *hp;
 	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
 	    Uint save_reds = reds;
@@ -2817,8 +2640,7 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 	    acr = ac_find_all_non_overlapping(&state, bytes, &reds);
 	    if (acr == AC_NOT_FOUND) {
 		hp = HAlloc(p, 2);
-		ret = NIL;
-		ret = CONS(hp, subject, ret);
+		*res_term = CONS(hp, subject, NIL);
 	    } else if (acr == AC_RESTART) {
 		int x = (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
 		    !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
@@ -2835,197 +2657,17 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		ac_clean_find_all(&state);
 		return DO_BIN_MATCH_RESTART;
 	    } else {
-		size_t orig_size;
-		Eterm orig;
-		Uint offset;
-		Uint bit_offset;
-		Uint bit_size;
-		ErlSubBin *sb;
-		FindallData *fad = state.out;
-		Sint i, j;
-		Sint drop = 0;
-		Sint head = 0;
-		Sint tail;
-		Uint list_size;
-		Uint tail_pos;
-
-		tail = state.m - 1;
-		list_size = state.m + 1;
-		orig_size = binary_size(subject);
-		tail_pos = (Uint)(orig_size);
-
-		if (hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) {
-		    if ((orig_size - fad[tail].pos - fad[tail].len) == 0) {
-			list_size--;
-			for (i = (tail - 1); i >= 0; --i) {
-			    if ((fad[i+1].pos - fad[i].pos - fad[i].len) != 0) {
-				break;
-			    }
-			    list_size--;
-			}
-			if (i == -1) {
-			    if (fad[head].pos == 0) {
-				ret = NIL;
-			    } else {
-				hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
-				ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-				sb = (ErlSubBin *)(hp);
-				sb->thing_word = HEADER_SUB_BIN;
-				sb->size = fad[head].pos;
-				sb->offs = offset;
-				sb->orig = orig;
-				sb->bitoffs = bit_offset;
-				sb->bitsize = bit_size;
-				sb->is_writable = 0;
-				fad[head].epos = make_binary(sb);
-				hp += ERL_SUB_BIN_SIZE;
-
-				ret = NIL;
-				ret = CONS(hp, make_binary(sb), ret);
-				hp += 2;
-			    }
-			    erts_free_aligned_binary_bytes(temp_alloc);
-			    ac_clean_find_all(&state);
-			    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-			    *res_term = ret;
-			    return DO_BIN_MATCH_OK;
-			}
-			tail = i;
-			tail_pos = fad[tail+1].pos;
-		    }
-		}
-		if (hsflags & BINARY_SPLIT_TRIM_ALL) {
-		    if (fad[head].pos == 0) {
-			drop++;
-			list_size--;
-			for (i = drop, j = tail; i <= j; ++i) {
-			    if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
-				break;
-			    }
-			    drop++;
-			    list_size--;
-			}
-			head = drop - 1;
-		    }
-		    for (i = (head+1), j = tail; i <= j; ++i) {
-			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) == 0) {
-			    list_size--;
-			}
-		    }
-
-		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
-		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-		    if (drop == 0) {
-			sb = (ErlSubBin *)(hp);
-			sb->thing_word = HEADER_SUB_BIN;
-			sb->size = fad[head].pos;
-			sb->offs = offset;
-			sb->orig = orig;
-			sb->bitoffs = bit_offset;
-			sb->bitsize = 0;
-			sb->is_writable = 0;
-			fad[head].epos = make_binary(sb);
-			hp += ERL_SUB_BIN_SIZE;
-		    }
-
-		    for (i = (head+1), j = tail; i <= j; ++i) {
-			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
-			    sb = (ErlSubBin *)(hp);
-			    sb->thing_word = HEADER_SUB_BIN;
-			    sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
-			    sb->offs = offset + fad[i-1].pos + fad[i-1].len;
-			    sb->orig = orig;
-			    sb->bitoffs = bit_offset;
-			    sb->bitsize = 0;
-			    sb->is_writable = 0;
-			    fad[i].epos = make_binary(sb);
-			    hp += ERL_SUB_BIN_SIZE;
-			}
-		    }
-
-		    sb = (ErlSubBin *)(hp);
-		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
-		    sb->offs = offset + fad[tail].pos + fad[tail].len;
-		    sb->orig = orig;
-		    sb->bitoffs = bit_offset;
-		    sb->bitsize = bit_size;
-		    sb->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    ret = NIL;
-		    ret = CONS(hp, make_binary(sb), ret);
-		    hp += 2;
-		    for (i = tail, j = head; i > j; --i) {
-			if ((fad[i].pos - fad[i-1].pos - fad[i-1].len) != 0) {
-			    ret = CONS(hp, fad[i].epos, ret);
-			    hp += 2;
-			}
-		    }
-		    if (drop == 0) {
-			ret = CONS(hp, fad[head].epos, ret);
-			hp += 2;
-		    }
-		} else {
-		    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
-		    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-
-		    sb = (ErlSubBin *)(hp);
-		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = fad[head].pos;
-		    sb->offs = offset;
-		    sb->orig = orig;
-		    sb->bitoffs = bit_offset;
-		    sb->bitsize = 0;
-		    sb->is_writable = 0;
-		    fad[head].epos = make_binary(sb);
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    for (i = (head+1), j = tail; i <= j; ++i) {
-			sb = (ErlSubBin *)(hp);
-			sb->thing_word = HEADER_SUB_BIN;
-			sb->size = fad[i].pos - fad[i-1].pos - fad[i-1].len;
-			sb->offs = offset + fad[i-1].pos + fad[i-1].len;
-			sb->orig = orig;
-			sb->bitoffs = bit_offset;
-			sb->bitsize = 0;
-			sb->is_writable = 0;
-			fad[i].epos = make_binary(sb);
-			hp += ERL_SUB_BIN_SIZE;
-		    }
-
-		    sb = (ErlSubBin *)(hp);
-		    sb->thing_word = HEADER_SUB_BIN;
-		    sb->size = tail_pos - fad[tail].pos - fad[tail].len;
-		    sb->offs = offset + fad[tail].pos + fad[tail].len;
-		    sb->orig = orig;
-		    sb->bitoffs = bit_offset;
-		    sb->bitsize = bit_size;
-		    sb->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    ret = NIL;
-		    ret = CONS(hp, make_binary(sb), ret);
-		    hp += 2;
-		    for (i = tail, j = head; i >= j; --i) {
-			ret = CONS(hp, fad[i].epos, ret);
-			hp += 2;
-		    }
-		}
+                *res_term = do_split_global_result(p, state.out, state.m, subject, hsflags);
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    ac_clean_find_all(&state);
 	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	    *res_term = ret;
 	    return DO_BIN_MATCH_OK;
 	}
     } else {
 	if (type == am_bm) {
 	    BMData *bm;
 	    Sint pos;
-	    Eterm ret;
 	    Eterm *hp;
 	    BMFindFirstState state;
 	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
@@ -3049,8 +2691,7 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 	    pos = bm_find_first_match(&state, bm, bytes, &reds);
 	    if (pos == BM_NOT_FOUND) {
 		hp = HAlloc(p, 2);
-		ret = NIL;
-		ret = CONS(hp, subject, ret);
+		*res_term = CONS(hp, subject, NIL);
 	    } else if (pos == BM_RESTART) {
 		int x =
 		    (sizeof(state) / sizeof(Eterm)) +
@@ -3067,84 +2708,16 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		erts_free_aligned_binary_bytes(temp_alloc);
 		return DO_BIN_MATCH_RESTART;
 	    } else {
-		size_t orig_size;
-		Eterm orig;
-		Uint offset;
-		Uint bit_offset;
-		Uint bit_size;
-		ErlSubBin *sb1;
-		ErlSubBin *sb2;
-
-		orig_size = binary_size(subject);
-
-		if ((hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && (orig_size - pos - bm->len) == 0) {
-		    if (pos == 0) {
-			ret = NIL;
-		    } else {
-			hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
-			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-			sb1 = (ErlSubBin *) hp;
-			sb1->thing_word = HEADER_SUB_BIN;
-			sb1->size = pos;
-			sb1->offs = offset;
-			sb1->orig = orig;
-			sb1->bitoffs = bit_offset;
-			sb1->bitsize = bit_size;
-			sb1->is_writable = 0;
-			hp += ERL_SUB_BIN_SIZE;
-
-			ret = NIL;
-			ret = CONS(hp, make_binary(sb1), ret);
-			hp += 2;
-		    }
-		} else {
-		    if ((hsflags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
-			hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2));
-			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-			sb1 = NULL;
-		    } else {
-			hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
-			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-			sb1 = (ErlSubBin *) hp;
-			sb1->thing_word = HEADER_SUB_BIN;
-			sb1->size = pos;
-			sb1->offs = offset;
-			sb1->orig = orig;
-			sb1->bitoffs = bit_offset;
-			sb1->bitsize = 0;
-			sb1->is_writable = 0;
-			hp += ERL_SUB_BIN_SIZE;
-		    }
-
-		    sb2 = (ErlSubBin *) hp;
-		    sb2->thing_word = HEADER_SUB_BIN;
-		    sb2->size = orig_size - pos - bm->len;
-		    sb2->offs = offset + pos + bm->len;
-		    sb2->orig = orig;
-		    sb2->bitoffs = bit_offset;
-		    sb2->bitsize = bit_size;
-		    sb2->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    ret = NIL;
-		    ret = CONS(hp, make_binary(sb2), ret);
-		    hp += 2;
-		    if (sb1 != NULL) {
-			ret = CONS(hp, make_binary(sb1), ret);
-			hp += 2;
-		    }
-		}
+		*res_term = do_split_single_result(p, subject, pos, bm->len, hsflags);
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	    *res_term = ret;
 	    return DO_BIN_MATCH_OK;
 	} else if (type == am_ac) {
 	    ACTrie *act;
 	    Uint pos, rlen;
 	    int acr;
 	    ACFindFirstState state;
-	    Eterm ret;
 	    Eterm *hp;
 	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
 	    Uint save_reds = reds;
@@ -3162,8 +2735,7 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 	    acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
 	    if (acr == AC_NOT_FOUND) {
 		hp = HAlloc(p, 2);
-		ret = NIL;
-		ret = CONS(hp, subject, ret);
+		*res_term = CONS(hp, subject, NIL);
 	    } else if (acr == AC_RESTART) {
 		int x =
 		    (sizeof(state) / sizeof(Eterm)) +
@@ -3180,77 +2752,10 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
 		erts_free_aligned_binary_bytes(temp_alloc);
 		return DO_BIN_MATCH_RESTART;
 	    } else {
-		size_t orig_size;
-		Eterm orig;
-		Uint offset;
-		Uint bit_offset;
-		Uint bit_size;
-		ErlSubBin *sb1;
-		ErlSubBin *sb2;
-
-		orig_size = binary_size(subject);
-
-		if ((hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && (orig_size - pos - rlen) == 0) {
-		    if (pos == 0) {
-			ret = NIL;
-		    } else {
-			hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
-			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-			sb1 = (ErlSubBin *) hp;
-			sb1->thing_word = HEADER_SUB_BIN;
-			sb1->size = pos;
-			sb1->offs = offset;
-			sb1->orig = orig;
-			sb1->bitoffs = bit_offset;
-			sb1->bitsize = bit_size;
-			sb1->is_writable = 0;
-			hp += ERL_SUB_BIN_SIZE;
-
-			ret = NIL;
-			ret = CONS(hp, make_binary(sb1), ret);
-			hp += 2;
-		    }
-		} else {
-		    if ((hsflags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
-			hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
-			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-			sb1 = NULL;
-		    } else {
-			hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
-			ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-			sb1 = (ErlSubBin *) hp;
-			sb1->thing_word = HEADER_SUB_BIN;
-			sb1->size = pos;
-			sb1->offs = offset;
-			sb1->orig = orig;
-			sb1->bitoffs = bit_offset;
-			sb1->bitsize = 0;
-			sb1->is_writable = 0;
-			hp += ERL_SUB_BIN_SIZE;
-		    }
-
-		    sb2 = (ErlSubBin *) hp;
-		    sb2->thing_word = HEADER_SUB_BIN;
-		    sb2->size = orig_size - pos - rlen;
-		    sb2->offs = offset + pos + rlen;
-		    sb2->orig = orig;
-		    sb2->bitoffs = bit_offset;
-		    sb2->bitsize = bit_size;
-		    sb2->is_writable = 0;
-		    hp += ERL_SUB_BIN_SIZE;
-
-		    ret = NIL;
-		    ret = CONS(hp, make_binary(sb2), ret);
-		    hp += 2;
-		    if (sb1 != NULL) {
-			ret = CONS(hp, make_binary(sb1), ret);
-			hp += 2;
-		    }
-		}
+		*res_term = do_split_single_result(p, subject, pos, rlen, hsflags);
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	    *res_term = ret;
 	    return DO_BIN_MATCH_OK;
 	}
     }
@@ -3258,6 +2763,144 @@ static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
     return DO_BIN_MATCH_BADARG;
 }
 
+static Eterm do_split_single_result(Process* p, Eterm subject,
+				    Sint pos, Sint len, Uint hsflags)
+{
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    ErlSubBin *sb1;
+    ErlSubBin *sb2;
+    Eterm* hp;
+    Eterm ret;
+
+    orig_size = binary_size(subject);
+
+    if ((hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && (orig_size - pos - len) == 0) {
+	if (pos != 0) {
+	    ret = NIL;
+	} else {
+	    hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+	    sb1 = (ErlSubBin *) hp;
+	    sb1->thing_word = HEADER_SUB_BIN;
+	    sb1->size = pos;
+	    sb1->offs = offset;
+	    sb1->orig = orig;
+	    sb1->bitoffs = bit_offset;
+	    sb1->bitsize = bit_size;
+	    sb1->is_writable = 0;
+	    hp += ERL_SUB_BIN_SIZE;
+
+	    ret = CONS(hp, make_binary(sb1), NIL);
+	    hp += 2;
+	}
+    } else {
+	if ((hsflags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
+	    hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2));
+	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+	    sb1 = NULL;
+	} else {
+	    hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
+	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+	    sb1 = (ErlSubBin *) hp;
+	    sb1->thing_word = HEADER_SUB_BIN;
+	    sb1->size = pos;
+	    sb1->offs = offset;
+	    sb1->orig = orig;
+	    sb1->bitoffs = bit_offset;
+	    sb1->bitsize = 0;
+	    sb1->is_writable = 0;
+	    hp += ERL_SUB_BIN_SIZE;
+	}
+
+	sb2 = (ErlSubBin *) hp;
+	sb2->thing_word = HEADER_SUB_BIN;
+	sb2->size = orig_size - pos - len;
+	sb2->offs = offset + pos + len;
+	sb2->orig = orig;
+	sb2->bitoffs = bit_offset;
+	sb2->bitsize = bit_size;
+	sb2->is_writable = 0;
+	hp += ERL_SUB_BIN_SIZE;
+
+	ret = CONS(hp, make_binary(sb2), NIL);
+	hp += 2;
+	if (sb1 != NULL) {
+	    ret = CONS(hp, make_binary(sb1), ret);
+	    hp += 2;
+	}
+    }
+    return ret;
+}
+
+static Eterm do_split_global_result(Process* p, FindallData *fad, Uint fad_sz,
+				    Uint subject, Uint hsflags)
+{
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    ErlSubBin *sb;
+    Sint i;
+    Sint tail;
+    Uint list_size;
+    Uint end_pos;
+    Uint do_trim = hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL);
+    Eterm* hp;
+    Eterm* hendp;
+    Eterm ret;
+
+    tail = fad_sz - 1;
+    list_size = fad_sz + 1;
+    orig_size = binary_size(subject);
+    end_pos = (Uint)(orig_size);
+
+    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
+    hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2);
+    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+    ASSERT(bit_size == 0);
+
+    ret = NIL;
+
+    for (i = tail; i >= 0; --i) {
+        sb = (ErlSubBin *)(hp);
+        sb->size = end_pos - (fad[i].pos + fad[i].len);
+        if (!(sb->size == 0 && do_trim)) {
+            sb->thing_word = HEADER_SUB_BIN;
+            sb->offs = offset + fad[i].pos + fad[i].len;
+            sb->orig = orig;
+            sb->bitoffs = bit_offset;
+            sb->bitsize = 0;
+            sb->is_writable = 0;
+            hp += ERL_SUB_BIN_SIZE;
+            ret = CONS(hp, make_binary(sb), ret);
+            hp += 2;
+            do_trim &= ~BINARY_SPLIT_TRIM;
+        }
+	end_pos = fad[i].pos;
+    }
+
+    sb = (ErlSubBin *)(hp);
+    sb->size = fad[0].pos;
+    if (!(sb->size == 0 && do_trim)) {
+        sb->thing_word = HEADER_SUB_BIN;
+        sb->offs = offset;
+        sb->orig = orig;
+        sb->bitoffs = bit_offset;
+        sb->bitsize = 0;
+        sb->is_writable = 0;
+        hp += ERL_SUB_BIN_SIZE;
+        ret = CONS(hp, make_binary(sb), ret);
+        hp += 2;
+    }
+    HRelease(p, hendp, hp);
+    return ret;
+}
+
 static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uint *optp)
 {
     Eterm *tp;
-- 
cgit v1.2.3


From 55777538791419a6c3d86c1c440c7eb7fbdd5e51 Mon Sep 17 00:00:00 2001
From: Andrew Bennett <potatosaladx@gmail.com>
Date: Thu, 10 Sep 2015 13:40:21 -0600
Subject: erts: Refactor BIF for binary:match,matches,split

with an common do_binary_find() used by match, matches and split.
---
 erts/emulator/beam/atom.names       |    4 +-
 erts/emulator/beam/erl_bif_binary.c | 1659 ++++++++++++++---------------------
 2 files changed, 678 insertions(+), 985 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 5f27aaa14b..7a50b24818 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -117,11 +117,9 @@ atom bif_timer_server
 atom binary
 atom binary_bin_to_list_trap
 atom binary_copy_trap
+atom binary_find_trap
 atom binary_longest_prefix_trap
 atom binary_longest_suffix_trap
-atom binary_match_trap
-atom binary_matches_trap
-atom binary_split_trap
 atom binary_to_list_continue
 atom binary_to_term_trap
 atom block
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 6adc61df19..9f72b8c0ac 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -55,10 +55,8 @@
 
 /* Init and local variables */
 
-static Export binary_match_trap_export;
-static BIF_RETTYPE binary_match_trap(BIF_ALIST_3);
-static Export binary_matches_trap_export;
-static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3);
+static Export binary_find_trap_export;
+static BIF_RETTYPE binary_find_trap(BIF_ALIST_3);
 static Export binary_longest_prefix_trap_export;
 static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3);
 static Export binary_longest_suffix_trap_export;
@@ -67,26 +65,18 @@ static Export binary_bin_to_list_trap_export;
 static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3);
 static Export binary_copy_trap_export;
 static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2);
-static Export binary_split_trap_export;
-static BIF_RETTYPE binary_split_trap(BIF_ALIST_3);
 static Uint max_loop_limit;
 
 static BIF_RETTYPE
-binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
-static BIF_RETTYPE
-binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
+binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags);
 static BIF_RETTYPE
 binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
 
 void erts_init_bif_binary(void)
 {
-    erts_init_trap_export(&binary_match_trap_export,
-			  am_erlang, am_binary_match_trap, 3,
-			  &binary_match_trap);
-
-    erts_init_trap_export(&binary_matches_trap_export,
-			  am_erlang, am_binary_matches_trap, 3,
-			  &binary_matches_trap);
+    erts_init_trap_export(&binary_find_trap_export,
+			  am_erlang, am_binary_find_trap, 3,
+			  &binary_find_trap);
 
     erts_init_trap_export(&binary_longest_prefix_trap_export,
 			  am_erlang, am_binary_longest_prefix_trap, 3,
@@ -104,10 +94,6 @@ void erts_init_bif_binary(void)
 			  am_erlang, am_binary_copy_trap, 2,
 			  &binary_copy_trap);
 
-    erts_init_trap_export(&binary_split_trap_export,
-			  am_erlang, am_binary_split_trap, 3,
-			  &binary_split_trap);
-
     max_loop_limit = 0;
     return;
 }
@@ -322,8 +308,8 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len,
 /*
  * Aho Corasick - Build a Trie and fill in the failure functions
  * when all strings are added.
- * The algorithm is nicely described by Dieter Bühler of University of
- * Tübingen:
+ * The algorithm is nicely described by Dieter Bühler of University of
+ * Tübingen:
  * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html
  */
 
@@ -573,9 +559,6 @@ static void ac_clean_find_all(ACFindAllState *state)
 #endif
 }
 
-#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \
-          (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m))
-
 /*
  * Differs to the find_first function in that it stores all matches and the values
  * arte returned only in the state.
@@ -853,9 +836,6 @@ static void bm_clean_find_all(BMFindAllState *state)
 #endif
 }
 
-#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \
-          (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m))
-
 /*
  * Differs to the find_first function in that it stores all matches and the
  * values are returned only in the state.
@@ -1038,267 +1018,36 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1)
 #define DO_BIN_MATCH_BADARG -1
 #define DO_BIN_MATCH_RESTART -2
 
-static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend,
-			   Eterm type, Binary *bin, Eterm state_term,
-			   Eterm *res_term)
-{
-    byte *bytes;
-    Uint bitoffs, bitsize;
-    byte *temp_alloc = NULL;
-
-    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
-    if (bitsize != 0) {
-	goto badarg;
-    }
-    if (bitoffs != 0) {
-	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
-    }
-    if (state_term != NIL) {
-	Eterm *ptr = big_val(state_term);
-	type = ptr[1];
-    }
-
-    if (type == am_bm) {
-	BMData *bm;
-	Sint pos;
-	Eterm ret;
-	Eterm *hp;
-	BMFindFirstState state;
-	Uint reds = get_reds(p, BM_LOOP_FACTOR);
-	Uint save_reds = reds;
-
-	bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	dump_bm_data(bm);
-#endif
-	if (state_term == NIL) {
-	    bm_init_find_first_match(&state, hsstart, hsend);
-	} else {
-	    Eterm *ptr = big_val(state_term);
-	    memcpy(&state,ptr+2,sizeof(state));
-	}
-#ifdef HARDDEBUG
-	erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos,
-		    state.len);
-#endif
-	pos = bm_find_first_match(&state, bm, bytes, &reds);
-	if (pos == BM_NOT_FOUND) {
-	    ret = am_nomatch;
-	} else if (pos == BM_RESTART) {
-	    int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) +
-		!!(sizeof(BMFindFirstState) % sizeof(Eterm));
-#ifdef HARDDEBUG
-	    erts_printf("Trap bm!\n");
-#endif
-	    hp = HAlloc(p,x+2);
-	    hp[0] = make_pos_bignum_header(x+1);
-	    hp[1] = type;
-	    memcpy(hp+2,&state,sizeof(state));
-	    *res_term = make_big(hp);
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    return DO_BIN_MATCH_RESTART;
-	} else {
-	    Eterm erlen = erts_make_integer((Uint) bm->len, p);
-	    ret = erts_make_integer(pos,p);
-	    hp = HAlloc(p,3);
-	    ret = TUPLE2(hp, ret, erlen);
-	}
-	erts_free_aligned_binary_bytes(temp_alloc);
-	BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	*res_term = ret;
-	return DO_BIN_MATCH_OK;
-    } else if (type == am_ac) {
-	ACTrie *act;
-	Uint pos, rlen;
-	int acr;
-	ACFindFirstState state;
-	Eterm ret;
-	Eterm *hp;
-	Uint reds = get_reds(p, AC_LOOP_FACTOR);
-	Uint save_reds = reds;
-
-	act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	dump_ac_trie(act);
-#endif
-	if (state_term == NIL) {
-	    ac_init_find_first_match(&state, act, hsstart, hsend);
-	} else {
-	    Eterm *ptr = big_val(state_term);
-	    memcpy(&state,ptr+2,sizeof(state));
-	}
-	acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
-	if (acr == AC_NOT_FOUND) {
-	    ret = am_nomatch;
-	} else if (acr == AC_RESTART) {
-	    int x = (sizeof(state) / sizeof(Eterm)) +
-		!!(sizeof(ACFindFirstState) % sizeof(Eterm));
-#ifdef HARDDEBUG
-	    erts_printf("Trap ac!\n");
-#endif
-	    hp = HAlloc(p,x+2);
-	    hp[0] = make_pos_bignum_header(x+1);
-	    hp[1] = type;
-	    memcpy(hp+2,&state,sizeof(state));
-	    *res_term = make_big(hp);
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    return DO_BIN_MATCH_RESTART;
-	} else {
-	    Eterm epos = erts_make_integer(pos,p);
-	    Eterm erlen = erts_make_integer(rlen,p);
-	    hp = HAlloc(p,3);
-	    ret = TUPLE2(hp, epos, erlen);
-	}
-	erts_free_aligned_binary_bytes(temp_alloc);
-	BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	*res_term = ret;
-	return DO_BIN_MATCH_OK;
-    }
- badarg:
-    return DO_BIN_MATCH_BADARG;
-}
-
-static int do_binary_matches(Process *p, Eterm subject, Uint hsstart,
-			     Uint hsend, Eterm type, Binary *bin,
-			     Eterm state_term, Eterm *res_term)
-{
-    byte *bytes;
-    Uint bitoffs, bitsize;
-    byte *temp_alloc = NULL;
-
-    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
-    if (bitsize != 0) {
-	goto badarg;
-    }
-    if (bitoffs != 0) {
-	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
-    }
-    if (state_term != NIL) {
-	Eterm *ptr = big_val(state_term);
-	type = ptr[1];
-    }
-
-    if (type == am_bm) {
-	BMData *bm;
-	Sint pos;
-	Eterm ret,tpl;
-	Eterm *hp;
-	BMFindAllState state;
-	Uint reds = get_reds(p, BM_LOOP_FACTOR);
-	Uint save_reds = reds;
-
-	bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	dump_bm_data(bm);
-#endif
-	if (state_term == NIL) {
-	    bm_init_find_all(&state, hsstart, hsend);
-	} else {
-	    Eterm *ptr = big_val(state_term);
-	    bm_restore_find_all(&state,(char *) (ptr+2));
-	}
-
-	pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
-	if (pos == BM_NOT_FOUND) {
-	    ret = NIL;
-	} else if (pos == BM_RESTART) {
-	    int x =
-		(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
-		!!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-	    erts_printf("Trap bm!\n");
-#endif
-	    hp = HAlloc(p,x+2);
-	    hp[0] = make_pos_bignum_header(x+1);
-	    hp[1] = type;
-	    bm_serialize_find_all(&state, (char *) (hp+2));
-	    *res_term = make_big(hp);
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    bm_clean_find_all(&state);
-	    return DO_BIN_MATCH_RESTART;
-	} else {
-	    FindallData *fad = state.out;
-	    int i;
-	    for (i = 0; i < state.m; ++i) {
-		fad[i].epos = erts_make_integer(fad[i].pos,p);
-		fad[i].elen = erts_make_integer(fad[i].len,p);
-	    }
-	    hp = HAlloc(p,state.m * (3 + 2));
-	    ret = NIL;
-	    for (i = state.m - 1; i >= 0; --i) {
-		tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
-		hp +=3;
-		ret = CONS(hp,tpl,ret);
-		hp += 2;
-	    }
-	}
-	erts_free_aligned_binary_bytes(temp_alloc);
-	bm_clean_find_all(&state);
-	BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	*res_term = ret;
-	return DO_BIN_MATCH_OK;
-    } else if (type == am_ac) {
-	ACTrie *act;
-	int acr;
-	ACFindAllState state;
-	Eterm ret,tpl;
-	Eterm *hp;
-	Uint reds = get_reds(p, AC_LOOP_FACTOR);
-	Uint save_reds = reds;
+#define BINARY_FIND_ALL		0x01
+#define BINARY_SPLIT_TRIM	0x02
+#define BINARY_SPLIT_TRIM_ALL	0x04
 
-	act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	dump_ac_trie(act);
-#endif
-	if (state_term == NIL) {
-	    ac_init_find_all(&state, act, hsstart, hsend);
-	} else {
-	    Eterm *ptr = big_val(state_term);
-	    ac_restore_find_all(&state,(char *) (ptr+2));
-	}
-	acr = ac_find_all_non_overlapping(&state, bytes, &reds);
-	if (acr == AC_NOT_FOUND) {
-	    ret = NIL;
-	} else if (acr == AC_RESTART) {
-	    int x =
-		(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
-		!!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-	    erts_printf("Trap ac!\n");
-#endif
-	    hp = HAlloc(p,x+2);
-	    hp[0] = make_pos_bignum_header(x+1);
-	    hp[1] = type;
-	    ac_serialize_find_all(&state, (char *) (hp+2));
-	    *res_term = make_big(hp);
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    ac_clean_find_all(&state);
-	    return DO_BIN_MATCH_RESTART;
-	} else {
-	    FindallData *fad = state.out;
-	    int i;
-	    for (i = 0; i < state.m; ++i) {
-		fad[i].epos = erts_make_integer(fad[i].pos,p);
-		fad[i].elen = erts_make_integer(fad[i].len,p);
-	    }
-	    hp = HAlloc(p,state.m * (3 + 2));
-	    ret = NIL;
-	    for (i = state.m - 1; i >= 0; --i) {
-		tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
-		hp +=3;
-		ret = CONS(hp,tpl,ret);
-		hp += 2;
-	    }
-	}
-	erts_free_aligned_binary_bytes(temp_alloc);
-	ac_clean_find_all(&state);
-	BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	*res_term = ret;
-	return DO_BIN_MATCH_OK;
-    }
- badarg:
-    return DO_BIN_MATCH_BADARG;
-}
+typedef struct BinaryFindState {
+    Eterm type;
+    Uint  flags;
+    Uint  hsstart;
+    Uint  hsend;
+    Eterm (*not_found_result) (Process *, Eterm, struct BinaryFindState *);
+    Eterm (*single_result)    (Process *, Eterm, struct BinaryFindState *, Sint, Sint);
+    Eterm (*global_result)    (Process *, Eterm, struct BinaryFindState *, FindallData *, Uint);
+} BinaryFindState;
+
+#define SIZEOF_BINARY_FIND_STATE(S) \
+	  (sizeof(BinaryFindState)+sizeof(S))
+
+#define SIZEOF_BINARY_FIND_ALL_STATE(S) \
+	  (sizeof(BinaryFindState)+sizeof(S)+(sizeof(FindallData)*(S).m))
+
+static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs);
+static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    Sint pos, Sint len);
+static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    FindallData *fad, Uint fad_sz);
+static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs);
+static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    Sint pos, Sint len);
+static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    FindallData *fad, Uint fad_sz);
 
 static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
 {
@@ -1363,116 +1112,298 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
     }
 }
 
-static BIF_RETTYPE binary_match_trap(BIF_ALIST_3)
-{
-    int runres;
-    Eterm result;
-    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
-    runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result);
-    if (runres == DO_BIN_MATCH_OK) {
-	BIF_RET(result);
-    } else {
-	BUMP_ALL_REDS(BIF_P);
-	BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result,
-		  BIF_ARG_3);
-    }
-}
-
-static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3)
+static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uint *optp)
 {
-    int runres;
-    Eterm result;
-    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
-    runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result);
-    if (runres == DO_BIN_MATCH_OK) {
-	BIF_RET(result);
+    Eterm *tp;
+    Uint pos;
+    Sint len;
+    *optp = 0;
+    *posp = 0;
+    *endp = binary_size(bin);
+    if (l == THE_NON_VALUE || l == NIL) {
+	return 0;
+    } else if (is_list(l)) {
+	while(is_list(l)) {
+	    Eterm t = CAR(list_val(l));
+	    Uint orig_size;
+	    if (is_atom(t)) {
+		if (t == am_global) {
+		    *optp |= BINARY_FIND_ALL;
+		    l = CDR(list_val(l));
+		    continue;
+		}
+		if (t == am_trim) {
+		    *optp |= BINARY_SPLIT_TRIM;
+		    l = CDR(list_val(l));
+		    continue;
+		}
+		if (t == am_trim_all) {
+		    *optp |= BINARY_SPLIT_TRIM_ALL;
+		    l = CDR(list_val(l));
+		    continue;
+		}
+	    }
+	    if (!is_tuple(t)) {
+		goto badarg;
+	    }
+	    tp = tuple_val(t);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (tp[1] != am_scope || is_not_tuple(tp[2])) {
+		goto badarg;
+	    }
+	    tp = tuple_val(tp[2]);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (!term_to_Uint(tp[1], &pos)) {
+		goto badarg;
+	    }
+	    if (!term_to_Sint(tp[2], &len)) {
+		goto badarg;
+	    }
+	    if (len < 0) {
+		Uint lentmp = -(Uint)len;
+		/* overflow */
+		if ((Sint)lentmp < 0) {
+		    goto badarg;
+		}
+		len = lentmp;
+		pos -= len;
+	    }
+	    /* overflow */
+	    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+		goto badarg;
+	    }
+	    *endp = len + pos;
+	    *posp = pos;
+	    if ((orig_size = binary_size(bin)) < pos ||
+		orig_size < (*endp)) {
+		goto badarg;
+	    }
+	    l = CDR(list_val(l));
+	}
+	return 0;
     } else {
-	BUMP_ALL_REDS(BIF_P);
-	BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result,
-		  BIF_ARG_3);
+    badarg:
+	return 1;
     }
 }
 
-BIF_RETTYPE binary_match_3(BIF_ALIST_3)
-{
-    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
-}
-
-static BIF_RETTYPE
-binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
+static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binary *bin,
+			  Eterm state_term, Eterm *res_term)
 {
-    Uint hsstart;
-    Uint hsend;
-    Eterm *tp;
-    Eterm type;
-    Binary *bin;
-    Eterm bin_term = NIL;
-    int runres;
-    Eterm result;
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    byte *temp_alloc = NULL;
+    char *state_ptr = NULL;
 
-    if (is_not_binary(arg1)) {
-	goto badarg;
-    }
-    if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) {
-	goto badarg;
-    }
-    if (hsend == 0) {
-	BIF_RET(am_nomatch);
-    }
-    if (is_tuple(arg2)) {
-	tp = tuple_val(arg2);
-	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
-	    goto badarg;
-	}
-	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
-	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
-	    goto badarg;
-	}
-	type = tp[1];
-	bin = ((ProcBin *) binary_val(tp[2]))->val;
-	if (type == am_bm &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
-	    goto badarg;
-	}
-	if (type == am_ac &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
-	    goto badarg;
-	}
-	bin_term = tp[2];
-    } else if (do_binary_match_compile(arg2,&type,&bin)) {
+    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
 	goto badarg;
     }
-    runres = do_binary_match(p,arg1,hsstart,hsend,type,bin,NIL,&result);
-    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
-	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
-	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
-    } else if (bin_term == NIL) {
-	erts_bin_free(bin);
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
     }
-    switch (runres) {
-    case DO_BIN_MATCH_OK:
-	BIF_RET(result);
-    case DO_BIN_MATCH_RESTART:
-	BUMP_ALL_REDS(p);
-	BIF_TRAP3(&binary_match_trap_export, p, arg1, result, bin_term);
-    default:
-	goto badarg;
+    if (state_term != NIL) {
+	state_ptr = (char *)(big_val(state_term));
+	state_ptr += sizeof(Eterm);
+	bfs = (BinaryFindState *)(state_ptr);
+	state_ptr += sizeof(BinaryFindState);
     }
- badarg:
-    BIF_ERROR(p,BADARG);
-}
 
-BIF_RETTYPE binary_matches_3(BIF_ALIST_3)
-{
-    return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+    if (bfs->flags & BINARY_FIND_ALL) {
+	if (bfs->type == am_bm) {
+	    BMData *bm;
+	    Sint pos;
+	    Eterm *hp;
+	    BMFindAllState state;
+	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_bm_data(bm);
+#endif
+	    if (state_term == NIL) {
+		bm_init_find_all(&state, bfs->hsstart, bfs->hsend);
+	    } else {
+		bm_restore_find_all(&state, state_ptr);
+	    }
+
+	    pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
+	    if (pos == BM_NOT_FOUND) {
+		*res_term = bfs->not_found_result(p, subject, bfs);
+	    } else if (pos == BM_RESTART) {
+		int x =
+		    (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		    !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap bm!\n");
+#endif
+		hp = HAlloc(p, x+1);
+		hp[0] = make_pos_bignum_header(x);
+		state_ptr = (char *)(hp);
+		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
+		bm_serialize_find_all(&state, state_ptr+sizeof(Eterm)+sizeof(BinaryFindState));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		bm_clean_find_all(&state);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		*res_term = bfs->global_result(p, subject, bfs, state.out, state.m);
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    bm_clean_find_all(&state);
+	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	    return DO_BIN_MATCH_OK;
+	} else if (bfs->type == am_ac) {
+	    ACTrie *act;
+	    int acr;
+	    ACFindAllState state;
+	    Eterm *hp;
+	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_ac_trie(act);
+#endif
+	    if (state_term == NIL) {
+		ac_init_find_all(&state, act, bfs->hsstart, bfs->hsend);
+	    } else {
+		ac_restore_find_all(&state, state_ptr);
+	    }
+	    acr = ac_find_all_non_overlapping(&state, bytes, &reds);
+	    if (acr == AC_NOT_FOUND) {
+		*res_term = bfs->not_found_result(p, subject, bfs);
+	    } else if (acr == AC_RESTART) {
+		int x =
+		    (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		    !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap ac!\n");
+#endif
+		hp = HAlloc(p, x+1);
+		hp[0] = make_pos_bignum_header(x);
+		state_ptr = (char *)(hp);
+		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
+		ac_serialize_find_all(&state, state_ptr+sizeof(Eterm)+sizeof(BinaryFindState));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		ac_clean_find_all(&state);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		*res_term = bfs->global_result(p, subject, bfs, state.out, state.m);
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    ac_clean_find_all(&state);
+	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	    return DO_BIN_MATCH_OK;
+	}
+    } else {
+	if (bfs->type == am_bm) {
+	    BMData *bm;
+	    Sint pos;
+	    Eterm *hp;
+	    BMFindFirstState state;
+	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_bm_data(bm);
+#endif
+	    if (state_term == NIL) {
+		bm_init_find_first_match(&state, bfs->hsstart, bfs->hsend);
+	    } else {
+		memcpy((void *)(&state), (const void *)(state_ptr), sizeof(BMFindFirstState));
+	    }
+
+#ifdef HARDDEBUG
+	    erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos,
+			state.len);
+#endif
+	    pos = bm_find_first_match(&state, bm, bytes, &reds);
+	    if (pos == BM_NOT_FOUND) {
+		*res_term = bfs->not_found_result(p, subject, bfs);
+	    } else if (pos == BM_RESTART) {
+		int x =
+		    (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) +
+		    !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap bm!\n");
+#endif
+		hp = HAlloc(p, x+1);
+		hp[0] = make_pos_bignum_header(x);
+		state_ptr = (char *)(hp);
+		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
+		memcpy((void *)(state_ptr+sizeof(Eterm)+sizeof(BinaryFindState)),
+		       (const void *)(&state), sizeof(BMFindFirstState));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		*res_term = bfs->single_result(p, subject, bfs, pos, bm->len);
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	    return DO_BIN_MATCH_OK;
+	} else if (bfs->type == am_ac) {
+	    ACTrie *act;
+	    Uint pos, rlen;
+	    int acr;
+	    ACFindFirstState state;
+	    Eterm *hp;
+	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	    Uint save_reds = reds;
+
+	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	    dump_ac_trie(act);
+#endif
+	    if (state_term == NIL) {
+		ac_init_find_first_match(&state, act, bfs->hsstart, bfs->hsend);
+	    } else {
+		memcpy((void *)(&state), (const void *)(state_ptr), sizeof(ACFindFirstState));
+	    }
+	    acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
+	    if (acr == AC_NOT_FOUND) {
+		*res_term = bfs->not_found_result(p, subject, bfs);
+	    } else if (acr == AC_RESTART) {
+		int x =
+		    (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) +
+		    !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+		erts_printf("Trap ac!\n");
+#endif
+		hp = HAlloc(p, x+1);
+		hp[0] = make_pos_bignum_header(x);
+		state_ptr = (char *)(hp);
+		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
+		memcpy((void *)(state_ptr+sizeof(Eterm)+sizeof(BinaryFindState)),
+		       (const void *)(&state), sizeof(ACFindFirstState));
+		*res_term = make_big(hp);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		return DO_BIN_MATCH_RESTART;
+	    } else {
+		*res_term = bfs->single_result(p, subject, bfs, pos, rlen);
+	    }
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	    return DO_BIN_MATCH_OK;
+	}
+    }
+ badarg:
+    return DO_BIN_MATCH_BADARG;
 }
 
 static BIF_RETTYPE
-binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
+binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags)
 {
-    Uint hsstart, hsend;
+    BinaryFindState bfs;
     Eterm *tp;
-    Eterm type;
     Binary *bin;
     Eterm bin_term = NIL;
     int runres;
@@ -1481,11 +1412,12 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
     if (is_not_binary(arg1)) {
 	goto badarg;
     }
-    if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) {
+    bfs.flags = flags;
+    if (parse_match_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend))) {
 	goto badarg;
     }
-    if (hsend == 0) {
-	BIF_RET(NIL);
+    if (bfs.hsend == 0) {
+	BIF_RET(do_match_not_found_result(p, arg1, &bfs));
     }
     if (is_tuple(arg2)) {
 	tp = tuple_val(arg2);
@@ -1496,22 +1428,24 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
 	    goto badarg;
 	}
-	type = tp[1];
+	bfs.type = tp[1];
 	bin = ((ProcBin *) binary_val(tp[2]))->val;
-	if (type == am_bm &&
+	if (bfs.type == am_bm &&
 	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
 	    goto badarg;
 	}
-	if (type == am_ac &&
+	if (bfs.type == am_ac &&
 	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
 	    goto badarg;
 	}
 	bin_term = tp[2];
-    } else if (do_binary_match_compile(arg2,&type,&bin)) {
+    } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) {
 	goto badarg;
     }
-    runres = do_binary_matches(p,arg1,hsstart,hsend,type,bin,
-			       NIL,&result);
+    bfs.not_found_result = &do_match_not_found_result;
+    bfs.single_result = &do_match_single_result;
+    bfs.global_result = &do_match_global_result;
+    runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result);
     if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
 	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
 	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
@@ -1523,8 +1457,7 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	BIF_RET(result);
     case DO_BIN_MATCH_RESTART:
 	BUMP_ALL_REDS(p);
-	BIF_TRAP3(&binary_matches_trap_export, p, arg1, result,
-		  bin_term);
+	BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term);
     default:
 	goto badarg;
     }
@@ -1532,98 +1465,392 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
     BIF_ERROR(p,BADARG);
 }
 
-
 BIF_RETTYPE binary_match_2(BIF_ALIST_2)
 {
-    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE);
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, 0);
 }
 
+BIF_RETTYPE binary_match_3(BIF_ALIST_3)
+{
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0);
+}
 
 BIF_RETTYPE binary_matches_2(BIF_ALIST_2)
 {
-    return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE);
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BINARY_FIND_ALL);
 }
 
+BIF_RETTYPE binary_matches_3(BIF_ALIST_3)
+{
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BINARY_FIND_ALL);
+}
 
-BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen)
+static BIF_RETTYPE
+binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 {
-    Uint pos;
-    Sint len;
-    size_t orig_size;
-    Eterm orig;
-    Uint offset;
-    Uint bit_offset;
-    Uint bit_size;
-    Eterm* hp;
-    ErlSubBin* sb;
+    BinaryFindState bfs;
+    Eterm *tp;
+    Binary *bin;
+    Eterm bin_term = NIL;
+    int runres;
+    Eterm result;
 
-    if (is_not_binary(binary)) {
+    if (is_not_binary(arg1)) {
 	goto badarg;
     }
-    if (!term_to_Uint(epos, &pos)) {
+    if (parse_split_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend), &(bfs.flags))) {
 	goto badarg;
     }
-    if (!term_to_Sint(elen, &len)) {
-	goto badarg;
+    if (bfs.hsend == 0) {
+	result = do_split_not_found_result(p, arg1, &bfs);
+	BIF_RET(result);
     }
-    if (len < 0) {
-	Uint lentmp = -(Uint)len;
-	/* overflow */
-	if ((Sint)lentmp < 0) {
+    if (is_tuple(arg2)) {
+	tp = tuple_val(arg2);
+	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
 	    goto badarg;
 	}
-	len = lentmp;
-	if (len > pos) {
+	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
+	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
 	    goto badarg;
 	}
-	pos -= len;
-    }
-    /* overflow */
-    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)){
+	bfs.type = tp[1];
+	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	if (bfs.type == am_bm &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
+	    goto badarg;
+	}
+	if (bfs.type == am_ac &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
+	    goto badarg;
+	}
+	bin_term = tp[2];
+    } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) {
 	goto badarg;
     }
-    if ((orig_size = binary_size(binary)) < pos ||
-	orig_size < (pos + len)) {
+    bfs.not_found_result = &do_split_not_found_result;
+    bfs.single_result = &do_split_single_result;
+    bfs.global_result = &do_split_global_result;
+    runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result);
+    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
+	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
+    } else if (bin_term == NIL) {
+	erts_bin_free(bin);
+    }
+    switch(runres) {
+    case DO_BIN_MATCH_OK:
+	BIF_RET(result);
+    case DO_BIN_MATCH_RESTART:
+	BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term);
+    default:
 	goto badarg;
     }
+ badarg:
+    BIF_ERROR(p, BADARG);
+}
 
+BIF_RETTYPE binary_split_2(BIF_ALIST_2)
+{
+    return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE);
+}
 
+BIF_RETTYPE binary_split_3(BIF_ALIST_3)
+{
+    return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
 
-    hp = HAlloc(p, ERL_SUB_BIN_SIZE);
+static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs)
+{
+    if (bfs->flags & BINARY_FIND_ALL) {
+	return NIL;
+    } else {
+	return am_nomatch;
+    }
+}
 
-    ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size);
-    sb = (ErlSubBin *) hp;
-    sb->thing_word = HEADER_SUB_BIN;
-    sb->size = len;
-    sb->offs = offset + pos;
-    sb->orig = orig;
-    sb->bitoffs = bit_offset;
-    sb->bitsize = 0;
-    sb->is_writable = 0;
+static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    Sint pos, Sint len)
+{
+    Eterm erlen;
+    Eterm *hp;
+    Eterm ret;
 
-    BIF_RET(make_binary(sb));
+    erlen = erts_make_integer((Uint)(len), p);
+    ret = erts_make_integer(pos, p);
+    hp = HAlloc(p, 3);
+    ret = TUPLE2(hp, ret, erlen);
 
- badarg:
-    BIF_ERROR(p, BADARG);
+    return ret;
 }
 
-#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need))
-
-BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple)
+static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    FindallData *fad, Uint fad_sz)
 {
-    Uint pos;
-    Sint len;
-    size_t orig_size;
-    Eterm orig;
-    Uint offset;
-    Uint bit_offset;
-    Uint bit_size;
-    Eterm* hp;
-    ErlSubBin* sb;
-    Eterm binary;
-    Eterm *tp;
-    Eterm epos, elen;
-    int extra_args;
+    Sint i;
+    Eterm tpl;
+    Eterm *hp;
+    Eterm ret;
+
+    for (i = 0; i < fad_sz; ++i) {
+	fad[i].epos = erts_make_integer(fad[i].pos, p);
+	fad[i].elen = erts_make_integer(fad[i].len, p);
+    }
+    hp = HAlloc(p, fad_sz * (3 + 2));
+    ret = NIL;
+    for (i = fad_sz - 1; i >= 0; --i) {
+	tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
+	hp += 3;
+	ret = CONS(hp, tpl, ret);
+	hp += 2;
+    }
+
+    return ret;
+}
+
+static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs)
+{
+    Eterm *hp;
+    Eterm ret;
+
+    hp = HAlloc(p, 2);
+    ret = CONS(hp, subject, NIL);
+
+    return ret;
+}
+
+static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    Sint pos, Sint len)
+{
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    ErlSubBin *sb1;
+    ErlSubBin *sb2;
+    Eterm *hp;
+    Eterm ret;
+
+    orig_size = binary_size(subject);
+
+    if ((bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) &&
+	(orig_size - pos - len) == 0) {
+	if (pos == 0) {
+	    ret = NIL;
+	} else {
+	    hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
+	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+	    sb1 = (ErlSubBin *) hp;
+	    sb1->thing_word = HEADER_SUB_BIN;
+	    sb1->size = pos;
+	    sb1->offs = offset;
+	    sb1->orig = orig;
+	    sb1->bitoffs = bit_offset;
+	    sb1->bitsize = bit_size;
+	    sb1->is_writable = 0;
+	    hp += ERL_SUB_BIN_SIZE;
+
+	    ret = CONS(hp, make_binary(sb1), NIL);
+	    hp += 2;
+	}
+    } else {
+	if ((bfs->flags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
+	    hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2));
+	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+	    sb1 = NULL;
+	} else {
+	    hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
+	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+	    sb1 = (ErlSubBin *) hp;
+	    sb1->thing_word = HEADER_SUB_BIN;
+	    sb1->size = pos;
+	    sb1->offs = offset;
+	    sb1->orig = orig;
+	    sb1->bitoffs = bit_offset;
+	    sb1->bitsize = 0;
+	    sb1->is_writable = 0;
+	    hp += ERL_SUB_BIN_SIZE;
+	}
+
+	sb2 = (ErlSubBin *) hp;
+	sb2->thing_word = HEADER_SUB_BIN;
+	sb2->size = orig_size - pos - len;
+	sb2->offs = offset + pos + len;
+	sb2->orig = orig;
+	sb2->bitoffs = bit_offset;
+	sb2->bitsize = bit_size;
+	sb2->is_writable = 0;
+	hp += ERL_SUB_BIN_SIZE;
+
+	ret = CONS(hp, make_binary(sb2), NIL);
+	hp += 2;
+	if (sb1 != NULL) {
+	    ret = CONS(hp, make_binary(sb1), ret);
+	    hp += 2;
+	}
+    }
+    return ret;
+}
+
+static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
+				    FindallData *fad, Uint fad_sz)
+{
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    ErlSubBin *sb;
+    Sint i;
+    Sint tail;
+    Uint list_size;
+    Uint end_pos;
+    Uint do_trim = bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL);
+    Eterm *hp;
+    Eterm *hendp;
+    Eterm ret;
+
+    tail = fad_sz - 1;
+    list_size = fad_sz + 1;
+    orig_size = binary_size(subject);
+    end_pos = (Uint)(orig_size);
+
+    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
+    hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2);
+    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
+    ASSERT(bit_size == 0);
+
+    ret = NIL;
+
+    for (i = tail; i >= 0; --i) {
+	sb = (ErlSubBin *)(hp);
+	sb->size = end_pos - (fad[i].pos + fad[i].len);
+	if (!(sb->size == 0 && do_trim)) {
+	    sb->thing_word = HEADER_SUB_BIN;
+	    sb->offs = offset + fad[i].pos + fad[i].len;
+	    sb->orig = orig;
+	    sb->bitoffs = bit_offset;
+	    sb->bitsize = 0;
+	    sb->is_writable = 0;
+	    hp += ERL_SUB_BIN_SIZE;
+	    ret = CONS(hp, make_binary(sb), ret);
+	    hp += 2;
+	    do_trim &= ~BINARY_SPLIT_TRIM;
+	}
+	end_pos = fad[i].pos;
+    }
+
+    sb = (ErlSubBin *)(hp);
+    sb->size = fad[0].pos;
+    if (!(sb->size == 0 && do_trim)) {
+	sb->thing_word = HEADER_SUB_BIN;
+	sb->offs = offset;
+	sb->orig = orig;
+	sb->bitoffs = bit_offset;
+	sb->bitsize = 0;
+	sb->is_writable = 0;
+	hp += ERL_SUB_BIN_SIZE;
+	ret = CONS(hp, make_binary(sb), ret);
+	hp += 2;
+    }
+    HRelease(p, hendp, hp);
+    return ret;
+}
+
+static BIF_RETTYPE binary_find_trap(BIF_ALIST_3)
+{
+    int runres;
+    Eterm result;
+    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    runres = do_binary_find(BIF_P, BIF_ARG_1, THE_NON_VALUE, bin, BIF_ARG_2, &result);
+    if (runres == DO_BIN_MATCH_OK) {
+	BIF_RET(result);
+    } else {
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, result, BIF_ARG_3);
+    }
+}
+
+BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen)
+{
+    Uint pos;
+    Sint len;
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    Eterm* hp;
+    ErlSubBin* sb;
+
+    if (is_not_binary(binary)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(epos, &pos)) {
+	goto badarg;
+    }
+    if (!term_to_Sint(elen, &len)) {
+	goto badarg;
+    }
+    if (len < 0) {
+	Uint lentmp = -(Uint)len;
+	/* overflow */
+	if ((Sint)lentmp < 0) {
+	    goto badarg;
+	}
+	len = lentmp;
+	if (len > pos) {
+	    goto badarg;
+	}
+	pos -= len;
+    }
+    /* overflow */
+    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)){
+	goto badarg;
+    }
+    if ((orig_size = binary_size(binary)) < pos ||
+	orig_size < (pos + len)) {
+	goto badarg;
+    }
+
+
+
+    hp = HAlloc(p, ERL_SUB_BIN_SIZE);
+
+    ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size);
+    sb = (ErlSubBin *) hp;
+    sb->thing_word = HEADER_SUB_BIN;
+    sb->size = len;
+    sb->offs = offset + pos;
+    sb->orig = orig;
+    sb->bitoffs = bit_offset;
+    sb->bitsize = 0;
+    sb->is_writable = 0;
+
+    BIF_RET(make_binary(sb));
+
+ badarg:
+    BIF_ERROR(p, BADARG);
+}
+
+#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need))
+
+BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple)
+{
+    Uint pos;
+    Sint len;
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    Eterm* hp;
+    ErlSubBin* sb;
+    Eterm binary;
+    Eterm *tp;
+    Eterm epos, elen;
+    int extra_args;
 
 
     if (range_is_tuple) {
@@ -2542,538 +2769,6 @@ BIF_RETTYPE binary_copy_2(BIF_ALIST_2)
     return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2);
 }
 
-static Eterm do_split_single_result(Process*, Eterm subject,
-				    Sint pos, Sint len, Uint hsflags);
-static Eterm do_split_global_result(Process*, FindallData *fad, Uint fad_sz,
-				    Eterm subject, Uint hsflags);
-
-#define BINARY_SPLIT_GLOBAL	0x01
-#define BINARY_SPLIT_TRIM	0x02
-#define BINARY_SPLIT_TRIM_ALL	0x04
-
-static int do_binary_split(Process *p, Eterm subject, Uint hsstart,
-			   Uint hsend, Uint hsflags, Eterm type, Binary *bin,
-			   Eterm state_term, Eterm *res_term)
-{
-    byte *bytes;
-    Uint bitoffs, bitsize;
-    byte *temp_alloc = NULL;
-
-    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
-    if (bitsize != 0) {
-	goto badarg;
-    }
-    if (bitoffs != 0) {
-	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
-    }
-    if (state_term != NIL) {
-	Eterm *ptr = big_val(state_term);
-	type = ptr[1];
-	hsflags = (Uint)(ptr[2]);
-    }
-
-    if (hsflags & BINARY_SPLIT_GLOBAL) {
-	if (type == am_bm) {
-	    BMData *bm;
-	    Sint pos;
-	    Eterm *hp;
-	    BMFindAllState state;
-	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
-	    Uint save_reds = reds;
-
-	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	    dump_bm_data(bm);
-#endif
-	    if (state_term == NIL) {
-		bm_init_find_all(&state, hsstart, hsend);
-	    } else {
-		Eterm *ptr = big_val(state_term);
-		bm_restore_find_all(&state, (char *)(ptr+3));
-	    }
-
-	    pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
-	    if (pos == BM_NOT_FOUND) {
-		hp = HAlloc(p, 2);
-		*res_term = CONS(hp, subject, NIL);
-	    } else if (pos == BM_RESTART) {
-		int x =
-		    (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
-		    !!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-		erts_printf("Trap bm!\n");
-#endif
-		hp = HAlloc(p, x+3);
-		hp[0] = make_pos_bignum_header(x+2);
-		hp[1] = type;
-		hp[2] = (Eterm)(hsflags);
-		bm_serialize_find_all(&state, (char *)(hp+3));
-		*res_term = make_big(hp);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		bm_clean_find_all(&state);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-                *res_term = do_split_global_result(p, state.out, state.m, subject, hsflags);
-	    }
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    bm_clean_find_all(&state);
-	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
-	} else if (type == am_ac) {
-	    ACTrie *act;
-	    int acr;
-	    ACFindAllState state;
-	    Eterm *hp;
-	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
-	    Uint save_reds = reds;
-
-	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	    dump_ac_trie(act);
-#endif
-	    if (state_term == NIL) {
-		ac_init_find_all(&state, act, hsstart, hsend);
-	    } else {
-		Eterm *ptr = big_val(state_term);
-		ac_restore_find_all(&state, (char *)(ptr+3));
-	    }
-	    acr = ac_find_all_non_overlapping(&state, bytes, &reds);
-	    if (acr == AC_NOT_FOUND) {
-		hp = HAlloc(p, 2);
-		*res_term = CONS(hp, subject, NIL);
-	    } else if (acr == AC_RESTART) {
-		int x = (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
-		    !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-		erts_printf("Trap ac!\n");
-#endif
-		hp = HAlloc(p, x+3);
-		hp[0] = make_pos_bignum_header(x+2);
-		hp[1] = type;
-		hp[2] = (Eterm)(hsflags);
-		ac_serialize_find_all(&state, (char *)(hp+3));
-		*res_term = make_big(hp);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		ac_clean_find_all(&state);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-                *res_term = do_split_global_result(p, state.out, state.m, subject, hsflags);
-	    }
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    ac_clean_find_all(&state);
-	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
-	}
-    } else {
-	if (type == am_bm) {
-	    BMData *bm;
-	    Sint pos;
-	    Eterm *hp;
-	    BMFindFirstState state;
-	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
-	    Uint save_reds = reds;
-
-	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	    dump_bm_data(bm);
-#endif
-	    if (state_term == NIL) {
-		bm_init_find_first_match(&state, hsstart, hsend);
-	    } else {
-		Eterm *ptr = big_val(state_term);
-		memcpy((void *)(&state), (const void *)(ptr+3), sizeof(BMFindFirstState));
-	    }
-
-#ifdef HARDDEBUG
-	    erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos,
-			state.len);
-#endif
-	    pos = bm_find_first_match(&state, bm, bytes, &reds);
-	    if (pos == BM_NOT_FOUND) {
-		hp = HAlloc(p, 2);
-		*res_term = CONS(hp, subject, NIL);
-	    } else if (pos == BM_RESTART) {
-		int x =
-		    (sizeof(state) / sizeof(Eterm)) +
-		    !!(sizeof(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-		erts_printf("Trap bm!\n");
-#endif
-		hp = HAlloc(p, x+3);
-		hp[0] = make_pos_bignum_header(x+2);
-		hp[1] = type;
-		hp[2] = (Eterm)(hsflags);
-		memcpy((void *)(hp+3), (const void *)(&state), sizeof(state));
-		*res_term = make_big(hp);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-		*res_term = do_split_single_result(p, subject, pos, bm->len, hsflags);
-	    }
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
-	} else if (type == am_ac) {
-	    ACTrie *act;
-	    Uint pos, rlen;
-	    int acr;
-	    ACFindFirstState state;
-	    Eterm *hp;
-	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
-	    Uint save_reds = reds;
-
-	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	    dump_ac_trie(act);
-#endif
-	    if (state_term == NIL) {
-		ac_init_find_first_match(&state, act, hsstart, hsend);
-	    } else {
-		Eterm *ptr = big_val(state_term);
-		memcpy((void *)(&state), (const void *)(ptr+3), sizeof(ACFindFirstState));
-	    }
-	    acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
-	    if (acr == AC_NOT_FOUND) {
-		hp = HAlloc(p, 2);
-		*res_term = CONS(hp, subject, NIL);
-	    } else if (acr == AC_RESTART) {
-		int x =
-		    (sizeof(state) / sizeof(Eterm)) +
-		    !!(sizeof(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-		erts_printf("Trap ac!\n");
-#endif
-		hp = HAlloc(p, x+3);
-		hp[0] = make_pos_bignum_header(x+2);
-		hp[1] = type;
-		hp[2] = (Eterm)(hsflags);
-		memcpy((void *)(hp+3), (const void *)(&state), sizeof(state));
-		*res_term = make_big(hp);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-		*res_term = do_split_single_result(p, subject, pos, rlen, hsflags);
-	    }
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
-	}
-    }
- badarg:
-    return DO_BIN_MATCH_BADARG;
-}
-
-static Eterm do_split_single_result(Process* p, Eterm subject,
-				    Sint pos, Sint len, Uint hsflags)
-{
-    size_t orig_size;
-    Eterm orig;
-    Uint offset;
-    Uint bit_offset;
-    Uint bit_size;
-    ErlSubBin *sb1;
-    ErlSubBin *sb2;
-    Eterm* hp;
-    Eterm ret;
-
-    orig_size = binary_size(subject);
-
-    if ((hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && (orig_size - pos - len) == 0) {
-	if (pos != 0) {
-	    ret = NIL;
-	} else {
-	    hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2));
-	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-	    sb1 = (ErlSubBin *) hp;
-	    sb1->thing_word = HEADER_SUB_BIN;
-	    sb1->size = pos;
-	    sb1->offs = offset;
-	    sb1->orig = orig;
-	    sb1->bitoffs = bit_offset;
-	    sb1->bitsize = bit_size;
-	    sb1->is_writable = 0;
-	    hp += ERL_SUB_BIN_SIZE;
-
-	    ret = CONS(hp, make_binary(sb1), NIL);
-	    hp += 2;
-	}
-    } else {
-	if ((hsflags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
-	    hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2));
-	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-	    sb1 = NULL;
-	} else {
-	    hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2));
-	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-	    sb1 = (ErlSubBin *) hp;
-	    sb1->thing_word = HEADER_SUB_BIN;
-	    sb1->size = pos;
-	    sb1->offs = offset;
-	    sb1->orig = orig;
-	    sb1->bitoffs = bit_offset;
-	    sb1->bitsize = 0;
-	    sb1->is_writable = 0;
-	    hp += ERL_SUB_BIN_SIZE;
-	}
-
-	sb2 = (ErlSubBin *) hp;
-	sb2->thing_word = HEADER_SUB_BIN;
-	sb2->size = orig_size - pos - len;
-	sb2->offs = offset + pos + len;
-	sb2->orig = orig;
-	sb2->bitoffs = bit_offset;
-	sb2->bitsize = bit_size;
-	sb2->is_writable = 0;
-	hp += ERL_SUB_BIN_SIZE;
-
-	ret = CONS(hp, make_binary(sb2), NIL);
-	hp += 2;
-	if (sb1 != NULL) {
-	    ret = CONS(hp, make_binary(sb1), ret);
-	    hp += 2;
-	}
-    }
-    return ret;
-}
-
-static Eterm do_split_global_result(Process* p, FindallData *fad, Uint fad_sz,
-				    Uint subject, Uint hsflags)
-{
-    size_t orig_size;
-    Eterm orig;
-    Uint offset;
-    Uint bit_offset;
-    Uint bit_size;
-    ErlSubBin *sb;
-    Sint i;
-    Sint tail;
-    Uint list_size;
-    Uint end_pos;
-    Uint do_trim = hsflags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL);
-    Eterm* hp;
-    Eterm* hendp;
-    Eterm ret;
-
-    tail = fad_sz - 1;
-    list_size = fad_sz + 1;
-    orig_size = binary_size(subject);
-    end_pos = (Uint)(orig_size);
-
-    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
-    hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2);
-    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
-    ASSERT(bit_size == 0);
-
-    ret = NIL;
-
-    for (i = tail; i >= 0; --i) {
-        sb = (ErlSubBin *)(hp);
-        sb->size = end_pos - (fad[i].pos + fad[i].len);
-        if (!(sb->size == 0 && do_trim)) {
-            sb->thing_word = HEADER_SUB_BIN;
-            sb->offs = offset + fad[i].pos + fad[i].len;
-            sb->orig = orig;
-            sb->bitoffs = bit_offset;
-            sb->bitsize = 0;
-            sb->is_writable = 0;
-            hp += ERL_SUB_BIN_SIZE;
-            ret = CONS(hp, make_binary(sb), ret);
-            hp += 2;
-            do_trim &= ~BINARY_SPLIT_TRIM;
-        }
-	end_pos = fad[i].pos;
-    }
-
-    sb = (ErlSubBin *)(hp);
-    sb->size = fad[0].pos;
-    if (!(sb->size == 0 && do_trim)) {
-        sb->thing_word = HEADER_SUB_BIN;
-        sb->offs = offset;
-        sb->orig = orig;
-        sb->bitoffs = bit_offset;
-        sb->bitsize = 0;
-        sb->is_writable = 0;
-        hp += ERL_SUB_BIN_SIZE;
-        ret = CONS(hp, make_binary(sb), ret);
-        hp += 2;
-    }
-    HRelease(p, hendp, hp);
-    return ret;
-}
-
-static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uint *optp)
-{
-    Eterm *tp;
-    Uint pos;
-    Sint len;
-    *optp = 0;
-    *posp = 0;
-    *endp = binary_size(bin);
-    if (l == THE_NON_VALUE || l == NIL) {
-	return 0;
-    } else if (is_list(l)) {
-	while(is_list(l)) {
-	    Eterm t = CAR(list_val(l));
-	    Uint orig_size;
-	    if (is_atom(t)) {
-		if (t == am_global) {
-		    *optp |= BINARY_SPLIT_GLOBAL;
-		    l = CDR(list_val(l));
-		    continue;
-		}
-		if (t == am_trim) {
-		    *optp |= BINARY_SPLIT_TRIM;
-		    l = CDR(list_val(l));
-		    continue;
-		}
-		if (t == am_trim_all) {
-		    *optp |= BINARY_SPLIT_TRIM_ALL;
-		    l = CDR(list_val(l));
-		    continue;
-		}
-	    }
-	    if (!is_tuple(t)) {
-		goto badarg;
-	    }
-	    tp = tuple_val(t);
-	    if (arityval(*tp) != 2) {
-		goto badarg;
-	    }
-	    if (tp[1] != am_scope || is_not_tuple(tp[2])) {
-		goto badarg;
-	    }
-	    tp = tuple_val(tp[2]);
-	    if (arityval(*tp) != 2) {
-		goto badarg;
-	    }
-	    if (!term_to_Uint(tp[1], &pos)) {
-		goto badarg;
-	    }
-	    if (!term_to_Sint(tp[2], &len)) {
-		goto badarg;
-	    }
-	    if (len < 0) {
-		Uint lentmp = -(Uint)len;
-		/* overflow */
-		if ((Sint)lentmp < 0) {
-		    goto badarg;
-		}
-		len = lentmp;
-		pos -= len;
-	    }
-	    /* overflow */
-	    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
-		goto badarg;
-	    }
-	    *endp = len + pos;
-	    *posp = pos;
-	    if ((orig_size = binary_size(bin)) < pos ||
-		orig_size < (*endp)) {
-		goto badarg;
-	    }
-	    l = CDR(list_val(l));
-	}
-	return 0;
-    } else {
-    badarg:
-	return 1;
-    }
-}
-
-static BIF_RETTYPE binary_split_trap(BIF_ALIST_3)
-{
-    int runres;
-    Eterm result;
-    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
-    runres = do_binary_split(BIF_P,BIF_ARG_1,0,0,0,NIL,bin,BIF_ARG_2,&result);
-    if (runres == DO_BIN_MATCH_OK) {
-	BIF_RET(result);
-    } else {
-	BUMP_ALL_REDS(BIF_P);
-	BIF_TRAP3(&binary_split_trap_export, BIF_P, BIF_ARG_1, result,
-		  BIF_ARG_3);
-    }
-}
-
-BIF_RETTYPE binary_split_3(BIF_ALIST_3)
-{
-    return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
-}
-
-static BIF_RETTYPE
-binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
-{
-    Uint hsflags;
-    Uint hsstart;
-    Uint hsend;
-    Eterm *tp;
-    Eterm type;
-    Binary *bin;
-    Eterm bin_term = NIL;
-    int runres;
-    Eterm result;
-
-    if (is_not_binary(arg1)) {
-	goto badarg;
-    }
-    if (parse_split_opts_list(arg3, arg1, &hsstart, &hsend, &hsflags)) {
-	goto badarg;
-    }
-    if (hsend == 0) {
-	tp = HAlloc(p, 2);
-	result = NIL;
-	result = CONS(tp, arg1, result);
-	BIF_RET(result);
-    }
-    if (is_tuple(arg2)) {
-	tp = tuple_val(arg2);
-	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
-	    goto badarg;
-	}
-	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
-	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
-	    goto badarg;
-	}
-	type = tp[1];
-	bin = ((ProcBin *) binary_val(tp[2]))->val;
-	if (type == am_bm &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
-	    goto badarg;
-	}
-	if (type == am_ac &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
-	    goto badarg;
-	}
-	bin_term = tp[2];
-    } else if (do_binary_match_compile(arg2, &type, &bin)) {
-	goto badarg;
-    }
-    runres = do_binary_split(p, arg1, hsstart, hsend, hsflags, type, bin, NIL, &result);
-    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
-	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
-	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
-    } else if (bin_term == NIL) {
-	erts_bin_free(bin);
-    }
-    switch(runres) {
-    case DO_BIN_MATCH_OK:
-	BIF_RET(result);
-    case DO_BIN_MATCH_RESTART:
-	BIF_TRAP3(&binary_split_trap_export, p, arg1, result, bin_term);
-    default:
-	goto badarg;
-    }
- badarg:
-    BIF_ERROR(p,BADARG);
-}
-
-
-BIF_RETTYPE binary_split_2(BIF_ALIST_2)
-{
-    return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE);
-}
-
-
 BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1)
 {
     ErlSubBin *sb;
-- 
cgit v1.2.3


From 8067992954f9b882ffd7332ee669a254c000f1b2 Mon Sep 17 00:00:00 2001
From: Andrew Bennett <potatosaladx@gmail.com>
Date: Fri, 18 Sep 2015 09:38:46 -0600
Subject: erts: Minor refactor for binary find BIF backend

* Use NULL instead of THE_NON_VALUE for non-Eterm variable.
* Add BinaryFindState_bignum struct to avoid unnecessary type casting.
---
 erts/emulator/beam/erl_bif_binary.c | 109 ++++++++++++++++++------------------
 1 file changed, 55 insertions(+), 54 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 9f72b8c0ac..b1ebf0327e 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -531,21 +531,23 @@ static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos,
     state->out = NULL;
 }
 
-static void ac_restore_find_all(ACFindAllState *state, char *buff)
+static void ac_restore_find_all(ACFindAllState *state,
+                                const ACFindAllState *src)
 {
-    memcpy(state,buff,sizeof(ACFindAllState));
+    memcpy(state, src, sizeof(ACFindAllState));
     if (state->allocated > 0) {
 	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated));
-	memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m);
+	memcpy(state->out, src+1, sizeof(FindallData)*state->m);
     } else {
 	state->out = NULL;
     }
 }
 
-static void ac_serialize_find_all(ACFindAllState *state, char *buff)
+static void ac_serialize_find_all(const ACFindAllState *state,
+                                  ACFindAllState *dst)
 {
-    memcpy(buff,state,sizeof(ACFindAllState));
-    memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m);
+    memcpy(dst, state, sizeof(ACFindAllState));
+    memcpy(dst+1, state->out, sizeof(FindallData)*state->m);
 }
 
 static void ac_clean_find_all(ACFindAllState *state)
@@ -805,24 +807,24 @@ static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len)
     state->out = NULL;
 }
 
-static void bm_restore_find_all(BMFindAllState *state, char *buff)
+static void bm_restore_find_all(BMFindAllState *state,
+                                const BMFindAllState *src)
 {
-    memcpy(state,buff,sizeof(BMFindAllState));
+    memcpy(state, src, sizeof(BMFindAllState));
     if (state->allocated > 0) {
 	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) *
 				(state->allocated));
-	memcpy(state->out,buff+sizeof(BMFindAllState),
-	       sizeof(FindallData)*state->m);
+	memcpy(state->out, src+1, sizeof(FindallData)*state->m);
     } else {
 	state->out = NULL;
     }
 }
 
-static void bm_serialize_find_all(BMFindAllState *state, char *buff)
+static void bm_serialize_find_all(const BMFindAllState *state,
+                                  BMFindAllState *dst)
 {
-    memcpy(buff,state,sizeof(BMFindAllState));
-    memcpy(buff+sizeof(BMFindAllState),state->out,
-	   sizeof(FindallData)*state->m);
+    memcpy(dst, state, sizeof(BMFindAllState));
+    memcpy(dst+1, state->out, sizeof(FindallData)*state->m);
 }
 
 static void bm_clean_find_all(BMFindAllState *state)
@@ -1032,6 +1034,17 @@ typedef struct BinaryFindState {
     Eterm (*global_result)    (Process *, Eterm, struct BinaryFindState *, FindallData *, Uint);
 } BinaryFindState;
 
+typedef struct BinaryFindState_bignum {
+    Eterm           bignum_hdr;
+    BinaryFindState bfs;
+    union {
+	BMFindFirstState bmffs;
+	BMFindAllState   bmfas;
+	ACFindFirstState acffs;
+	ACFindAllState   acfas;
+    } data;
+} BinaryFindState_bignum;
+
 #define SIZEOF_BINARY_FIND_STATE(S) \
 	  (sizeof(BinaryFindState)+sizeof(S))
 
@@ -1197,7 +1210,7 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
     byte *bytes;
     Uint bitoffs, bitsize;
     byte *temp_alloc = NULL;
-    char *state_ptr = NULL;
+    BinaryFindState_bignum *state_ptr = NULL;
 
     ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
     if (bitsize != 0) {
@@ -1207,17 +1220,14 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
     }
     if (state_term != NIL) {
-	state_ptr = (char *)(big_val(state_term));
-	state_ptr += sizeof(Eterm);
-	bfs = (BinaryFindState *)(state_ptr);
-	state_ptr += sizeof(BinaryFindState);
+	state_ptr = (BinaryFindState_bignum *)(big_val(state_term));
+	bfs = &(state_ptr->bfs);
     }
 
     if (bfs->flags & BINARY_FIND_ALL) {
 	if (bfs->type == am_bm) {
 	    BMData *bm;
 	    Sint pos;
-	    Eterm *hp;
 	    BMFindAllState state;
 	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
 	    Uint save_reds = reds;
@@ -1229,7 +1239,7 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	    if (state_term == NIL) {
 		bm_init_find_all(&state, bfs->hsstart, bfs->hsend);
 	    } else {
-		bm_restore_find_all(&state, state_ptr);
+		bm_restore_find_all(&state, &(state_ptr->data.bmfas));
 	    }
 
 	    pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
@@ -1242,12 +1252,11 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 #ifdef HARDDEBUG
 		erts_printf("Trap bm!\n");
 #endif
-		hp = HAlloc(p, x+1);
-		hp[0] = make_pos_bignum_header(x);
-		state_ptr = (char *)(hp);
-		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
-		bm_serialize_find_all(&state, state_ptr+sizeof(Eterm)+sizeof(BinaryFindState));
-		*res_term = make_big(hp);
+		state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
+		state_ptr->bignum_hdr = make_pos_bignum_header(x);
+		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
+		bm_serialize_find_all(&state, &state_ptr->data.bmfas);
+		*res_term = make_big(&state_ptr->bignum_hdr);
 		erts_free_aligned_binary_bytes(temp_alloc);
 		bm_clean_find_all(&state);
 		return DO_BIN_MATCH_RESTART;
@@ -1262,7 +1271,6 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	    ACTrie *act;
 	    int acr;
 	    ACFindAllState state;
-	    Eterm *hp;
 	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
 	    Uint save_reds = reds;
 
@@ -1273,7 +1281,7 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	    if (state_term == NIL) {
 		ac_init_find_all(&state, act, bfs->hsstart, bfs->hsend);
 	    } else {
-		ac_restore_find_all(&state, state_ptr);
+		ac_restore_find_all(&state, &(state_ptr->data.acfas));
 	    }
 	    acr = ac_find_all_non_overlapping(&state, bytes, &reds);
 	    if (acr == AC_NOT_FOUND) {
@@ -1285,12 +1293,11 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 #ifdef HARDDEBUG
 		erts_printf("Trap ac!\n");
 #endif
-		hp = HAlloc(p, x+1);
-		hp[0] = make_pos_bignum_header(x);
-		state_ptr = (char *)(hp);
-		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
-		ac_serialize_find_all(&state, state_ptr+sizeof(Eterm)+sizeof(BinaryFindState));
-		*res_term = make_big(hp);
+                state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
+		state_ptr->bignum_hdr = make_pos_bignum_header(x);
+		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
+		ac_serialize_find_all(&state, &state_ptr->data.acfas);
+		*res_term = make_big(&state_ptr->bignum_hdr);
 		erts_free_aligned_binary_bytes(temp_alloc);
 		ac_clean_find_all(&state);
 		return DO_BIN_MATCH_RESTART;
@@ -1306,7 +1313,6 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	if (bfs->type == am_bm) {
 	    BMData *bm;
 	    Sint pos;
-	    Eterm *hp;
 	    BMFindFirstState state;
 	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
 	    Uint save_reds = reds;
@@ -1318,7 +1324,7 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	    if (state_term == NIL) {
 		bm_init_find_first_match(&state, bfs->hsstart, bfs->hsend);
 	    } else {
-		memcpy((void *)(&state), (const void *)(state_ptr), sizeof(BMFindFirstState));
+		memcpy(&state, &state_ptr->data.bmffs, sizeof(BMFindFirstState));
 	    }
 
 #ifdef HARDDEBUG
@@ -1335,13 +1341,11 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 #ifdef HARDDEBUG
 		erts_printf("Trap bm!\n");
 #endif
-		hp = HAlloc(p, x+1);
-		hp[0] = make_pos_bignum_header(x);
-		state_ptr = (char *)(hp);
-		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
-		memcpy((void *)(state_ptr+sizeof(Eterm)+sizeof(BinaryFindState)),
-		       (const void *)(&state), sizeof(BMFindFirstState));
-		*res_term = make_big(hp);
+                state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
+                state_ptr->bignum_hdr = make_pos_bignum_header(x);
+		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
+		memcpy(&state_ptr->data.acffs, &state, sizeof(BMFindFirstState));
+		*res_term = make_big(&state_ptr->bignum_hdr);
 		erts_free_aligned_binary_bytes(temp_alloc);
 		return DO_BIN_MATCH_RESTART;
 	    } else {
@@ -1355,7 +1359,6 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	    Uint pos, rlen;
 	    int acr;
 	    ACFindFirstState state;
-	    Eterm *hp;
 	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
 	    Uint save_reds = reds;
 
@@ -1366,7 +1369,7 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 	    if (state_term == NIL) {
 		ac_init_find_first_match(&state, act, bfs->hsstart, bfs->hsend);
 	    } else {
-		memcpy((void *)(&state), (const void *)(state_ptr), sizeof(ACFindFirstState));
+		memcpy(&state, &state_ptr->data.acffs, sizeof(ACFindFirstState));
 	    }
 	    acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
 	    if (acr == AC_NOT_FOUND) {
@@ -1378,13 +1381,11 @@ static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binar
 #ifdef HARDDEBUG
 		erts_printf("Trap ac!\n");
 #endif
-		hp = HAlloc(p, x+1);
-		hp[0] = make_pos_bignum_header(x);
-		state_ptr = (char *)(hp);
-		memcpy((void *)(state_ptr+sizeof(Eterm)), bfs, sizeof(BinaryFindState));
-		memcpy((void *)(state_ptr+sizeof(Eterm)+sizeof(BinaryFindState)),
-		       (const void *)(&state), sizeof(ACFindFirstState));
-		*res_term = make_big(hp);
+		state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
+		state_ptr->bignum_hdr = make_pos_bignum_header(x);
+		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
+		memcpy(&state_ptr->data.acffs, &state, sizeof(ACFindFirstState));
+		*res_term = make_big(&state_ptr->bignum_hdr);
 		erts_free_aligned_binary_bytes(temp_alloc);
 		return DO_BIN_MATCH_RESTART;
 	    } else {
@@ -1763,7 +1764,7 @@ static BIF_RETTYPE binary_find_trap(BIF_ALIST_3)
     int runres;
     Eterm result;
     Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
-    runres = do_binary_find(BIF_P, BIF_ARG_1, THE_NON_VALUE, bin, BIF_ARG_2, &result);
+    runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result);
     if (runres == DO_BIN_MATCH_OK) {
 	BIF_RET(result);
     } else {
-- 
cgit v1.2.3


From 949de78331b9c4ecb9c628bb651cecb113790e2e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Nov 2015 15:43:15 +0100
Subject: erts: Fix bug in setnode/2

that could lead strange things to happen when renaming a node with
a name that already exist in node and dist tables.

Problem:
erts_set_this_node() is a bit brutal and does not handle the case
when an old remote node name is reused as the new local node name.

Solution:
Treat erts_this_node and erts_this_dist_entry as all the others
with correct reference counting.
---
 erts/emulator/beam/bif.c             |   1 +
 erts/emulator/beam/dist.c            |  14 ++--
 erts/emulator/beam/erl_node_tables.c | 144 ++++++++++++-----------------------
 3 files changed, 57 insertions(+), 102 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 4e3a1cef69..6c61b0de6b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4231,6 +4231,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
 	  goto bad;
       
       enp = erts_find_or_insert_node(dep->sysname, dep->creation);
+      ASSERT(enp != erts_this_node);
 
       etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1);
       etp->header = make_external_pid_header(1);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 5a0f8388f8..56a8633f85 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2572,7 +2572,9 @@ int distribution_info(int to, void *arg)	/* Called by break handler */
     }
 
     for (dep = erts_not_connected_dist_entries; dep; dep = dep->next) {
-	info_dist_entry(to, arg, dep, 0, 0);
+        if (dep != erts_this_dist_entry) {
+            info_dist_entry(to, arg, dep, 0, 0);
+        }
     }
 
     return(0);
@@ -3012,11 +3014,11 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
 
-    ASSERT(erts_no_of_not_connected_dist_entries >= 0);
+    ASSERT(erts_no_of_not_connected_dist_entries > 0);
     ASSERT(erts_no_of_hidden_dist_entries >= 0);
     ASSERT(erts_no_of_visible_dist_entries >= 0);
     if(not_connected)
-      length += erts_no_of_not_connected_dist_entries;
+      length += (erts_no_of_not_connected_dist_entries - 1);
     if(hidden)
       length += erts_no_of_hidden_dist_entries;
     if(visible)
@@ -3038,8 +3040,10 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 #endif
     if(not_connected)
       for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) {
-	result = CONS(hp, dep->sysname, result);
-	hp += 2;
+          if (dep != erts_this_dist_entry) {
+            result = CONS(hp, dep->sysname, result);
+            hp += 2;
+          }
       }
     if(hidden)
       for(dep = erts_hidden_dist_entries; dep; dep = dep->next) {
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 865de1726d..a7d0511bf9 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -37,18 +37,18 @@ erts_smp_rwmtx_t erts_node_table_rwmtx;
 
 DistEntry *erts_hidden_dist_entries;
 DistEntry *erts_visible_dist_entries;
-DistEntry *erts_not_connected_dist_entries;
+DistEntry *erts_not_connected_dist_entries; /* including erts_this_dist_entry */
 Sint erts_no_of_hidden_dist_entries;
 Sint erts_no_of_visible_dist_entries;
-Sint erts_no_of_not_connected_dist_entries;
+Sint erts_no_of_not_connected_dist_entries; /* including erts_this_dist_entry */
 
 DistEntry *erts_this_dist_entry;
 ErlNode *erts_this_node;
 char erts_this_node_sysname_BUFFER[256],
     *erts_this_node_sysname = "uninitialized yet";
 
-static Uint node_entries;
-static Uint dist_entries;
+static Uint node_entries = 0;
+static Uint dist_entries = 0;
 
 static int references_atoms_need_init = 1;
 
@@ -91,9 +91,6 @@ dist_table_alloc(void *dep_tmpl)
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 
-    if(((DistEntry *) dep_tmpl) == erts_this_dist_entry)
-	return dep_tmpl;
-
     sysname = ((DistEntry *) dep_tmpl)->sysname;
     chnl_nr = make_small((Uint) atom_val(sysname));
     dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry));
@@ -132,7 +129,9 @@ dist_table_alloc(void *dep_tmpl)
 
     /* Link in */
 
-    /* All new dist entries are "not connected" */
+    /* All new dist entries are "not connected".
+     * erts_this_dist_entry is also always included among "not connected"
+     */
     dep->next = erts_not_connected_dist_entries;
     if(erts_not_connected_dist_entries) {
 	ASSERT(erts_not_connected_dist_entries->prev == NULL);
@@ -149,9 +148,6 @@ dist_table_free(void *vdep)
 {
     DistEntry *dep = (DistEntry *) vdep;
 
-    if(dep == erts_this_dist_entry)
-	return;
-
     ASSERT(is_nil(dep->cid));
     ASSERT(dep->nlinks == NULL);
     ASSERT(dep->node_links == NULL);
@@ -186,7 +182,7 @@ dist_table_free(void *vdep)
 #endif
     erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep);
 
-    ASSERT(dist_entries > 1);
+    ASSERT(dist_entries > 0);
     dist_entries--;
 }
 
@@ -306,7 +302,7 @@ static void try_delete_dist_entry(void *vdep)
      * thread incremented refc twice. Once for the new reference
      * and once for this thread.
      *
-     * If refc reach -1, noone has used the entry since we
+     * If refc reach -1, no one has used the entry since we
      * set up the timer. Delete the entry.
      *
      * If refc reach 0, the entry is currently not in use
@@ -369,8 +365,7 @@ erts_dist_table_size(void)
 
     ASSERT(dist_entries == (erts_no_of_visible_dist_entries
 			    + erts_no_of_hidden_dist_entries
-			    + erts_no_of_not_connected_dist_entries
-			    + 1 /* erts_this_dist_entry */));
+			    + erts_no_of_not_connected_dist_entries));
 #endif
 
     res = (hash_table_sz(&erts_dist_table)
@@ -543,9 +538,6 @@ node_table_alloc(void *venp_tmpl)
 {
     ErlNode *enp;
 
-    if(((ErlNode *) venp_tmpl) == erts_this_node)
-	return venp_tmpl;
-
     enp = (ErlNode *) erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode));
 
     node_entries++;
@@ -563,8 +555,7 @@ node_table_free(void *venp)
 {
     ErlNode *enp = (ErlNode *) venp;
 
-    if(enp == erts_this_node)
-	return;
+    ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking());
 
     erts_deref_dist_entry(enp->dist_entry);
 #ifdef DEBUG
@@ -572,7 +563,7 @@ node_table_free(void *venp)
 #endif
     erts_free(ERTS_ALC_T_NODE_ENTRY, venp);
 
-    ASSERT(node_entries > 1);
+    ASSERT(node_entries > 0);
     node_entries--;
 }
 
@@ -650,7 +641,7 @@ static void try_delete_node(void *venp)
      * thread incremented refc twice. Once for the new reference
      * and once for this thread.
      *
-     * If refc reach -1, noone has used the entry since we
+     * If refc reach -1, no one has used the entry since we
      * set up the timer. Delete the entry.
      *
      * If refc reach 0, the entry is currently not in use
@@ -747,25 +738,20 @@ void erts_print_node_info(int to,
 void
 erts_set_this_node(Eterm sysname, Uint creation)
 {
-    erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
-    erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking());
+    ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2));
 
-    (void) hash_erase(&erts_dist_table, (void *) erts_this_dist_entry);
-    erts_this_dist_entry->sysname = sysname;
-    erts_this_dist_entry->creation = creation;
-    (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry);
+    if (erts_refc_dectest(&erts_this_node->refc, 0) == 0)
+        try_delete_node(erts_this_node);
 
-    (void) hash_erase(&erts_node_table, (void *) erts_this_node);
-    erts_this_node->sysname = sysname;
-    erts_this_node->creation = creation;
-    erts_this_node_sysname = erts_this_node_sysname_BUFFER;
-    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
-                  "%T", sysname);
-    (void) hash_put(&erts_node_table, (void *) erts_this_node);
+    if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0)
+        try_delete_dist_entry(erts_this_dist_entry);
 
-    erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
-    erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+    erts_this_node = NULL; /* to make sure refc is bumped for this node */
+    erts_this_node = erts_find_or_insert_node(sysname, creation);
+    erts_this_dist_entry = erts_this_node->dist_entry;
 
+    erts_refc_inc(&erts_this_dist_entry->refc, 2);
 }
 
 Uint
@@ -782,6 +768,7 @@ void erts_init_node_tables(int dd_sec)
 {
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     HashFunctions f;
+    ErlNode node_tmpl;
 
     if (dd_sec == ERTS_NODE_TAB_DELAY_GC_INFINITY)
 	node_tab_delete_delay = (ErtsMonotonicTime) -1;
@@ -793,16 +780,21 @@ void erts_init_node_tables(int dd_sec)
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
+    erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table");
+    erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table");
+
     f.hash  = (H_FUN)			dist_table_hash;
     f.cmp   = (HCMP_FUN)		dist_table_cmp;
     f.alloc = (HALLOC_FUN)		dist_table_alloc;
     f.free  = (HFREE_FUN)		dist_table_free;
-
-    erts_this_dist_entry = erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry));
-    dist_entries = 1;
-
     hash_init(ERTS_ALC_T_DIST_TABLE, &erts_dist_table, "dist_table", 11, f);
 
+    f.hash  = (H_FUN)      			node_table_hash;
+    f.cmp   = (HCMP_FUN)   			node_table_cmp;
+    f.alloc = (HALLOC_FUN) 			node_table_alloc;
+    f.free  = (HFREE_FUN)  			node_table_free;
+    hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f);
+
     erts_hidden_dist_entries				= NULL;
     erts_visible_dist_entries				= NULL;
     erts_not_connected_dist_entries			= NULL;
@@ -810,69 +802,23 @@ void erts_init_node_tables(int dd_sec)
     erts_no_of_visible_dist_entries			= 0;
     erts_no_of_not_connected_dist_entries		= 0;
 
-    erts_this_dist_entry->next				= NULL;
-    erts_this_dist_entry->prev				= NULL;
-    erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */
-
-    erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx,
-			      &rwmtx_opt,
-			      "dist_entry",
-			      make_small(ERST_INTERNAL_CHANNEL_NO));
-    erts_this_dist_entry->sysname			= am_Noname;
-    erts_this_dist_entry->cid				= NIL;
-    erts_this_dist_entry->connection_id			= 0;
-    erts_this_dist_entry->status			= 0;
-    erts_this_dist_entry->flags				= 0;
-    erts_this_dist_entry->version			= 0;
-
-    erts_smp_mtx_init_x(&erts_this_dist_entry->lnk_mtx,
-			"dist_entry_links",
-			make_small(ERST_INTERNAL_CHANNEL_NO));
-    erts_this_dist_entry->node_links			= NULL;
-    erts_this_dist_entry->nlinks			= NULL;
-    erts_this_dist_entry->monitors			= NULL;
-
-    erts_smp_mtx_init_x(&erts_this_dist_entry->qlock,
-			"dist_entry_out_queue",
-			make_small(ERST_INTERNAL_CHANNEL_NO));
-    erts_this_dist_entry->qflgs				= 0;
-    erts_this_dist_entry->qsize				= 0;
-    erts_this_dist_entry->out_queue.first		= NULL;
-    erts_this_dist_entry->out_queue.last		= NULL;
-    erts_this_dist_entry->suspended			= NULL;
-
-    erts_this_dist_entry->finalized_out_queue.first	= NULL;
-    erts_this_dist_entry->finalized_out_queue.last	= NULL;
-    erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0);
-    erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd);
-    erts_this_dist_entry->send				= NULL;
-    erts_this_dist_entry->cache				= NULL;
-
-    (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry);
+    node_tmpl.sysname = am_Noname;
+    node_tmpl.creation = 0;
+    erts_this_node = hash_put(&erts_node_table, &node_tmpl);
+     /* +1 for erts_this_node */
+    erts_refc_init(&erts_this_node->refc, 1);
 
-    f.hash  = (H_FUN)      			node_table_hash;
-    f.cmp   = (HCMP_FUN)   			node_table_cmp;
-    f.alloc = (HALLOC_FUN) 			node_table_alloc;
-    f.free  = (HFREE_FUN)  			node_table_free;
-
-    hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f);
+    ASSERT(erts_this_node->dist_entry != NULL);
+    erts_this_dist_entry = erts_this_node->dist_entry;
+    /* +1 for erts_this_dist_entry */
+    /* +1 for erts_this_node->dist_entry */
+    erts_refc_init(&erts_this_dist_entry->refc, 2);
 
-    erts_this_node = erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode));
-    node_entries = 1;
 
-    erts_refc_init(&erts_this_node->refc, 1); /* The system itself */
-    erts_this_node->sysname			= am_Noname;
-    erts_this_node->creation			= 0;
-    erts_this_node->dist_entry			= erts_this_dist_entry;
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
     erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
                   "%T", erts_this_node->sysname);
 
-    (void) hash_put(&erts_node_table, (void *) erts_this_node);
-
-    erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table");
-    erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table");
-
     references_atoms_need_init = 1;
 }
 
@@ -1410,6 +1356,10 @@ setup_reference_table(void)
 		SYSTEM_REF,
 		TUPLE2(&heap[0], AM_system, am_undefined));
 
+    insert_dist_entry(erts_this_dist_entry,
+                      SYSTEM_REF,
+                      TUPLE2(&heap[0], AM_system, am_undefined),
+                      erts_this_node->creation);
     UnUseTmpHeapNoproc(3);
 
     max = erts_ptab_max(&erts_proc);
-- 
cgit v1.2.3


From b72ad981e96fcc14c245ef2bd10c5c98c6a239f6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 5 Dec 2014 15:13:07 +0100
Subject: erts: Add TEST allocator

---
 erts/emulator/beam/erl_alloc.c     | 53 ++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_alloc.types |  5 +++-
 2 files changed, 55 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 55c164bf11..4223b357f1 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -134,6 +134,7 @@ static ErtsAllocatorState_t binary_alloc_state;
 static ErtsAllocatorState_t ets_alloc_state;
 static ErtsAllocatorState_t driver_alloc_state;
 static ErtsAllocatorState_t fix_alloc_state;
+static ErtsAllocatorState_t test_alloc_state;
 
 typedef struct {
     erts_smp_atomic32_t refc;
@@ -233,6 +234,7 @@ typedef struct {
     struct au_init std_low_alloc;
     struct au_init ll_low_alloc;
 #endif
+    struct au_init test_alloc;
 } erts_alc_hndl_args_init_t;
 
 #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
@@ -432,6 +434,33 @@ set_default_fix_alloc_opts(struct au_init *ip,
     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
 }
 
+static void
+set_default_test_alloc_opts(struct au_init *ip)
+{
+    SET_DEFAULT_ALLOC_OPTS(ip);
+    ip->enable			= 0; /* Disabled by default */
+    ip->thr_spec		= -1 * erts_no_schedulers;
+    ip->atype			= AOFIRSTFIT;
+    ip->init.aoff.flavor        = AOFF_BF;
+    ip->init.util.name_prefix	= "test_";
+    ip->init.util.alloc_no	= ERTS_ALC_A_TEST;
+    ip->init.util.mmbcs 	= 0; /* Main carrier size */
+    ip->init.util.ts 		= ERTS_ALC_MTA_TEST;
+    ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
+
+    /* Use a constant minimal MBC size */
+#if ERTS_SA_MB_CARRIERS
+    ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ;
+    ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ;
+    ip->init.util.sbct  = ERTS_SACRR_UNIT_SZ;
+#else
+    ip->init.util.smbcs = 1 << 12;
+    ip->init.util.lmbcs = 1 << 12;
+    ip->init.util.sbct  = 1 << 12;
+#endif
+}
+
+
 #ifdef ERTS_SMP
 
 static void
@@ -613,6 +642,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_default_driver_alloc_opts(&init.driver_alloc);
     set_default_fix_alloc_opts(&init.fix_alloc,
 			       fix_type_sizes);
+    set_default_test_alloc_opts(&init.test_alloc);
 
     if (argc && argv)
 	handle_args(argc, argv, &init);
@@ -772,6 +802,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu);
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	if (!erts_allctrs[i].alloc)
@@ -833,6 +864,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 		       &init.fix_alloc,
 		       &fix_alloc_state);
 
+    start_au_allocator(ERTS_ALC_A_TEST,
+		       &init.test_alloc,
+		       &test_alloc_state);
+
     erts_mtrace_install_wrapper_functions();
     extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
 
@@ -1418,6 +1453,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 	&init->fix_alloc,
 	&init->sl_alloc,
 	&init->temp_alloc
+	/* test_alloc not affected by +Mea??? or +Mu???  */
     };
     int aui_sz = (int) sizeof(aui)/sizeof(aui[0]);
     char *arg;
@@ -1508,6 +1544,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		case 'T':
 		    handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0);
 		    break;
+		case 'Z':
+		    handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0);
+		    break;
 		case 'Y': { /* sys_alloc */
 		    if (has_prefix("tt", param+2)) {
 			/* set trim threshold */
@@ -2222,11 +2261,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	    return am_badarg;
     }
 
-    /* All alloc_util allocators *have* to be enabled */
+    /* All alloc_util allocators *have* to be enabled, except test_alloc */
     
     for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
 	switch (ai) {
 	case ERTS_ALC_A_SYSTEM:
+        case ERTS_ALC_A_TEST:
 	    break;
 	default:
 	    if (!erts_allctrs_info[ai].enabled
@@ -2267,6 +2307,8 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		      * contain any allocated memory.
 		      */
 		    continue;
+                case ERTS_ALC_A_TEST:
+                    continue;
 		case ERTS_ALC_A_EHEAP:
 		    save = &size.processes;
 		    break;
@@ -2675,14 +2717,17 @@ erts_alloc_util_allocators(void *proc)
     /*
      * Currently all allocators except sys_alloc are
      * alloc_util allocators.
+     * Also hide test_alloc which is disabled by default
+     * and only intended for our own testing.
      */
-    sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2;
+    sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2;
     ASSERT(sz > 0);
     hp = HAlloc((Process *) proc, sz);
     res = NIL;
     for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) {
 	switch (i) {
 	case ERTS_ALC_A_SYSTEM:
+        case ERTS_ALC_A_TEST:
 	    break;
 	default: {
 	    char *alc_str = (char *) ERTS_ALC_A2AD(i);
@@ -3566,6 +3611,10 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 #else
 	case 0xf13: return (UWord) 0;
 #endif
+	case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1);
+
+	case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0;
+
 	default:
 	    break;
 	}
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index b1d511ab78..1ecebdeb07 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -112,7 +112,7 @@ allocator	STANDARD_LOW	false		std_low_alloc
 allocator	BINARY		true		binary_alloc
 allocator	DRIVER		true		driver_alloc
 
-
+allocator	TEST            true            test_alloc
 
 # --- Class declarations -----------------------------------------------------
 #
@@ -456,4 +456,7 @@ type	CON_VPRINTF_BUF	TEMPORARY	SYSTEM		con_vprintf_buf
 
 +endif
 
+# This type should only be used for test
+type	TEST		TEST		SYSTEM		testing
+
 # ----------------------------------------------------------------------------
-- 
cgit v1.2.3


From 38ddf72b48377bd6b2fb8c4b6981360ae7d44d79 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 8 Dec 2014 15:38:04 +0100
Subject: erts: Add alloc_SUITE:migration

---
 erts/emulator/beam/erl_alloc.c      | 27 +++++++++++++++++++++++++++
 erts/emulator/beam/erl_alloc_util.c | 10 ++++++++++
 2 files changed, 37 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 4223b357f1..0aa45acd82 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3615,6 +3615,33 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 
 	case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0;
 
+	case 0xf16: {
+            Uint extra_hdr_sz = UNIT_CEILING((Uint)a1);
+	    ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
+	    Uint offset = ts->allctr[0]->mbc_header_size;
+	    void* orig_creating_mbc = ts->allctr[0]->creating_mbc;
+	    void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc;
+	    void* new_creating_mbc = *(void**)a2; /* inout arg */
+	    void* new_destroying_mbc = *(void**)a3; /* inout arg */
+	    int i;
+
+	    for (i=0; i < ts->size; i++) {
+		Allctr_t* ap = ts->allctr[i];
+		if (ap->mbc_header_size != offset
+		    || ap->creating_mbc != orig_creating_mbc
+		    || ap->destroying_mbc != orig_destroying_mbc
+		    || ap->mbc_list.first != NULL)
+		    return -1;
+	    }
+	    for (i=0; i < ts->size; i++) {
+		ts->allctr[i]->mbc_header_size += extra_hdr_sz;
+		ts->allctr[i]->creating_mbc = new_creating_mbc;
+		ts->allctr[i]->destroying_mbc = new_destroying_mbc;
+	    }
+	    *(void**)a2 = orig_creating_mbc;
+	    *(void**)a3 = orig_destroying_mbc;
+	    return offset;
+	}
 	default:
 	    break;
 	}
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 173cb091b6..8229a15824 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -6074,6 +6074,16 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
     case 0x023: return (UWord) 0;
     case 0x024: return (UWord) 0;
 #endif
+    case 0x025: /* UMEM2BLK_TEST*/
+#ifdef DEBUG
+# ifdef HARD_DEBUG
+	return (UWord)UMEM2BLK(a1-3*sizeof(UWord));
+# else
+	return (UWord)UMEM2BLK(a1-2*sizeof(UWord));
+# endif
+#else
+	return (UWord)UMEM2BLK(a1);
+#endif
 
     default:	ASSERT(0); return ~((UWord) 0);
     }
-- 
cgit v1.2.3


From 01cc99b35c00be86d832693776ee8ed880b59882 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 8 Dec 2014 20:36:28 +0100
Subject: erts: Make key argument constant for erl_drv_{get|put}env

This should be a harmless and compatible API change.
---
 erts/emulator/beam/erl_driver.h | 4 ++--
 erts/emulator/beam/io.c         | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 6b406d069c..dbb4d719c1 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -696,8 +696,8 @@ EXTERN int driver_dl_close(void *);
 EXTERN char *driver_dl_error(void);
 
 /* environment */
-EXTERN int erl_drv_putenv(char *key, char *value);
-EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
+EXTERN int erl_drv_putenv(const char *key, char *value);
+EXTERN int erl_drv_getenv(const char *key, char *value, size_t *value_size);
 
 #ifdef __OSE__
 typedef ErlDrvUInt ErlDrvOseEventId;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 900616c981..c64c8802b9 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7596,15 +7596,15 @@ int null_func(void)
 }
 
 int
-erl_drv_putenv(char *key, char *value)
+erl_drv_putenv(const char *key, char *value)
 {
-    return erts_sys_putenv_raw(key, value);
+    return erts_sys_putenv_raw((char*)key, value);
 }
 
 int
-erl_drv_getenv(char *key, char *value, size_t *value_size)
+erl_drv_getenv(const char *key, char *value, size_t *value_size)
 {
-    return erts_sys_getenv_raw(key, value, value_size);
+    return erts_sys_getenv_raw((char*)key, value, value_size);
 }
 
 /* get heart_port
-- 
cgit v1.2.3


From 63466c5522ed58b6e73e35dc29c7c7584f073768 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 2 Sep 2015 18:35:29 +0200
Subject: erts: Refactor erl_mmap to allow several mapper instances

---
 erts/emulator/beam/erl_alloc.c    | 5 +++--
 erts/emulator/beam/erl_bif_info.c | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d83a2930d6..bb3998769c 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2668,7 +2668,7 @@ erts_allocator_info(int to, void *arg)
 	    erts_mseg_info(i, &to, arg, 0, NULL, NULL);
 	}
 	erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n");
-	erts_mmap_info(&to, arg, NULL, NULL, &emis);
+	erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis);
     }
 #endif
 
@@ -3020,7 +3020,8 @@ reply_alloc_info(void *vair)
 
 		    ai_list = erts_bld_cons(hpp, szp,
 					    ainfo, ai_list);
-		    ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis));
+		    ainfo = (air->only_sz ? NIL :
+                             erts_mmap_info(&erts_dflt_mmapper, NULL, NULL, hpp, szp, &emis));
 		    ainfo = erts_bld_tuple3(hpp, szp,
                                             alloc_atom,
                                             erts_bld_atom(hpp,szp,"erts_mmap"),
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 79ccf8db64..a73ad826db 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3441,7 +3441,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    BIF_RET(res);
 	}
         else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) {
-            BIF_RET(erts_mmap_debug_info(BIF_P));
+            BIF_RET(erts_mmap_debug_info(&erts_dflt_mmapper, BIF_P));
         }
 	else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) {
 	    BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P));
-- 
cgit v1.2.3


From b7f760cee119e1824de0cfc2b34ae6fe971bf505 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 11 Sep 2015 16:03:00 +0200
Subject: erts: Add support for fast erts_is_literal()

---
 erts/emulator/beam/beam_load.c      |   1 +
 erts/emulator/beam/erl_alloc.c      |  40 +++--
 erts/emulator/beam/erl_alloc.h      |  24 +++
 erts/emulator/beam/erl_alloc_util.c | 286 ++++++++++++++++++++++++++++++++----
 erts/emulator/beam/erl_alloc_util.h |  64 +++++++-
 erts/emulator/beam/sys.h            |   3 +
 6 files changed, 375 insertions(+), 43 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index f0b4be4c3d..a846e96204 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4431,6 +4431,7 @@ freeze_code(LoaderState* stp)
             if (stp->literals[i].heap_frags) {
                 move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags,
                                  &stp->literals[i].term, 1);
+                ASSERT(erts_is_literal(ptr_val(stp->literals[i].term)));
             }
             else ASSERT(is_immed(stp->literals[i].term));
 	}
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index bb3998769c..3e300f88ea 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -310,6 +310,25 @@ set_default_literal_alloc_opts(struct au_init *ip)
     ip->init.util.rsbcmt	= 0;
     ip->init.util.rmbcmt	= 0;
     ip->init.util.acul		= 0;
+
+#if defined(ARCH_32)
+# if HAVE_ERTS_MSEG
+    ip->init.util.mseg_alloc   = &erts_alcu_literal_32_mseg_alloc;
+    ip->init.util.mseg_realloc = &erts_alcu_literal_32_mseg_realloc;
+    ip->init.util.mseg_dealloc = &erts_alcu_literal_32_mseg_dealloc;
+# endif
+    ip->init.util.sys_alloc    = &erts_alcu_literal_32_sys_alloc;
+    ip->init.util.sys_realloc  = &erts_alcu_literal_32_sys_realloc;
+    ip->init.util.sys_dealloc  = &erts_alcu_literal_32_sys_dealloc;
+#elif defined(ARCH_64)
+# ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
+    ip->init.util.mseg_alloc    = &erts_alcu_literal_64_mseg_alloc;
+    ip->init.util.mseg_realloc  = &erts_alcu_literal_64_mseg_realloc;
+    ip->init.util.mseg_dealloc  = &erts_alcu_literal_64_mseg_dealloc;
+# endif
+#else
+# error Unknown architecture
+#endif
 }
 
 static void
@@ -720,6 +739,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #if HAVE_ERTS_MSEG
     init.mseg.nos = erts_no_schedulers;
     erts_mseg_init(&init.mseg);
+# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+    erts_mmap_init(&erts_literal_mmapper, &init.mseg.literal_mmap);
+# endif
 #endif
 
     erts_alcu_init(&init.alloc_util);
@@ -991,7 +1013,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
     }
 
     for (i = 0; i < size; i++) {
-	void *as;
+	Allctr_t *as;
 	atype = init->atype;
 
 	if (!init->thr_spec)
@@ -1028,22 +1050,22 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 
 	switch (atype) {
 	case GOODFIT:
-	    as = (void *) erts_gfalc_start((GFAllctr_t *) as0,
+	    as = erts_gfalc_start((GFAllctr_t *) as0,
 					   &init->init.gf,
 					   &init->init.util);
 	    break;
 	case BESTFIT:
-	    as = (void *) erts_bfalc_start((BFAllctr_t *) as0,
+	    as = erts_bfalc_start((BFAllctr_t *) as0,
 					   &init->init.bf,
 					   &init->init.util);
 	    break;
 	case AFIT:
-	    as = (void *) erts_afalc_start((AFAllctr_t *) as0,
+	    as = erts_afalc_start((AFAllctr_t *) as0,
 					   &init->init.af,
 					   &init->init.util);
 	    break;
     	case AOFIRSTFIT:
-	    as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0,
+	    as = erts_aoffalc_start((AOFFAllctr_t *) as0,
 					     &init->init.aoff,
 					     &init->init.util);
 	    break;
@@ -1445,25 +1467,25 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    }
 		    else if (has_prefix("scs", argv[i]+3)) {
 #if HAVE_ERTS_MSEG
-			init->mseg.mmap.scs =
+			init->mseg.dflt_mmap.scs =
 #endif
 			    get_mb_value(argv[i]+6, argv, &i);
 		    }
 		    else if (has_prefix("sco", argv[i]+3)) {
 #if HAVE_ERTS_MSEG
-			init->mseg.mmap.sco =
+			init->mseg.dflt_mmap.sco =
 #endif
 			    get_bool_value(argv[i]+6, argv, &i);
 		    }
 		    else if (has_prefix("scrpm", argv[i]+3)) {
 #if HAVE_ERTS_MSEG
-			init->mseg.mmap.scrpm =
+			init->mseg.dflt_mmap.scrpm =
 #endif
 			    get_bool_value(argv[i]+8, argv, &i);
 		    }
 		    else if (has_prefix("scrfsd", argv[i]+3)) {
 #if HAVE_ERTS_MSEG
-			init->mseg.mmap.scrfsd =
+			init->mseg.dflt_mmap.scrfsd =
 #endif
 			    get_amount_value(argv[i]+9, argv, &i);
 		    }
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index f540bae20d..a0d8561e84 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -30,6 +30,7 @@
 #ifdef USE_THREADS
 #include "erl_threads.h"
 #endif
+#include "erl_mmap.h"
 
 #ifdef DEBUG
 #  undef ERTS_ALC_WANT_INLINE
@@ -204,6 +205,7 @@ void erts_free(ErtsAlcType_t type, void *ptr);
 void *erts_alloc_fnf(ErtsAlcType_t type, Uint size);
 void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size);
 int erts_is_allctr_wrapper_prelocked(void);
+int erts_is_literal(void* ptr);
 
 #endif /* #if !ERTS_ALC_DO_INLINE */
 
@@ -281,6 +283,28 @@ int erts_is_allctr_wrapper_prelocked(void)
 	&& !!erts_tsd_get(erts_allctr_prelock_tsd_key);  /* by me  */
 }
 
+ERTS_ALC_INLINE
+int erts_is_literal(void* ptr)
+{
+#if defined(ARCH_32)
+    Uint ix = (UWord)ptr >> ERTS_MMAP_SUPERALIGNED_BITS;
+
+    return erts_literal_vspace_map[ix / ERTS_VSPACE_WORD_BITS]
+                  & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
+
+#elif defined(ARCH_64)
+# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+    extern char* erts_literals_start;
+    extern UWord erts_literals_size;
+    return ErtsInArea(ptr, erts_literals_start, erts_literals_size);
+# else
+#  error Do the tag thing
+# endif
+#else
+# error No ARCH_xx
+#endif
+}
+
 #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
 
 #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id())
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 444d7055c8..f34916f1ab 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -751,12 +751,77 @@ internal_free(void *ptr)
 
 #endif
 
+#ifdef ARCH_32
+
+/*
+ * Bit vector for the entire 32-bit virtual address space
+ * with one bit for each super aligned memory segment.
+ */
+
+#define VSPACE_MAP_BITS  (1 << (32 - ERTS_MMAP_SUPERALIGNED_BITS))
+#define VSPACE_MAP_SZ    (VSPACE_MAP_BITS / ERTS_VSPACE_WORD_BITS)
+
+static ERTS_INLINE void set_bit(UWord* map, Uint ix)
+{
+    ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ);
+    map[ix / ERTS_VSPACE_WORD_BITS]
+        |= ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
+}
+
+static ERTS_INLINE void clr_bit(UWord* map, Uint ix)
+{
+    ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ);
+    map[ix / ERTS_VSPACE_WORD_BITS]
+        &= ~((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
+}
+
+static ERTS_INLINE int is_bit_set(UWord* map, Uint ix)
+{
+    ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ);
+    return map[ix / ERTS_VSPACE_WORD_BITS]
+        & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
+}
+
+UWord erts_literal_vspace_map[VSPACE_MAP_SZ];
+
+static void set_literal_range(void* start, Uint size)
+{
+    Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS;
+    Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS;
+
+    ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK));
+    ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK));
+    ASSERT(n);
+    while (n--) {
+        ASSERT(!is_bit_set(erts_literal_vspace_map, ix));
+        set_bit(erts_literal_vspace_map, ix);
+        ix++;
+    }
+}
+
+static void clear_literal_range(void* start, Uint size)
+{
+    Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS;
+    Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS;
+
+    ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK));
+    ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK));
+    ASSERT(n);
+    while (n--) {
+        ASSERT(is_bit_set(erts_literal_vspace_map, ix));
+        clr_bit(erts_literal_vspace_map, ix);
+        ix++;
+    }
+}
+
+#endif /* ARCH_32 */
+
 /* mseg ... */
 
 #if HAVE_ERTS_MSEG
 
-static ERTS_INLINE void *
-alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+void*
+erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
 {
     void *res;
     UWord size = (UWord) *size_p;
@@ -766,8 +831,9 @@ alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
     return res;
 }
 
-static ERTS_INLINE void *
-alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p)
+void*
+erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg,
+                       Uint old_size, Uint *new_size_p)
 {
     void *res;
     UWord new_size = (UWord) *new_size_p;
@@ -778,17 +844,103 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p)
     return res;
 }
 
-static ERTS_INLINE void
-alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
+void
+erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
 {
     erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt);
     INC_CC(allctr->calls.mseg_dealloc);
 }
 
-#endif
 
-static ERTS_INLINE void *
-alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+#if defined(ARCH_32)
+
+void*
+erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+{
+    void* res;
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
+                   && !allctr->t && allctr->thread_safe);
+
+    res = erts_alcu_mseg_alloc(allctr, size_p, flags);
+    if (res)
+        set_literal_range(res, *size_p);
+    return res;
+}
+
+void*
+erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg,
+                               Uint old_size, Uint *new_size_p)
+{
+    void* res;
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
+                   && !allctr->t && allctr->thread_safe);
+
+    if (seg && old_size)
+        clear_literal_range(seg, old_size);
+    res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p);
+    if (res)
+        set_literal_range(res, *new_size_p);
+    return res;
+}
+
+void
+erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
+                               Uint flags)
+{
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
+                   && !allctr->t && allctr->thread_safe);
+
+    erts_alcu_mseg_dealloc(allctr, seg, size, flags);
+
+    clear_literal_range(seg, size);
+}
+
+#elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+
+void*
+erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+{
+    void* res;
+    UWord size = (UWord) *size_p;
+    Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY;
+    if (flags & ERTS_MSEG_FLG_2POW)
+        mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
+
+    res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size);
+    *size_p = (Uint)size;
+    INC_CC(allctr->calls.mseg_alloc);
+    return res;
+}
+
+void*
+erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg,
+                               Uint old_size, Uint *new_size_p)
+{
+    void *res;
+    UWord new_size = (UWord) *new_size_p;
+    res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
+    *new_size_p = (Uint) new_size;
+    INC_CC(allctr->calls.mseg_realloc);
+    return res;
+}
+
+void
+erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
+                               Uint flags)
+{
+    Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY;
+    if (flags & ERTS_MSEG_FLG_2POW)
+        mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
+
+    erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size);
+    INC_CC(allctr->calls.mseg_dealloc);
+}
+#endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
+
+#endif /* HAVE_ERTS_MSEG */
+
+void*
+erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
 {
     void *res;
 #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
@@ -803,8 +955,8 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
     return res;
 }
 
-static ERTS_INLINE void *
-alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+void*
+erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
 {
     void *res;
 
@@ -824,8 +976,8 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int supe
     return res;
 }
 
-static ERTS_INLINE void
-alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign)
+void
+erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
 {
 #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
     if (superalign)
@@ -838,6 +990,49 @@ alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign)
 	erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr);
 }
 
+#ifdef ARCH_32
+
+void*
+erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+{
+    void* res;
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
+                   && !allctr->t && allctr->thread_safe);
+
+    res = erts_alcu_sys_alloc(allctr, size, 1);
+    if (res)
+        set_literal_range(res, size);
+    return res;
+}
+
+void*
+erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+{
+    void* res;
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
+                   && !allctr->t && allctr->thread_safe);
+
+    if (ptr && old_size)
+        clear_literal_range(ptr, old_size);
+    res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1);
+    if (res)
+        set_literal_range(res, size);
+    return res;
+}
+
+void
+erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
+{
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
+                   && !allctr->t && allctr->thread_safe);
+
+    erts_alcu_sys_dealloc(allctr, ptr, size, 1);
+
+    clear_literal_range(ptr, size);
+}
+
+#endif /* ARCH_32 */
+
 static Uint
 get_next_mbc_size(Allctr_t *allctr)
 {
@@ -3519,6 +3714,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	return NULL;
 #endif
     }
+    flags |= allctr->crr_set_flgs;
+    flags &= ~allctr->crr_clr_flgs;
 
     ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC))
 	   || (flags & CFLG_MBC && !(flags & CFLG_SBC)));
@@ -3602,7 +3799,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
         mseg_flags = ERTS_MSEG_FLG_2POW;
     }
 
-    crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags);
+    crr = (Carrier_t *) allctr->mseg_alloc(allctr, &crr_sz, mseg_flags);
     if (!crr) {
 	have_tried_mseg = 1;
 	if (!(have_tried_sys_alloc || flags & CFLG_FORCE_MSEG))
@@ -3644,12 +3841,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	      ? UNIT_CEILING(bcrr_sz)
 	      : SYS_ALLOC_CARRIER_CEILING(bcrr_sz));
 
-    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+    crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
 	
     if (!crr) {
 	if (crr_sz > UNIT_CEILING(bcrr_sz)) {
 	    crr_sz = UNIT_CEILING(bcrr_sz);
-	    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+	    crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
 	}
 	if (!crr) {
 #if HAVE_ERTS_MSEG
@@ -3748,7 +3945,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 
 	    new_crr_sz = new_blk_sz + SBC_HEADER_SIZE;
 	    new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz);
-	    new_crr = (Carrier_t *) alcu_mseg_realloc(allctr,
+	    new_crr = (Carrier_t *) allctr->mseg_realloc(allctr,
 						      old_crr,
 						      old_crr_sz,
 						      &new_crr_sz);
@@ -3773,7 +3970,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		       (void *) BLK2UMEM(old_blk),
 		       MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ);
 	    unlink_carrier(&allctr->sbc_list, old_crr);
-	    alcu_mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE);
+            allctr->mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE);
 	}
 	else {
 	    /* Old carrier unchanged; restore stat */
@@ -3790,7 +3987,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 			  ? UNIT_CEILING(new_bcrr_sz)
 			  : SYS_ALLOC_CARRIER_CEILING(new_bcrr_sz));
 
-	    new_crr = (Carrier_t *) alcu_sys_realloc(allctr,
+	    new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
 						     (void *) old_crr,
 						     new_crr_sz,
 						     old_crr_sz,
@@ -3811,7 +4008,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 	    else if (new_crr_sz > UNIT_CEILING(new_bcrr_sz)) {
 		new_crr_sz = new_blk_sz + SBC_HEADER_SIZE;
 		new_crr_sz = UNIT_CEILING(new_crr_sz);
-		new_crr = (Carrier_t *) alcu_sys_realloc(allctr,
+		new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
 							 (void *) old_crr,
 							 new_crr_sz,
 							 old_crr_sz,
@@ -3834,7 +4031,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		       (void *) BLK2UMEM(old_blk),
 		       MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ);
 	    unlink_carrier(&allctr->sbc_list, old_crr);
-	    alcu_sys_free(allctr, old_crr, 0);
+	    allctr->sys_dealloc(allctr, old_crr, CARRIER_SZ(old_crr), 0);
 	}
 	else {
 	    /* Old carrier unchanged; restore... */
@@ -3850,13 +4047,13 @@ dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned)
 {
 #if HAVE_ERTS_MSEG
     if (IS_MSEG_CARRIER(crr))
-	alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr),
+	allctr->mseg_dealloc(allctr, crr, CARRIER_SZ(crr),
 			  (superaligned
 			   ? ERTS_MSEG_FLG_2POW
 			   : ERTS_MSEG_FLG_NONE));
     else
 #endif
-	alcu_sys_free(allctr, crr, superaligned);
+	allctr->sys_dealloc(allctr, crr, CARRIER_SZ(crr), superaligned);
 }
 
 static void
@@ -5854,17 +6051,52 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 					    + ABLK_HDR_SZ)
 			       - ABLK_HDR_SZ);
 
+    if (init->sys_alloc) {
+        ASSERT(init->sys_realloc && init->sys_dealloc);
+        allctr->sys_alloc   = init->sys_alloc;
+        allctr->sys_realloc = init->sys_realloc;
+        allctr->sys_dealloc = init->sys_dealloc;
+    }
+    else {
+        ASSERT(!init->sys_realloc && !init->sys_dealloc);
+        allctr->sys_alloc   = &erts_alcu_sys_alloc;
+        allctr->sys_realloc = &erts_alcu_sys_realloc;
+        allctr->sys_dealloc = &erts_alcu_sys_dealloc;
+    }
+#if HAVE_ERTS_MSEG
+    if (init->mseg_alloc) {
+        ASSERT(init->mseg_realloc && init->mseg_dealloc);
+        allctr->mseg_alloc   = init->mseg_alloc;
+        allctr->mseg_realloc = init->mseg_realloc;
+        allctr->mseg_dealloc = init->mseg_dealloc;
+    }
+    else {
+        ASSERT(!init->mseg_realloc && !init->mseg_dealloc);
+        allctr->mseg_alloc   = &erts_alcu_mseg_alloc;
+        allctr->mseg_realloc = &erts_alcu_mseg_realloc;
+        allctr->mseg_dealloc = &erts_alcu_mseg_dealloc;
+    }
+    /* If a custom carrier alloc function is specified, make sure it's used */
+    if (init->mseg_alloc && !init->sys_alloc) {
+        allctr->crr_set_flgs = CFLG_FORCE_MSEG;
+        allctr->crr_clr_flgs = CFLG_FORCE_SYS_ALLOC;
+    }
+    else if (!init->mseg_alloc && init->sys_alloc) {
+        allctr->crr_set_flgs = CFLG_FORCE_SYS_ALLOC;
+        allctr->crr_clr_flgs = CFLG_FORCE_MSEG;
+    }
+#endif
+
     if (allctr->main_carrier_size) {
 	Block_t *blk;
 
 	blk = create_carrier(allctr,
 			     allctr->main_carrier_size,
-			     CFLG_MBC
+                             (ERTS_SUPER_ALIGNED_MSEG_ONLY
+                              ? CFLG_FORCE_MSEG : CFLG_FORCE_SYS_ALLOC)
+                             | CFLG_MBC
 			     | CFLG_FORCE_SIZE
 			     | CFLG_NO_CPOOL
-#if !ERTS_SUPER_ALIGNED_MSEG_ONLY
-			     | CFLG_FORCE_SYS_ALLOC
-#endif
 			     | CFLG_MAIN_CARRIER);
 	if (!blk) {
 #ifdef USE_THREADS
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f314eab4c2..16ad673d26 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -24,6 +24,12 @@
 #define ERTS_ALCU_VSN_STR "3.0"
 
 #include "erl_alloc_types.h"
+#ifdef USE_THREADS
+#define ERL_THREADS_EMU_INTERNAL__
+#include "erl_threads.h"
+#endif
+
+#include "erl_mseg.h"
 
 #define ERTS_AU_PREF_ALLOC_BITS 11
 #define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS)
@@ -60,6 +66,15 @@ typedef struct {
 
     void *fix;
     size_t *fix_type_size;
+
+#if HAVE_ERTS_MSEG
+    void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
+    void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+    void  (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+#endif
+    void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
+    void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
+    void  (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
 } AllctrInit_t;
 
 typedef struct {
@@ -173,20 +188,44 @@ void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *
 #endif
 erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
+#ifdef ARCH_32
+extern UWord erts_literal_vspace_map[];
+# define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8)
+#endif
+
+void* erts_alcu_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void  erts_alcu_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+
+#if HAVE_ERTS_MSEG
+# if defined(ARCH_32)
+void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void  erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+
+# elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+void* erts_alcu_literal_64_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_literal_64_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void  erts_alcu_literal_64_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+# endif
+#endif /* HAVE_ERTS_MSEG */
+
+void* erts_alcu_sys_alloc(Allctr_t*, Uint size, int superalign);
+void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign);
+void  erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
+#ifdef ARCH_32
+void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint size, int superalign);
+void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign);
+void  erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
 #endif
 
+#endif /* !ERL_ALLOC_UTIL__ */
+
 #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
 #define ERL_ALLOC_UTIL_IMPL__
 
 #define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE		(((Uint32) 1) << 0)
 
-#ifdef USE_THREADS
-#define ERL_THREADS_EMU_INTERNAL__
-#include "erl_threads.h"
-#endif
-
-#include "erl_mseg.h"
-
 #undef ERTS_ALLOC_UTIL_HARD_DEBUG
 #ifdef DEBUG
 #  if 0
@@ -498,6 +537,8 @@ struct Allctr_t_ {
     Uint		min_mbc_size;
     Uint		min_mbc_first_free_size;
     Uint		min_block_size;
+    UWord               crr_set_flgs;
+    UWord               crr_clr_flgs;
 
     /* Carriers */
     CarrierList_t	mbc_list;
@@ -543,6 +584,15 @@ struct Allctr_t_ {
     void		(*remove_mbc)	        (Allctr_t *, Carrier_t *);
     UWord		(*largest_fblk_in_mbc)  (Allctr_t *, Carrier_t *);
 
+#if HAVE_ERTS_MSEG
+    void*               (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
+    void*               (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+    void                (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+#endif
+    void*               (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
+    void*               (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
+    void                (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
+
     void		(*init_atoms)		(void);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 34011147d9..46577bd07f 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1045,6 +1045,9 @@ extern int erts_use_kernel_poll;
 #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)
 
 
+#define ErtsInArea(PTR,START,NBYTES) \
+    ((UWord)((char*)(PTR) - (char*)(START)) < (NBYTES))
+
 /*
  * Use DEBUGF as you would use printf, but use double parentheses:
  *
-- 
cgit v1.2.3


From 49d2f809cf8435b17d54f0fd2f37a8aa939ea457 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Sep 2015 12:01:17 +0200
Subject: fix check_process_code for separate literal area

---
 erts/emulator/beam/beam_bif_load.c | 28 ++++++++++++----------------
 erts/emulator/beam/erl_gc.c        |  4 ++--
 2 files changed, 14 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 68689b8e7f..adc3bd37d9 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -728,8 +728,8 @@ static Eterm
 check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 {
     BeamInstr* start;
-    char* mod_start;
-    Uint mod_size;
+    char* literals;
+    Uint lit_bsize;
     BeamInstr* end;
     Eterm* sp;
     struct erl_off_heap_header* oh;
@@ -742,8 +742,6 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
      */
     start = modp->old.code;
     end = (BeamInstr *)((char *)start + modp->old.code_length);
-    mod_start = (char *) start;
-    mod_size = modp->old.code_length;
 
     /*
      * Check if current instruction or continuation pointer points into module.
@@ -834,22 +832,24 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
      * See if there are constants inside the module referenced by the process.
      */
     done_gc = 0;
+    literals = (char*) modp->old.code[MI_LITERALS_START];
+    lit_bsize = (char*) modp->old.code[MI_LITERALS_END] - literals;
     for (;;) {
 	ErlMessage* mp;
 
-	if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, mod_start, mod_size)) {
+	if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) {
 	    rp->freason = EXC_NULL;
 	    rp->fvalue = NIL;
 	    rp->ftrace = NIL;
 	}
-	if (any_heap_ref_ptrs(rp->stop, rp->hend, mod_start, mod_size)) {
+	if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) {
 	    goto need_gc;
 	}
-	if (any_heap_refs(rp->heap, rp->htop, mod_start, mod_size)) {
+	if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) {
 	    goto need_gc;
 	}
 
-	if (any_heap_refs(rp->old_heap, rp->old_htop, mod_start, mod_size)) {
+	if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) {
 	    goto need_gc;
 	}
 
@@ -857,13 +857,13 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	    Eterm* start = rp->dictionary->data;
 	    Eterm* end = start + rp->dictionary->used;
 
-	    if (any_heap_ref_ptrs(start, end, mod_start, mod_size)) {
+	    if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) {
 		goto need_gc;
 	    }
 	}
 
 	for (mp = rp->msg.first; mp != NULL; mp = mp->next) {
-	    if (any_heap_ref_ptrs(mp->m, mp->m+2, mod_start, mod_size)) {
+	    if (any_heap_ref_ptrs(mp->m, mp->m+2, literals, lit_bsize)) {
 		goto need_gc;
 	    }
 	}
@@ -873,8 +873,6 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	if (done_gc) {
 	    return am_true;
 	} else {
-	    Eterm* literals;
-	    Uint lit_size;
 	    struct erl_off_heap_header* oh;
 
 	    if (!allow_gc)
@@ -890,12 +888,10 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	    done_gc = 1;
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
 	    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
-	    literals = (Eterm *) modp->old.code[MI_LITERALS_START];
-	    lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals;
 	    oh = (struct erl_off_heap_header *)
 		modp->old.code[MI_LITERALS_OFF_HEAP];
-	    *redsp += lit_size / 10; /* Need, better value... */
-	    erts_garbage_collect_literals(rp, literals, lit_size, oh);
+	    *redsp += lit_bsize / 64; /* Need, better value... */
+	    erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh);
 	}
     }
     return am_false;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 734f120e09..89fabde67c 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -645,10 +645,10 @@ erts_garbage_collect_hibernate(Process* p)
 
 void
 erts_garbage_collect_literals(Process* p, Eterm* literals,
-			      Uint lit_size,
+			      Uint byte_lit_size,
 			      struct erl_off_heap_header* oh)
 {
-    Uint byte_lit_size = sizeof(Eterm)*lit_size;
+    Uint lit_size = byte_lit_size / sizeof(Eterm);
     Uint old_heap_size;
     Eterm* temp_lit;
     Sint offs;
-- 
cgit v1.2.3


From b21b604137c5cb5f5039a40994e429871e5b707b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 26 Aug 2015 19:47:10 +0200
Subject: Introduce literal tag

---
 erts/emulator/beam/beam_emu.c     |  3 +++
 erts/emulator/beam/beam_load.c    | 11 +++++-----
 erts/emulator/beam/copy.c         | 29 +++++++++++++++++++-------
 erts/emulator/beam/erl_alloc.h    | 20 ++++++++++++------
 erts/emulator/beam/erl_bif_info.c | 17 +++++++++++++++
 erts/emulator/beam/erl_gc.c       | 42 ++++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_gc.h       |  8 +++----
 erts/emulator/beam/erl_message.c  |  8 +++----
 erts/emulator/beam/erl_term.c     | 31 +++++++++++++++++++++++++++
 erts/emulator/beam/erl_term.h     | 44 ++++++++++++++++++++++++++++++++++++---
 erts/emulator/beam/global.h       | 25 ++++++++++++++++++++--
 erts/emulator/beam/sys.h          |  7 +++----
 12 files changed, 206 insertions(+), 39 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 6e4c8ecee3..4d19f52a52 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2963,6 +2963,9 @@ do {						\
 		     }
 		 }
 		 Op1 = small_to_big(ires, tmp_big);
+#ifdef TAG_LITERAL_PTR
+		 Op1 |= TAG_LITERAL_PTR;
+#endif
 
 	     big_shift:
 		 if (i > 0) {	/* Left shift. */
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 1c598601c6..7a1a563be2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4439,12 +4439,13 @@ freeze_code(LoaderState* stp)
 	code_hdr->literals_start = ptr;
 	code_hdr->literals_end = ptr + stp->total_literal_size;
 	for (i = 0; i < stp->num_literals; i++) {
-            if (stp->literals[i].heap_frags) {
-                move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags,
-                                 &stp->literals[i].term, 1);
-                ASSERT(erts_is_literal(ptr_val(stp->literals[i].term)));
+            if (is_not_immed(stp->literals[i].term)) {
+                erts_move_multi_frags(&ptr, &code_off_heap,
+				      stp->literals[i].heap_frags,
+				      &stp->literals[i].term, 1, 1);
+                ASSERT(erts_is_literal(stp->literals[i].term,
+				       ptr_val(stp->literals[i].term)));
             }
-            else ASSERT(is_immed(stp->literals[i].term));
 	}
 	code_hdr->literals_off_heap = code_off_heap.first;
 	lp = stp->literal_patches;
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index ec769c3b49..b185758b1d 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -35,7 +35,7 @@
 #include "erl_bits.h"
 #include "dtrace-wrapper.h"
 
-static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*);
+static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int);
 
 /*
  *  Copy object "obj" to process p.
@@ -621,17 +621,24 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
  * move markers.
  * Typically used to copy a multi-fragmented message (from NIF).
  */
-void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
-		      Eterm* refs, unsigned nrefs)
+void erts_move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
+			   Eterm* refs, unsigned nrefs, int literals)
 {
     ErlHeapFragment* bp;
     Eterm* hp_start = *hpp;
     Eterm* hp_end;
     Eterm* hp;
     unsigned i;
+    Eterm literal_tag;
+
+#ifdef TAG_LITERAL_PTR
+    literal_tag = (Eterm) literals ? TAG_LITERAL_PTR : 0;
+#else
+    literal_tag = (Eterm) 0;
+#endif
 
     for (bp=first; bp!=NULL; bp=bp->next) {
-	move_one_frag(hpp, bp, off_heap);
+	move_one_frag(hpp, bp, off_heap, literals);
     }
     hp_end = *hpp;
     for (hp=hp_start; hp<hp_end; ++hp) {
@@ -644,6 +651,9 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 	    val = *ptr;
 	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
+#ifdef TAG_LITERAL_PTR
+		val |= literal_tag;
+#endif
 		*hp = val;
 	    }
 	    break;
@@ -651,7 +661,11 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 	    ptr = list_val(gval);
 	    val = *ptr;
 	    if (IS_MOVED_CONS(val)) {
-		*hp = ptr[1];
+		val = ptr[1];
+#ifdef TAG_LITERAL_PTR
+		val |= literal_tag;
+#endif
+		*hp = val;
 	    }
 	    break;
 	case TAG_PRIMARY_HEADER:
@@ -662,12 +676,12 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 	}
     }
     for (i=0; i<nrefs; ++i) {
-	refs[i] = follow_moved(refs[i]);
+	refs[i] = follow_moved(refs[i], literal_tag);
     }
 }
 
 static void
-move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap)
+move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int literals)
 {
     Eterm* ptr = frag->mem;
     Eterm* end = ptr + frag->used_size;
@@ -704,4 +718,3 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap)
     OH_OVERHEAD(off_heap, frag->off_heap.overhead);
     frag->off_heap.first = NULL;
 }
-
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index a0d8561e84..9da9c823f7 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -178,6 +178,12 @@ void  sys_free(void *)               __deprecated; /* erts_free()        */
 void *sys_alloc(Uint )               __deprecated; /* erts_alloc_fnf()   */
 void *sys_realloc(void *, Uint)      __deprecated; /* erts_realloc_fnf() */
 
+#undef ERTS_HAVE_IS_IN_LITERAL_RANGE
+#if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+#  define ERTS_HAVE_IS_IN_LITERAL_RANGE
+#endif
+
+
 /*
  * erts_alloc[_fnf](), erts_realloc[_fnf](), erts_free() works as
  * malloc(), realloc(), and free() with the following exceptions:
@@ -205,7 +211,9 @@ void erts_free(ErtsAlcType_t type, void *ptr);
 void *erts_alloc_fnf(ErtsAlcType_t type, Uint size);
 void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size);
 int erts_is_allctr_wrapper_prelocked(void);
-int erts_is_literal(void* ptr);
+#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
+int erts_is_in_literal_range(void* ptr);
+#endif
 
 #endif /* #if !ERTS_ALC_DO_INLINE */
 
@@ -283,8 +291,10 @@ int erts_is_allctr_wrapper_prelocked(void)
 	&& !!erts_tsd_get(erts_allctr_prelock_tsd_key);  /* by me  */
 }
 
+#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
+
 ERTS_ALC_INLINE
-int erts_is_literal(void* ptr)
+int erts_is_in_literal_range(void* ptr)
 {
 #if defined(ARCH_32)
     Uint ix = (UWord)ptr >> ERTS_MMAP_SUPERALIGNED_BITS;
@@ -293,18 +303,16 @@ int erts_is_literal(void* ptr)
                   & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
 
 #elif defined(ARCH_64)
-# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     extern char* erts_literals_start;
     extern UWord erts_literals_size;
     return ErtsInArea(ptr, erts_literals_start, erts_literals_size);
-# else
-#  error Do the tag thing
-# endif
 #else
 # error No ARCH_xx
 #endif
 }
 
+#endif /* ERTS_HAVE_IS_IN_LITERAL_RANGE */
+
 #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
 
 #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id())
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a73ad826db..a684c81445 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2766,6 +2766,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) {
 	BIF_RET(erts_eager_check_io ? am_true : am_false);
     }
+    else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) {
+#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
+#ifdef ARCH_64
+	DECL_AM(range);
+	BIF_RET(AM_range);
+#else /* ARCH_32 */
+	DECL_AM(range_bitmask);
+	BIF_RET(AM_range_bitmask);
+#endif /* ARCH_32 */
+#else  /* ! ERTS_HAVE_IS_IN_LITERAL_RANGE */
+	DECL_AM(tag);
+	BIF_RET(AM_tag);
+#endif
+    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -4323,12 +4337,15 @@ static void os_info_init(void)
     erts_free(ERTS_ALC_T_TMP, (void *) buf);
     hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm));
     os_type_tuple = TUPLE2(hp, type, flav);
+    erts_set_literal_tag(&os_type_tuple, hp, 3);
+
     hp += 3;
     os_version(&major, &minor, &build);
     os_version_tuple = TUPLE3(hp,
 			      make_small(major),
 			      make_small(minor),
 			      make_small(build));
+    erts_set_literal_tag(&os_version_tuple, hp, 4);
 }
 
 void
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 89fabde67c..e316ab95ab 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1004,6 +1004,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
     Uint mature_size = (char *) HIGH_WATER(p) - heap;
     Eterm* old_htop = OLD_HTOP(p);
     Eterm* n_heap;
+    char* oh = (char *) OLD_HEAP(p);
+    Uint oh_size = (char *) OLD_HTOP(p) - oh;
 
     n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 					       sizeof(Eterm)*new_sz);
@@ -1035,8 +1037,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 } else if (in_area(ptr, heap, mature_size)) {
                     MOVE_BOXED(ptr,val,old_htop,g_ptr++);
                 } else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
                     MOVE_BOXED(ptr,val,n_htop,g_ptr++);
                 } else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
                 break;
@@ -1050,8 +1056,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 } else if (in_area(ptr, heap, mature_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
                 } else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
                     MOVE_CONS(ptr,val,n_htop,g_ptr++);
                 } else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
 		break;
@@ -1094,8 +1104,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		} else if (in_area(ptr, heap, mature_size)) {
 		    MOVE_BOXED(ptr,val,old_htop,n_hp++);
 		} else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
 		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1108,8 +1122,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		} else if (in_area(ptr, heap, mature_size)) {
 		    MOVE_CONS(ptr,val,old_htop,n_hp++);
 		} else if (in_area(ptr, heap, heap_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr)
+			   && !in_area(ptr, oh, oh_size));
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr)
+			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1131,9 +1149,15 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 			    MOVE_BOXED(ptr,val,old_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			} else if (in_area(ptr, heap, heap_size)) {
+			    ASSERT(!erts_is_literal(*origptr, origptr)
+				   && !in_area(ptr, oh, oh_size));
 			    MOVE_BOXED(ptr,val,n_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			}
+			else {
+			    ASSERT(erts_is_literal(*origptr, origptr)
+				   || in_area(ptr, oh, oh_size));
+			}
 		    }
 		    n_hp += (thing_arityval(gval)+1);
 		}
@@ -1275,8 +1299,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_BOXED(ptr,val,n_htop,g_ptr++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1288,8 +1314,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1330,8 +1358,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    n_hp++;
 		}
 		break;
@@ -1342,8 +1372,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
+		    ASSERT(!erts_is_literal(gval, ptr));
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
 		} else {
+		    ASSERT(erts_is_literal(gval, ptr));
 		    n_hp++;
 		}
 		break;
@@ -1363,12 +1395,16 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 			    *origptr = val;
 			    mb->base = binary_bytes(*origptr);
 			} else if (in_area(ptr, src, src_size) ||
-				   in_area(ptr, oh, oh_size)) {
+-				   in_area(ptr, oh, oh_size)) {
+			    ASSERT(!erts_is_literal(*origptr, origptr));
 			    MOVE_BOXED(ptr,val,n_htop,origptr); 
 			    mb->base = binary_bytes(*origptr);
 			    ptr = boxed_val(*origptr);
 			    val = *ptr;
 			}
+			else {
+			    ASSERT(erts_is_literal(*origptr, origptr));
+			}
 		    }
 		    n_hp += (thing_arityval(gval)+1);
 		}
@@ -1985,7 +2021,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
     }
 
     ASSERT((is_nil(p->seq_trace_token) ||
-	    is_tuple(follow_moved(p->seq_trace_token)) ||
+	    is_tuple(follow_moved(p->seq_trace_token, (Eterm) 0)) ||
 	    is_atom(p->seq_trace_token)));
     if (is_not_immed(p->seq_trace_token)) {
 	roots[n].v = &p->seq_trace_token;
@@ -2003,7 +2039,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	   is_internal_pid(ERTS_TRACER_PROC(p)) ||
 	   is_internal_port(ERTS_TRACER_PROC(p)));
 
-    ASSERT(is_pid(follow_moved(p->group_leader)));
+    ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0)));
     if (is_not_immed(p->group_leader)) {
 	roots[n].v  = &p->group_leader;
 	roots[n].sz = 1;
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index ecd1bf4d22..fdbf948f9d 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -76,10 +76,10 @@ do {                                                                    \
 int within(Eterm *ptr, Process *p);
 #endif
 
-ERTS_GLB_INLINE Eterm follow_moved(Eterm term);
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag)
 {
     Eterm* ptr;
     switch (primary_tag(term)) {
@@ -87,11 +87,11 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
 	break;
     case TAG_PRIMARY_BOXED:
 	ptr = boxed_val(term);
-	if (IS_MOVED_BOXED(*ptr)) term = *ptr;
+	if (IS_MOVED_BOXED(*ptr)) term = (*ptr) | xptr_tag;
 	break;
     case TAG_PRIMARY_LIST:
 	ptr = list_val(term);
-	if (IS_MOVED_CONS(ptr[0])) term = ptr[1];
+	if (IS_MOVED_CONS(ptr[0])) term = (ptr[1]) | xptr_tag;
 	break;
     default:
 	ASSERT(!"strange tag in follow_moved");
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ef52823287..e23c79d301 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -642,13 +642,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 #endif
 
     if (bp->next != NULL) {
-	move_multi_frags(hpp, off_heap, bp, msg->m, 
+	erts_move_multi_frags(hpp, off_heap, bp, msg->m, 
 #ifdef USE_VM_PROBES
-			 3
+			      3,
 #else
-			 2
+			      2,
 #endif
-			 );
+			      0);
 	goto copy_done;
     }
 
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 3a5fbcc284..8efbb8c554 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -28,6 +28,37 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+void
+erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz)
+{
+#ifdef TAG_LITERAL_PTR
+    Eterm *hp_end, *hp;
+    
+    hp_end = hp_start + hsz;
+    hp = hp_start;
+
+    while (hp < hp_end) {
+	switch (primary_tag(*hp)) {
+	case TAG_PRIMARY_BOXED:
+	case TAG_PRIMARY_LIST:
+	    *hp |= TAG_LITERAL_PTR;
+	    break;
+	case TAG_PRIMARY_HEADER:
+	    if (header_is_thing(*hp)) {
+		hp += thing_arityval(*hp);
+	    }
+	    break;
+	default:
+	    break;
+	}
+	
+	hp++;
+    }
+    if (is_boxed(*term) || is_list(*term))
+	*term |= TAG_LITERAL_PTR;
+#endif
+}
+
 __decl_noreturn static void __noreturn
 et_abort(const char *expr, const char *file, unsigned line)
 {
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 089eecf024..a80716fb0f 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -21,6 +21,8 @@
 #ifndef __ERL_TERM_H
 #define __ERL_TERM_H
 
+#include "erl_mmap.h"
+
 typedef UWord Wterm;  /* Full word terms */
 
 struct erl_node_; /* Declared in erl_node_tables.h */
@@ -48,6 +50,24 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #define _ET_APPLY(F,X)	_unchecked_##F(X)
 #endif
 
+#if defined(ARCH_64)
+#  define TAG_PTR_MASK__	0x7
+#  if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+#    ifdef HIPE
+#      error Hipe on 64-bit needs a real mmap as it does not support the literal tag
+#    endif
+#    define TAG_LITERAL_PTR	0x4
+#  else
+#    undef TAG_LITERAL_PTR
+#  endif
+#elif defined(ARCH_32)
+#  define TAG_PTR_MASK__	0x3
+#  undef TAG_LITERAL_PTR
+#else
+#  error Not supported arch
+#endif
+
+
 #define _TAG_PRIMARY_SIZE	2
 #define _TAG_PRIMARY_MASK	0x3
 #define TAG_PRIMARY_HEADER	0x0
@@ -165,10 +185,11 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 
 /* boxed object access methods */
 
-#define _is_taggable_pointer(x)	 (((Uint)(x) & 0x3) == 0)
+#define _is_taggable_pointer(x)	 (((Uint)(x) & TAG_PTR_MASK__) == 0)
+
 #define  _boxed_precond(x)       (is_boxed(x))
 
-#define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
+#define _is_aligned(x)		(((Uint)(x) & TAG_PTR_MASK__) == 0)
 #define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED)
 _ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*)
 #define make_boxed(x)		_ET_APPLY(make_boxed,(x))
@@ -180,7 +201,11 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm)
 #else
 #define is_boxed(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
 #endif
+#ifdef TAG_LITERAL_PTR
+#define _unchecked_boxed_val(x) _unchecked_ptr_val(x)
+#else
 #define _unchecked_boxed_val(x) ((Eterm*) ((x) - TAG_PRIMARY_BOXED))
+#endif
 _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
 #define boxed_val(x)		_ET_APPLY(boxed_val,(x))
 
@@ -198,7 +223,11 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
 #define is_not_list(x)		(!is_list((x)))
 #endif
 #define _list_precond(x)        (is_list(x))
+#ifdef TAG_LITERAL_PTR
+#define _unchecked_list_val(x) _unchecked_ptr_val(x)
+#else
 #define _unchecked_list_val(x) ((Eterm*) ((x) - TAG_PRIMARY_LIST))
+#endif
 _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define list_val(x)		_ET_APPLY(list_val,(x))
 
@@ -209,13 +238,20 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define CDR(x)  ((x)[1])
 
 /* generic tagged pointer (boxed or list) access methods */
-#define _unchecked_ptr_val(x)	((Eterm*) ((x) & ~((Uint) 0x3)))
+#define _unchecked_ptr_val(x)	((Eterm*) ((x) & ~((Uint) TAG_PTR_MASK__)))
 #define ptr_val(x)		_unchecked_ptr_val((x))	/*XXX*/
 #define _unchecked_offset_ptr(x,offs)	((x)+((offs)*sizeof(Eterm)))
 #define offset_ptr(x,offs)	_unchecked_offset_ptr(x,offs)	/*XXX*/
 #define _unchecked_byte_offset_ptr(x,byte_offs)	((x)+(offs))
 #define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs)  /*XXX*/
 
+#ifdef TAG_LITERAL_PTR
+#define _unchecked_is_not_literal_ptr(x) (!((x) & TAG_LITERAL_PTR))
+#define is_not_literal_ptr(x) _unchecked_is_not_literal_ptr((x)) /*XXX*/
+#define is_literal_ptr(x) (!is_not_literal_ptr((x))) /*XXX*/
+#endif
+
+
 /* fixnum ("small") access methods */
 #if defined(ARCH_64)
 #define SMALL_BITS	(64-4)
@@ -1113,5 +1149,7 @@ extern unsigned tag_val_def(Wterm);
 
 #define is_same(A,B) ((A)==(B))
 
+void erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz);
+
 #endif	/* __ERL_TERM_H */
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b4d02dd1dd..052994b972 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -961,8 +961,8 @@ Uint size_object(Eterm);
 Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
 
-void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
-		      Eterm* refs, unsigned nrefs);
+void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
+			   Eterm* refs, unsigned nrefs, int literals);
 
 /* Utilities */
 extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks);
@@ -1274,6 +1274,27 @@ int erts_print_system_version(int to, void *arg, Process *c_p);
 
 int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
 
+ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr)
+{
+    ASSERT(is_boxed(tptr) || is_list(tptr));
+    ASSERT(ptr == ptr_val(tptr));
+
+#if defined(ERTS_HAVE_IS_IN_LITERAL_RANGE)
+    return erts_is_in_literal_range(ptr);
+#elif defined(TAG_LITERAL_PTR)
+    return is_literal_ptr(tptr);
+#else
+#  error Not able to detect literals...
+#endif
+
+}
+
+#endif
+
 /*
 ** Call_trace uses this API for the parameter matching functions
 */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 46577bd07f..d63e81cb5e 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -72,6 +72,9 @@
 
 #define ERTS_I64_LITERAL(X) X##LL
 
+#define ErtsInArea(ptr,start,nbytes) \
+    ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
+
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
 #else
@@ -1044,10 +1047,6 @@ extern int erts_use_kernel_poll;
 
 #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)
 
-
-#define ErtsInArea(PTR,START,NBYTES) \
-    ((UWord)((char*)(PTR) - (char*)(START)) < (NBYTES))
-
 /*
  * Use DEBUGF as you would use printf, but use double parentheses:
  *
-- 
cgit v1.2.3


From b9caedf093d0ccf268562656e28cdda6a02631cb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 2 Oct 2015 19:05:57 +0200
Subject: erts: Refactor header of loaded beam code

to use a real C struct instead of array.
---
 erts/emulator/beam/beam_bif_load.c |  75 ++++----
 erts/emulator/beam/beam_bp.c       |  30 +--
 erts/emulator/beam/beam_debug.c    |   8 +-
 erts/emulator/beam/beam_emu.c      |   2 +-
 erts/emulator/beam/beam_load.c     | 384 +++++++++++++++++++------------------
 erts/emulator/beam/beam_load.h     | 112 +++++------
 erts/emulator/beam/beam_ranges.c   |  17 +-
 erts/emulator/beam/break.c         |  30 +--
 erts/emulator/beam/erl_nif.c       |  16 +-
 erts/emulator/beam/module.c        |   4 +-
 erts/emulator/beam/module.h        |   2 +-
 11 files changed, 345 insertions(+), 335 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index adc3bd37d9..2c275c4649 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -40,7 +40,7 @@
 static void set_default_trace_pattern(Eterm module);
 static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp);
 static void delete_code(Module* modp);
-static void decrement_refc(BeamInstr* code);
+static void decrement_refc(BeamCodeHeader*);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 
@@ -58,8 +58,8 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
 	return am_undefined;
     }
     erts_rlock_old_code(code_ix);
-    res = (erts_is_module_native(modp->curr.code) ||
-           erts_is_module_native(modp->old.code)) ?
+    res = (erts_is_module_native(modp->curr.code_hdr) ||
+           erts_is_module_native(modp->old.code_hdr)) ?
 		am_true : am_false;
     erts_runlock_old_code(code_ix);
     return res;
@@ -81,7 +81,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
 
     if (modp && modp->curr.num_breakpoints > 0) {
-	ASSERT(modp->curr.code != NULL);
+	ASSERT(modp->curr.code_hdr != NULL);
 	erts_clear_module_break(modp);
 	ASSERT(modp->curr.num_breakpoints == 0);
     }
@@ -281,7 +281,7 @@ finish_loading_1(BIF_ALIST_1)
     exceptions = 0;
     for (i = 0; i < n; i++) {
 	p[i].exception = 0;
-	if (p[i].modp->curr.code && p[i].modp->old.code) {
+	if (p[i].modp->curr.code_hdr && p[i].modp->old.code_hdr) {
 	    p[i].exception = 1;
 	    exceptions++;
 	}
@@ -417,7 +417,7 @@ check_old_code_1(BIF_ALIST_1)
     modp = erts_get_module(BIF_ARG_1, code_ix);
     if (modp != NULL) {
 	erts_rlock_old_code(code_ix);
-	if (modp->old.code != NULL) {
+	if (modp->old.code_hdr) {
 	    res = am_true;
 	}
 	erts_runlock_old_code(code_ix);
@@ -441,7 +441,7 @@ erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp)
     if (!modp)
 	return am_false;
     erts_rlock_old_code(code_ix);
-    res = modp->old.code ? check_process_code(c_p, modp, allow_gc, redsp) : am_false;
+    res = modp->old.code_hdr ? check_process_code(c_p, modp, allow_gc, redsp) : am_false;
     erts_runlock_old_code(code_ix);
 
     return res;
@@ -525,7 +525,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	if (!modp) {
 	    res = am_undefined;
 	}
-	else if (modp->old.code != 0) {
+	else if (modp->old.code_hdr) {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	    erts_dsprintf(dsbufp, "Module %T must be purged before loading\n",
 			  BIF_ARG_1);
@@ -563,8 +563,8 @@ BIF_RETTYPE module_loaded_1(BIF_ALIST_1)
     }
     code_ix = erts_active_code_ix();
     if ((modp = erts_get_module(BIF_ARG_1, code_ix)) != NULL) {
-	if (modp->curr.code != NULL
-	    && modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] == 0) {
+	if (modp->curr.code_hdr
+	    && modp->curr.code_hdr->on_load_function_ptr == NULL) {
 	    res = am_true;
 	}
     }
@@ -611,8 +611,8 @@ BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1)
 {
     Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
 
-    if (modp && modp->curr.code) {
-	BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]);
+    if (modp && modp->curr.code_hdr) {
+	BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code_hdr->on_load_function_ptr);
     }
     else {
 	BIF_ERROR(BIF_P, BADARG);
@@ -623,7 +623,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 {
     ErtsCodeIndex code_ix;
     Module* modp;
-    Eterm on_load;
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
 	ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2],
@@ -638,14 +637,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
     code_ix = erts_active_code_ix();
     modp = erts_get_module(BIF_ARG_1, code_ix);
 
-    if (!modp || modp->curr.code == 0) {
+    if (!modp || !modp->curr.code_hdr) {
     error:
 	erts_smp_thr_progress_unblock();
         erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_release_code_write_permission();
 	BIF_ERROR(BIF_P, BADARG);
     }
-    if ((on_load = modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]) == 0) {
+    if (modp->curr.code_hdr->on_load_function_ptr == NULL) {
 	goto error;
     }
     if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) {
@@ -667,7 +666,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 		ep->code[4] = 0;
 	    }
 	}
-	modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] = 0;
+	modp->curr.code_hdr->on_load_function_ptr = NULL;
 	set_default_trace_pattern(BIF_ARG_1);
     } else if (BIF_ARG_2 == am_false) {
 	BeamInstr* code;
@@ -679,16 +678,16 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	 * the current code; the old code is not touched.
 	 */
 	erts_total_code_size -= modp->curr.code_length;
-	code = modp->curr.code;
-	end = (BeamInstr *)((char *)code + modp->curr.code_length);
+	code = (BeamInstr*) modp->curr.code_hdr;
+	end = (BeamInstr *) ((char *)code + modp->curr.code_length);
 	erts_cleanup_funs_on_purge(code, end);
 	beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length,
 			    erts_active_code_ix());
-        if (code[MI_LITERALS_START]) {
-            erts_free(ERTS_ALC_T_LITERAL, (void *) code[MI_LITERALS_START]);
+        if (modp->curr.code_hdr->literals_start) {
+            erts_free(ERTS_ALC_T_LITERAL, modp->curr.code_hdr->literals_start);
         }
-	erts_free(ERTS_ALC_T_CODE, (void *) code);
-	modp->curr.code = NULL;
+	erts_free(ERTS_ALC_T_CODE, modp->curr.code_hdr);
+	modp->curr.code_hdr = NULL;
 	modp->curr.code_length = 0;
 	modp->curr.catches = BEAM_CATCHES_NIL;
 	erts_remove_from_ranges(code);
@@ -740,7 +739,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
     /*
      * Pick up limits for the module.
      */
-    start = modp->old.code;
+    start = (BeamInstr*) modp->old.code_hdr;
     end = (BeamInstr *)((char *)start + modp->old.code_length);
 
     /*
@@ -832,8 +831,8 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
      * See if there are constants inside the module referenced by the process.
      */
     done_gc = 0;
-    literals = (char*) modp->old.code[MI_LITERALS_START];
-    lit_bsize = (char*) modp->old.code[MI_LITERALS_END] - literals;
+    literals = (char*) modp->old.code_hdr->literals_start;
+    lit_bsize = (char*) modp->old.code_hdr->literals_end - literals;
     for (;;) {
 	ErlMessage* mp;
 
@@ -888,8 +887,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	    done_gc = 1;
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
 	    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
-	    oh = (struct erl_off_heap_header *)
-		modp->old.code[MI_LITERALS_OFF_HEAP];
+	    oh = modp->old.code_hdr->literals_off_heap;
 	    *redsp += lit_bsize / 64; /* Need, better value... */
 	    erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh);
 	}
@@ -989,7 +987,7 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 	/*
 	 * Any code to purge?
 	 */
-	if (modp->old.code == 0) {
+	if (!modp->old.code_hdr) {
 	    ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
 	}
 	else {
@@ -1012,17 +1010,17 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 	     */
 	    ASSERT(erts_total_code_size >= modp->old.code_length);
 	    erts_total_code_size -= modp->old.code_length;
-	    code = modp->old.code;
+	    code = (BeamInstr*) modp->old.code_hdr;
 	    end = (BeamInstr *)((char *)code + modp->old.code_length);
 	    erts_cleanup_funs_on_purge(code, end);
 	    beam_catches_delmod(modp->old.catches, code, modp->old.code_length,
 				code_ix);
-	    decrement_refc(code);
-            if (code[MI_LITERALS_START]) {
-                erts_free(ERTS_ALC_T_LITERAL, (void *) code[MI_LITERALS_START]);
+	    decrement_refc(modp->old.code_hdr);
+            if (modp->old.code_hdr->literals_start) {
+                erts_free(ERTS_ALC_T_LITERAL, modp->old.code_hdr->literals_start);
             }
 	    erts_free(ERTS_ALC_T_CODE, (void *) code);
-	    modp->old.code = NULL;
+	    modp->old.code_hdr = NULL;
 	    modp->old.code_length = 0;
 	    modp->old.catches = BEAM_CATCHES_NIL;
 	    erts_remove_from_ranges(code);
@@ -1039,10 +1037,9 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 }
 
 static void
-decrement_refc(BeamInstr* code)
+decrement_refc(BeamCodeHeader* code_hdr)
 {
-    struct erl_off_heap_header* oh =
-	(struct erl_off_heap_header *) code[MI_LITERALS_OFF_HEAP];
+    struct erl_off_heap_header* oh = code_hdr->literals_off_heap;
 
     while (oh) {
 	Binary* bptr;
@@ -1091,7 +1088,7 @@ delete_code(Module* modp)
     ASSERT(modp->curr.num_breakpoints == 0);
     ASSERT(modp->curr.num_traced_exports == 0);
     modp->old = modp->curr;
-    modp->curr.code = NULL;
+    modp->curr.code_hdr = NULL;
     modp->curr.code_length = 0;
     modp->curr.catches = BEAM_CATCHES_NIL;
     modp->curr.nif = NULL;
@@ -1108,9 +1105,9 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
      * if not, delete old code; error if old code already exists.
      */
 
-    if (modp->curr.code != NULL && modp->old.code != NULL)  {
+    if (modp->curr.code_hdr && modp->old.code_hdr)  {
 	return am_not_purged;
-    } else if (modp->old.code == NULL) { /* Make the current version old. */
+    } else if (!modp->old.code_hdr) { /* Make the current version old. */
 	delete_code(modp);
     }
     return NIL;
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 016d0aaa32..2a8663d7ee 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -152,8 +152,8 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified)
     num_modules = 0;
     for (current = 0; current < max_modules; current++) {
 	modp = module_code(current, code_ix);
-	if (modp->curr.code) {
-	    max_funcs += modp->curr.code[MI_NUM_FUNCTIONS];
+	if (modp->curr.code_hdr) {
+	    max_funcs += modp->curr.code_hdr->num_functions;
 	    module[num_modules++] = modp;
 	}
     }
@@ -161,9 +161,9 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified)
     f->matching = (BpFunction *) Alloc(max_funcs*sizeof(BpFunction));
     i = 0;
     for (current = 0; current < num_modules; current++) {
-	BeamInstr** code_base = (BeamInstr **) module[current]->curr.code;
+	BeamCodeHeader* code_hdr = module[current]->curr.code_hdr;
 	BeamInstr* code;
-	Uint num_functions = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS];
+	Uint num_functions = (Uint)(UWord) code_hdr->num_functions;
 	Uint fi;
 
 	if (specified > 0) {
@@ -177,7 +177,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified)
 	    BeamInstr* pc;
 	    int wi;
 
-	    code = code_base[MI_FUNCTIONS+fi];
+	    code = code_hdr->functions[fi];
 	    ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
 	    pc = code+5;
 	    if (erts_is_native_break(pc)) {
@@ -547,21 +547,21 @@ erts_clear_all_breaks(BpFunctions* f)
 
 int 
 erts_clear_module_break(Module *modp) {
-    BeamInstr** code_base;
+    BeamCodeHeader* code_hdr;
     Uint n;
     Uint i;
 
     ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     ASSERT(modp);
-    code_base = (BeamInstr **) modp->curr.code;
-    if (code_base == NULL) {
+    code_hdr = modp->curr.code_hdr;
+    if (!code_hdr) {
 	return 0;
     }
-    n = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS];
+    n = (Uint)(UWord) code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
 	BeamInstr* pc;
 
-	pc = code_base[MI_FUNCTIONS+i] + 5;
+	pc = code_hdr->functions[i] + 5;
 	if (erts_is_native_break(pc)) {
 	    continue;
 	}
@@ -573,7 +573,7 @@ erts_clear_module_break(Module *modp) {
     for (i = 0; i < n; ++i) {
 	BeamInstr* pc;
 
-	pc = code_base[MI_FUNCTIONS+i] + 5;
+	pc = code_hdr->functions[i] + 5;
 	if (erts_is_native_break(pc)) {
 	    continue;
 	}
@@ -1204,17 +1204,17 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) {
 BeamInstr *
 erts_find_local_func(Eterm mfa[3]) {
     Module *modp;
-    BeamInstr** code_base;
+    BeamCodeHeader* code_hdr;
     BeamInstr* code_ptr;
     Uint i,n;
 
     if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL)
 	return NULL;
-    if ((code_base = (BeamInstr **) modp->curr.code) == NULL)
+    if ((code_hdr = modp->curr.code_hdr) == NULL)
 	return NULL;
-    n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
+    n = (BeamInstr) code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
-	code_ptr = code_base[MI_FUNCTIONS+i];
+	code_ptr = code_hdr->functions[i];
 	ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]);
 	ASSERT(mfa[0] == ((Eterm) code_ptr[2]) ||
 	       is_nil((Eterm) code_ptr[2]));
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 3a1d328bbd..e961989fca 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -208,7 +208,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     Eterm bin;
     Eterm mfa;
     BeamInstr* funcinfo = NULL;	/* Initialized to eliminate warning. */
-    BeamInstr* code_base;
+    BeamCodeHeader* code_hdr;
     BeamInstr* code_ptr = NULL;	/* Initialized to eliminate warning. */
     BeamInstr instr;
     BeamInstr uaddr;
@@ -258,12 +258,12 @@ erts_debug_disassemble_1(BIF_ALIST_1)
 	     */
 	    code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5;
 	    funcinfo = code_ptr+2;
-	} else if (modp == NULL || (code_base = modp->curr.code) == NULL) {
+	} else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) {
 	    BIF_RET(am_undef);
 	} else {
-	    n = code_base[MI_NUM_FUNCTIONS];
+	    n = code_hdr->num_functions;
 	    for (i = 0; i < n; i++) {
-		code_ptr = (BeamInstr *) code_base[MI_FUNCTIONS+i];
+		code_ptr = code_hdr->functions[i];
 		if (code_ptr[3] == name && code_ptr[4] == arity) {
 		    funcinfo = code_ptr+2;
 		    break;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a42ce1c9f1..6e4c8ecee3 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6080,7 +6080,7 @@ call_fun(Process* p,		/* Current process. */
 		 */
 		module = fe->module;
 		if ((modp = erts_get_module(module, code_ix)) != NULL
-		    && modp->curr.code != NULL) {
+		    && modp->curr.code_hdr != NULL) {
 		    /*
 		     * There is a module loaded, but obviously the fun is not
 		     * defined in it. We must not call the error_handler
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index a846e96204..c56e6732a0 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -80,7 +80,7 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 
 typedef struct {
     Uint value;			/* Value of label (NULL if not known yet). */
-    Uint patches;		/* Index (into code buffer) to first location
+    Sint patches;		/* Index (into code buffer) to first location
 				 * which must be patched with the value of this label.
 				 */
 #ifdef ERTS_SMP
@@ -284,9 +284,10 @@ typedef struct LoaderState {
     int specific_op;		/* Specific opcode (-1 if not found). */
     int num_functions;		/* Number of functions in module. */
     int num_labels;		/* Number of labels. */
-    int code_buffer_size;	/* Size of code buffer in words.  */
-    BeamInstr* code;		/* Loaded code. */
-    int ci;			/* Current index into loaded code. */
+    BeamCodeHeader* hdr;	/* Loaded code header */
+    BeamInstr* codev;	        /* Loaded code buffer */
+    int        codev_size;      /* Size of code buffer in words. */
+    int ci;			/* Current index into loaded code buffer. */
     Label* labels;
     StringPatch* string_patches; /* Linked list of position into string table to patch. */
     BeamInstr catches;		/* Linked list of catch_yf instructions. */
@@ -480,7 +481,7 @@ static void free_literal_fragment(ErlHeapFragment*);
 static void loader_state_dtor(Binary* magic);
 static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 			     Eterm group_leader, Eterm module,
-			     BeamInstr* code, Uint size);
+			     BeamCodeHeader* code, Uint size);
 static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
@@ -526,15 +527,15 @@ static void new_string_patch(LoaderState* stp, int pos);
 static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size);
 static int genopargcompare(GenOpArg* a, GenOpArg* b);
 static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix,
-                             BeamInstr* code, Eterm module, Eterm what);
+                             BeamCodeHeader*, Eterm module, Eterm what);
 static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix,
                                   Eterm mod);
-static Eterm functions_in_module(Process* p, BeamInstr* code);
-static Eterm attributes_for_module(Process* p, BeamInstr* code);
-static Eterm compilation_info_for_module(Process* p, BeamInstr* code);
-static Eterm md5_of_module(Process* p, BeamInstr* code);
-static Eterm has_native(BeamInstr* code);
-static Eterm native_addresses(Process* p, BeamInstr* code);
+static Eterm functions_in_module(Process* p, BeamCodeHeader*);
+static Eterm attributes_for_module(Process* p, BeamCodeHeader*);
+static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*);
+static Eterm md5_of_module(Process* p, BeamCodeHeader*);
+static Eterm has_native(BeamCodeHeader*);
+static Eterm native_addresses(Process* p, BeamCodeHeader*);
 int patch_funentries(Eterm Patchlist);
 int patch(Eterm Addresses, Uint fe);
 static int safe_mul(UWord a, UWord b, UWord* resp);
@@ -601,6 +602,7 @@ extern void check_allocated_block(Uint type, void *blk);
 #define CHKBLK(TYPE,BLK) /* nothing */
 #endif
 
+
 Eterm
 erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
 		     Eterm* modp, byte* code, Uint unloaded_size)
@@ -641,21 +643,27 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
     /*
      * Initialize code area.
      */
-    stp->code_buffer_size = 2048 + stp->num_functions;
-    stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE,
-				    sizeof(BeamInstr) * stp->code_buffer_size);
-
-    stp->code[MI_NUM_FUNCTIONS] = stp->num_functions;
-    stp->ci = MI_FUNCTIONS + stp->num_functions + 1;
-
-    stp->code[MI_ATTR_PTR] = 0;
-    stp->code[MI_ATTR_SIZE] = 0;
-    stp->code[MI_ATTR_SIZE_ON_HEAP] = 0;
-    stp->code[MI_COMPILE_PTR] = 0;
-    stp->code[MI_COMPILE_SIZE] = 0;
-    stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
-    stp->code[MI_LITERALS_START] = 0;
-    stp->code[MI_MD5_PTR] = 0;
+    stp->codev_size = 2048 + stp->num_functions;
+    stp->hdr = (BeamCodeHeader*) erts_alloc(ERTS_ALC_T_CODE,
+                                            (offsetof(BeamCodeHeader,functions)
+                                             + sizeof(BeamInstr) * stp->codev_size));
+
+    stp->hdr->num_functions = stp->num_functions;
+
+    /* Let the codev array start at functions[0] in order to index
+     * both function pointers and the loaded code itself that follows.
+     */
+    stp->codev = (BeamInstr*) &stp->hdr->functions;
+    stp->ci = stp->num_functions + 1;
+
+    stp->hdr->attr_ptr = NULL;
+    stp->hdr->attr_size = 0;
+    stp->hdr->attr_size_on_heap = 0;
+    stp->hdr->compile_ptr = NULL;
+    stp->hdr->compile_size = 0;
+    stp->hdr->compile_size_on_heap = 0;
+    stp->hdr->literals_start = NULL;
+    stp->hdr->md5_ptr = NULL;
 
     /*
      * Read the atom table.
@@ -777,7 +785,7 @@ erts_finish_loading(Binary* magic, Process* c_p,
 
     CHKBLK(ERTS_ALC_T_CODE,stp->code);
     retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module,
-			     stp->code, stp->loaded_size);
+			     stp->hdr, stp->loaded_size);
     if (retval != NIL) {
 	goto load_error;
     }
@@ -800,7 +808,8 @@ erts_finish_loading(Binary* magic, Process* c_p,
     debug_dump_code(stp->code,stp->ci);
 #endif
 #endif
-    stp->code = NULL;		/* Prevent code from being freed. */
+    stp->hdr = NULL;		/* Prevent code from being freed. */
+    stp->codev = NULL;
     *modp = stp->module;
 
     /*
@@ -832,7 +841,8 @@ erts_alloc_loader_state(void)
     stp->specific_op = -1;
     stp->genop = NULL;
     stp->atom = NULL;
-    stp->code = NULL;
+    stp->hdr = NULL;
+    stp->codev = NULL;
     stp->labels = NULL;
     stp->import = NULL;
     stp->export = NULL;
@@ -871,7 +881,7 @@ erts_module_for_prepared_code(Binary* magic)
 	return NIL;
     }
     stp = ERTS_MAGIC_BIN_DATA(magic);
-    if (stp->code != 0) {
+    if (stp->hdr != 0) {
 	return stp->module;
     } else {
 	return NIL;
@@ -921,12 +931,13 @@ loader_state_dtor(Binary* magic)
 	driver_free_binary(stp->bin);
 	stp->bin = 0;
     }
-    if (stp->code != 0) {
-        if (stp->code[MI_LITERALS_START]) {
-            erts_free(ERTS_ALC_T_LITERAL, (void*) stp->code[MI_LITERALS_START]);
+    if (stp->hdr != 0) {
+        if (stp->hdr->literals_start) {
+            erts_free(ERTS_ALC_T_LITERAL, stp->hdr->literals_start);
         }
-	erts_free(ERTS_ALC_T_CODE, stp->code);
-	stp->code = 0;
+	erts_free(ERTS_ALC_T_CODE, stp->hdr);
+	stp->hdr = 0;
+        stp->codev = 0;
     }
     if (stp->labels != 0) {
 	erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels);
@@ -999,7 +1010,7 @@ loader_state_dtor(Binary* magic)
 
 static Eterm
 insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
-		Eterm group_leader, Eterm module, BeamInstr* code,
+		Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr,
 		Uint size)
 {
     Module* modp;
@@ -1020,7 +1031,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 
     erts_total_code_size += size;
     modp = erts_put_module(module);
-    modp->curr.code = code;
+    modp->curr.code_hdr = code_hdr;
     modp->curr.code_length = size;
     modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
 
@@ -1028,7 +1039,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
      * Update ranges (used for finding a function from a PC value).
      */
 
-    erts_update_ranges(code, size);
+    erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size);
     return NIL;
 }
 
@@ -1377,7 +1388,7 @@ read_export_table(LoaderState* stp)
 	if (value == 0) {
 	    LoadError2(stp, "export table entry %d: label %d not resolved", i, n);
 	}
-	stp->export[i].address = address = stp->code + value;
+	stp->export[i].address = address = stp->codev + value;
 
 	/*
 	 * Find out if there is a BIF with the same name.
@@ -1396,7 +1407,7 @@ read_export_table(LoaderState* stp)
 	 * any other functions that walk through all local functions.
 	 */
 
-	if (stp->labels[n].patches) {
+	if (stp->labels[n].patches >= 0) {
 	    LoadError3(stp, "there are local calls to the stub for "
 		       "the BIF %T:%T/%d",
 		       stp->module, func, arity);
@@ -1739,7 +1750,7 @@ read_code_header(LoaderState* stp)
 				       stp->num_labels * sizeof(Label));
     for (i = 0; i < stp->num_labels; i++) {
 	stp->labels[i].value = 0;
-	stp->labels[i].patches = 0;
+	stp->labels[i].patches = -1;
 #ifdef ERTS_SMP
 	stp->labels[i].looprec_targeted = 0;
 #endif
@@ -1758,13 +1769,14 @@ read_code_header(LoaderState* stp)
     } else {}
 
 #define CodeNeed(w) do {						\
-    ASSERT(ci <= code_buffer_size);					\
-    if (code_buffer_size < ci+(w)) {					\
-        code_buffer_size = 2*ci+(w);					\
-	stp->code = code =						\
-	    (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE,			\
-				     (void *) code,			\
-				     code_buffer_size * sizeof(BeamInstr));	\
+    ASSERT(ci <= codev_size);					        \
+    if (codev_size < ci+(w)) {					        \
+        codev_size = 2*ci+(w);	                                        \
+	stp->hdr = (BeamCodeHeader*) erts_realloc(ERTS_ALC_T_CODE,	\
+            (void *) stp->hdr,	                                        \
+            (offsetof(BeamCodeHeader,functions)                         \
+             + codev_size * sizeof(BeamInstr)));                        \
+        code = stp->codev = (BeamInstr*) &stp->hdr->functions;          \
     } 									\
 } while (0)
     
@@ -1780,7 +1792,7 @@ load_code(LoaderState* stp)
     int arg;			/* Number of current argument. */
     int num_specific;		/* Number of specific ops for current. */
     BeamInstr* code;
-    int code_buffer_size;
+    int codev_size;
     int specific;
     Uint last_label = 0;	/* Number of last label. */
     Uint function_number = 0;
@@ -1797,15 +1809,15 @@ load_code(LoaderState* stp)
 	FUNC_INFO_SZ = 5
     };
 
-    code = stp->code;
-    code_buffer_size = stp->code_buffer_size;
+    code = stp->codev;
+    codev_size = stp->codev_size;
     ci = stp->ci;
 
     for (;;) {
 	int new_op;
 	GenOp* tmp_op;
 
-	ASSERT(ci <= code_buffer_size);
+	ASSERT(ci <= codev_size);
 
     get_next_instr:
 	GetByte(stp, new_op);
@@ -2420,8 +2432,7 @@ load_code(LoaderState* stp)
 	switch (stp->specific_op) {
 	case op_i_func_info_IaaI:
 	    {
-		Uint offset;
-
+		Sint offset;
 		if (function_number >= stp->num_functions) {
 		    LoadError1(stp, "too many functions in module (header said %d)",
 			       stp->num_functions); 
@@ -2463,15 +2474,15 @@ load_code(LoaderState* stp)
 		stp->arity = code[ci-1];
 
 		ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ);
-		offset = MI_FUNCTIONS + function_number;
-		code[offset] = stp->labels[last_label].patches;
+		stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches;
+		offset = function_number;
 		stp->labels[last_label].patches = offset;
 		function_number++;
 		if (stp->arity > MAX_ARG) {
 		    LoadError1(stp, "too many arguments: %d", stp->arity);
 		}
 #ifdef DEBUG
-		ASSERT(stp->labels[0].patches == 0); /* Should not be referenced. */
+		ASSERT(stp->labels[0].patches < 0); /* Should not be referenced. */
 		for (i = 1; i < stp->num_labels; i++) {
 		    ASSERT(stp->labels[i].patches < ci);
 		}
@@ -2541,7 +2552,7 @@ load_code(LoaderState* stp)
 	     * End of code found.
 	     */
 	case op_int_code_end:
-	    stp->code_buffer_size = code_buffer_size;
+	    stp->codev_size = codev_size;
 	    stp->ci = ci;
 	    stp->function = THE_NON_VALUE;
 	    stp->genop = NULL;
@@ -4351,7 +4362,8 @@ gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
 static int
 freeze_code(LoaderState* stp)
 {
-    BeamInstr* code = stp->code;
+    BeamCodeHeader* code_hdr = stp->hdr;
+    BeamInstr* codev = (BeamInstr*) &stp->hdr->functions;
     int i;
     byte* str_table;
     unsigned strtab_size = stp->chunks[STR_CHUNK].size;
@@ -4380,53 +4392,50 @@ freeze_code(LoaderState* stp)
 		     (stp->current_li+1) + stp->num_fnames) *
 	    sizeof(Eterm) + (stp->current_li+1) * stp->loc_size;
     }
-    size = (stp->ci * sizeof(BeamInstr)) +
+    size = offsetof(BeamCodeHeader,functions) + (stp->ci * sizeof(BeamInstr)) +
 	strtab_size + attr_size + compile_size + MD5_SIZE + line_size;
 
     /*
      * Move the code to its final location.
      */
 
-    code = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size);
-    CHKBLK(ERTS_ALC_T_CODE,code);
+    code_hdr = (BeamCodeHeader*) erts_realloc(ERTS_ALC_T_CODE, (void *) code_hdr, size);
+    codev = (BeamInstr*) &code_hdr->functions;
+    CHKBLK(ERTS_ALC_T_CODE,code_hdr);
     /*
      * Place a pointer to the op_int_code_end instruction in the
      * function table in the beginning of the file.
      */
 
-    code[MI_FUNCTIONS+stp->num_functions] = (BeamInstr) (code + stp->ci - 1);
-    CHKBLK(ERTS_ALC_T_CODE,code);
+    code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1);
+    CHKBLK(ERTS_ALC_T_CODE,code_hdr);
 
     /*
      * Store the pointer to the on_load function.
      */
 
     if (stp->on_load) {
-	code[MI_ON_LOAD_FUNCTION_PTR] = (BeamInstr) (code + stp->on_load);
+	code_hdr->on_load_function_ptr = codev + stp->on_load;
     } else {
-	code[MI_ON_LOAD_FUNCTION_PTR] = 0;
+	code_hdr->on_load_function_ptr = NULL;
     }
-    CHKBLK(ERTS_ALC_T_CODE,code);
+    CHKBLK(ERTS_ALC_T_CODE,code_hdr);
 
     /*
      * Place the literals in their own allocated heap (for fast range check)
      * and fix up all instructions that refer to it.
      */
     {
-	Uint* ptr;
-	Uint* low;
-	Uint* high;
+	Eterm* ptr;
 	LiteralPatch* lp;
         ErlOffHeap code_off_heap;
 
         ERTS_INIT_OFF_HEAP(&code_off_heap);
 
-        low = erts_alloc(ERTS_ALC_T_LITERAL,
-                         stp->total_literal_size*sizeof(Eterm));
-	high = low + stp->total_literal_size;
-	code[MI_LITERALS_START] = (BeamInstr) low;
-	code[MI_LITERALS_END] = (BeamInstr) high;
-	ptr = low;
+        ptr = (Eterm*)erts_alloc(ERTS_ALC_T_LITERAL,
+                                 stp->total_literal_size*sizeof(Eterm));
+	code_hdr->literals_start = ptr;
+	code_hdr->literals_end = ptr + stp->total_literal_size;
 	for (i = 0; i < stp->num_literals; i++) {
             if (stp->literals[i].heap_frags) {
                 move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags,
@@ -4435,13 +4444,13 @@ freeze_code(LoaderState* stp)
             }
             else ASSERT(is_immed(stp->literals[i].term));
 	}
-        code[MI_LITERALS_OFF_HEAP] = (BeamInstr) code_off_heap.first;
+	code_hdr->literals_off_heap = code_off_heap.first;
 	lp = stp->literal_patches;
 	while (lp != 0) {
 	    BeamInstr* op_ptr;
 	    Literal* lit;
 
-	    op_ptr = code + lp->pos;
+	    op_ptr = codev + lp->pos;
 	    lit = &stp->literals[op_ptr[0]];
 	    op_ptr[0] = lit->term;
 	    lp = lp->next;
@@ -4453,16 +4462,16 @@ freeze_code(LoaderState* stp)
      * If there is line information, place it here.
      */
     if (stp->line_instr == 0) {
-	code[MI_LINE_TABLE] = (BeamInstr) 0;
-	str_table = (byte *) (code+stp->ci);
+	code_hdr->line_table = NULL;
+	str_table = (byte *) (codev + stp->ci);
     } else {
-	Eterm* line_tab = (Eterm *) (code+stp->ci);
+	Eterm* line_tab = (Eterm *) (codev+stp->ci);
 	Eterm* p;
 	int ftab_size = stp->num_functions;
 	int num_instrs = stp->current_li;
 	Eterm* first_line_item;
 
-	code[MI_LINE_TABLE] = (BeamInstr) line_tab;
+	code_hdr->line_table = line_tab;
 	p = line_tab + MI_LINE_FUNC_TAB;
 
 	first_line_item = (p + ftab_size + 1);
@@ -4472,9 +4481,9 @@ freeze_code(LoaderState* stp)
 	*p++ = (Eterm) (BeamInstr) (first_line_item + num_instrs);
 	ASSERT(p == first_line_item);
 	for (i = 0; i < num_instrs; i++) {
-		*p++ = (Eterm) (BeamInstr) (code + stp->line_instr[i].pos);
+		*p++ = (Eterm) (BeamInstr) (codev + stp->line_instr[i].pos);
 	}
-	*p++ = (Eterm) (BeamInstr) (code + stp->ci - 1);
+	*p++ = (Eterm) (BeamInstr) (codev + stp->ci - 1);
 
 	line_tab[MI_LINE_FNAME_PTR] = (Eterm) (BeamInstr) p;
 	memcpy(p, stp->fname, stp->num_fnames*sizeof(Eterm));
@@ -4509,13 +4518,13 @@ freeze_code(LoaderState* stp)
     if (attr_size) {
 	byte* attr = str_table + strtab_size;
 	sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size);
-	code[MI_ATTR_PTR] = (BeamInstr) attr;
-	code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size;
+	code_hdr->attr_ptr = attr;
+	code_hdr->attr_size = (BeamInstr) stp->chunks[ATTR_CHUNK].size;
 	decoded_size = erts_decode_ext_size(attr, attr_size);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of module attributes");
  	}
-	code[MI_ATTR_SIZE_ON_HEAP] = decoded_size;
+	code_hdr->attr_size_on_heap = decoded_size;
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
     if (compile_size) {
@@ -4525,9 +4534,9 @@ freeze_code(LoaderState* stp)
 	       stp->chunks[COMPILE_CHUNK].size);
 
 	CHKBLK(ERTS_ALC_T_CODE,code);
-	code[MI_COMPILE_PTR] = (BeamInstr) compile_info;
+	code_hdr->compile_ptr = compile_info;
 	CHKBLK(ERTS_ALC_T_CODE,code);
-	code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size;
+	code_hdr->compile_size = (BeamInstr) stp->chunks[COMPILE_CHUNK].size;
 	CHKBLK(ERTS_ALC_T_CODE,code);
 	decoded_size = erts_decode_ext_size(compile_info, compile_size);
 	CHKBLK(ERTS_ALC_T_CODE,code);
@@ -4535,7 +4544,7 @@ freeze_code(LoaderState* stp)
  	    LoadError0(stp, "bad external term representation of compilation information");
  	}
 	CHKBLK(ERTS_ALC_T_CODE,code);
-	code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size;
+	code_hdr->compile_size_on_heap = decoded_size;
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
     {
@@ -4543,7 +4552,7 @@ freeze_code(LoaderState* stp)
 	CHKBLK(ERTS_ALC_T_CODE,code);
 	sys_memcpy(md5_sum, stp->mod_md5, MD5_SIZE);
 	CHKBLK(ERTS_ALC_T_CODE,code);
-	code[MI_MD5_PTR] = (BeamInstr) md5_sum;
+	code_hdr->md5_ptr = md5_sum;
 	CHKBLK(ERTS_ALC_T_CODE,code);
     }
     CHKBLK(ERTS_ALC_T_CODE,code);
@@ -4552,7 +4561,7 @@ freeze_code(LoaderState* stp)
      * Make sure that we have not overflowed the allocated code space.
      */
     ASSERT(str_table + strtab_size + attr_size + compile_size + MD5_SIZE ==
-	   ((byte *) code) + size);
+	   ((byte *) code_hdr) + size);
 
     /*
      * Patch all instructions that refer to the string table.
@@ -4564,46 +4573,47 @@ freeze_code(LoaderState* stp)
 	    BeamInstr* op_ptr;
 	    byte* strp;
 
-	    op_ptr = code + sp->pos;
+	    op_ptr = codev + sp->pos;
 	    strp = str_table + op_ptr[0];
 	    op_ptr[0] = (BeamInstr) strp;
 	    sp = sp->next;
 	}
     }
-    CHKBLK(ERTS_ALC_T_CODE,code);
+    CHKBLK(ERTS_ALC_T_CODE,code_hdr);
 
     /*
      * Resolve all labels.
      */
 
     for (i = 0; i < stp->num_labels; i++) {
-	Uint this_patch;
-	Uint next_patch;
+	Sint this_patch;
+	Sint next_patch;
 	Uint value = stp->labels[i].value;
 	
-	if (value == 0 && stp->labels[i].patches != 0) {
+	if (value == 0 && stp->labels[i].patches >= 0) {
 	    LoadError1(stp, "label %d not resolved", i);
 	}
 	ASSERT(value < stp->ci);
 	this_patch = stp->labels[i].patches;
-	while (this_patch != 0) {
+	while (this_patch >= 0) {
 	    ASSERT(this_patch < stp->ci);
-	    next_patch = code[this_patch];
+	    next_patch = codev[this_patch];
 	    ASSERT(next_patch < stp->ci);
-	    code[this_patch] = (BeamInstr) (code + value);
+	    codev[this_patch] = (BeamInstr) (codev + value);
 	    this_patch = next_patch;
 	}
     }
-    CHKBLK(ERTS_ALC_T_CODE,code);
+    CHKBLK(ERTS_ALC_T_CODE,code_hdr);
 
     /*
      * Save the updated code pointer and code size.
      */
 
-    stp->code = code;
+    stp->hdr = code_hdr;
+    stp->codev = codev;
     stp->loaded_size = size;
 
-    CHKBLK(ERTS_ALC_T_CODE,code);
+    CHKBLK(ERTS_ALC_T_CODE,code_hdr);
     return 1;
 
  load_error:
@@ -4611,7 +4621,8 @@ freeze_code(LoaderState* stp)
      * Make sure that the caller frees the newly reallocated block, and
      * not the old one (in case it has moved).
      */
-    stp->code = code;
+    stp->hdr = code_hdr;
+    stp->codev = codev;
     return 0;
 }
 
@@ -4622,7 +4633,7 @@ final_touch(LoaderState* stp)
     int on_load = stp->on_load;
     unsigned catches;
     Uint index;
-    BeamInstr* code = stp->code;
+    BeamInstr* codev = stp->codev;
     Module* modp;
 
     /*
@@ -4632,10 +4643,10 @@ final_touch(LoaderState* stp)
     index = stp->catches;
     catches = BEAM_CATCHES_NIL;
     while (index != 0) {
-	BeamInstr next = code[index];
-	code[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((BeamInstr *)code[index+2], catches);
-	code[index+2] = make_catch(catches);
+	BeamInstr next = codev[index];
+	codev[index] = BeamOpCode(op_catch_yf);
+	catches = beam_catches_cons((BeamInstr *)codev[index+2], catches);
+	codev[index+2] = make_catch(catches);
 	index = next;
     }
     modp = erts_put_module(stp->module);
@@ -4686,8 +4697,8 @@ final_touch(LoaderState* stp)
 	current = stp->import[i].patches;
 	while (current != 0) {
 	    ASSERT(current < stp->ci);
-	    next = stp->code[current];
-	    stp->code[current] = import;
+	    next = stp->codev[current];
+	    stp->codev[current] = import;
 	    current = next;
 	}
     }
@@ -4700,7 +4711,7 @@ final_touch(LoaderState* stp)
 	for (i = 0; i < stp->num_lambdas; i++) {
 	    unsigned entry_label = stp->lambdas[i].label;
 	    ErlFunEntry* fe = stp->lambdas[i].fe;
-	    BeamInstr* code_ptr = (BeamInstr *) (stp->code + stp->labels[entry_label].value);
+	    BeamInstr* code_ptr = stp->codev + stp->labels[entry_label].value;
 
 	    if (fe->address[0] != 0) {
 		/*
@@ -5324,7 +5335,7 @@ new_label(LoaderState* stp)
 					 (void *) stp->labels,
 					 stp->num_labels * sizeof(Label));
     stp->labels[num].value = 0;
-    stp->labels[num].patches = 0;
+    stp->labels[num].patches = -1;
     return num;
 }
 
@@ -5384,7 +5395,7 @@ erts_module_info_0(Process* p, Eterm module)
 {
     Module* modp;
     ErtsCodeIndex code_ix = erts_active_code_ix();
-    BeamInstr* code;
+    BeamCodeHeader* code_hdr;
     Eterm *hp;
     Eterm list = NIL;
     Eterm tup;
@@ -5398,13 +5409,13 @@ erts_module_info_0(Process* p, Eterm module)
 	return THE_NON_VALUE;
     }
 
-    code = modp->curr.code;
-    if (code == NULL) {
+    code_hdr = modp->curr.code_hdr;
+    if (code_hdr == NULL) {
         return THE_NON_VALUE;
     }
 
 #define BUILD_INFO(What) \
-    tup = get_module_info(p, code_ix, code, module, What); \
+    tup = get_module_info(p, code_ix, code_hdr, module, What); \
     hp = HAlloc(p, 5); \
     tup = TUPLE2(hp, What, tup); \
     hp += 3; \
@@ -5427,7 +5438,7 @@ erts_module_info_1(Process* p, Eterm module, Eterm what)
 {
     Module* modp;
     ErtsCodeIndex code_ix = erts_active_code_ix();
-    BeamInstr* code;
+    BeamCodeHeader* code_hdr;
 
     if (is_not_atom(module)) {
         return THE_NON_VALUE;
@@ -5438,34 +5449,34 @@ erts_module_info_1(Process* p, Eterm module, Eterm what)
         return THE_NON_VALUE;
     }
 
-    code = modp->curr.code;
-    if (code == NULL) {
+    code_hdr = modp->curr.code_hdr;
+    if (code_hdr == NULL) {
         return THE_NON_VALUE;
     }
 
-    return get_module_info(p, code_ix, code, module, what);
+    return get_module_info(p, code_ix, code_hdr, module, what);
 }
 
 static Eterm
-get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code,
+get_module_info(Process* p, ErtsCodeIndex code_ix, BeamCodeHeader* code_hdr,
                 Eterm module, Eterm what)
 {
     if (what == am_module) {
 	return module;
     } else if (what == am_md5) {
-	return md5_of_module(p, code);
+	return md5_of_module(p, code_hdr);
     } else if (what == am_exports) {
 	return exported_from_module(p, code_ix, module);
     } else if (what == am_functions) {
-	return functions_in_module(p, code);
+	return functions_in_module(p, code_hdr);
     } else if (what == am_attributes) {
-	return attributes_for_module(p, code);
+	return attributes_for_module(p, code_hdr);
     } else if (what == am_compile) {
-	return compilation_info_for_module(p, code);
+	return compilation_info_for_module(p, code_hdr);
     } else if (what == am_native_addresses) {
-	return native_addresses(p, code);
+	return native_addresses(p, code_hdr);
     } else if (what == am_native) {
-	return has_native(code);
+	return has_native(code_hdr);
     }
     return THE_NON_VALUE;
 }
@@ -5477,7 +5488,7 @@ get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code,
 
 Eterm
 functions_in_module(Process* p, /* Process whose heap to use. */
-		    BeamInstr* code)
+		    BeamCodeHeader* code_hdr)
 {
     int i;
     Uint num_functions;
@@ -5486,12 +5497,12 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     Eterm* hp_end;
     Eterm result = NIL;
 
-    num_functions = code[MI_NUM_FUNCTIONS];
+    num_functions = code_hdr->num_functions;
     need = 5*num_functions;
     hp = HAlloc(p, need);
     hp_end = hp + need;
     for (i = num_functions-1; i >= 0 ; i--) {
-	BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+	BeamInstr* func_info = code_hdr->functions[i];
 	Eterm name = (Eterm) func_info[3];
 	int arity = (int) func_info[4];
 	Eterm tuple;
@@ -5517,11 +5528,11 @@ functions_in_module(Process* p, /* Process whose heap to use. */
  */
 
 static Eterm
-has_native(BeamInstr *code)
+has_native(BeamCodeHeader *code_hdr)
 {
     Eterm result = am_false;
 #ifdef HIPE
-    if (erts_is_module_native(code)) {
+    if (erts_is_module_native(code_hdr)) {
         result = am_true;
     }
 #endif
@@ -5529,15 +5540,15 @@ has_native(BeamInstr *code)
 }
 
 int
-erts_is_module_native(BeamInstr* code)
+erts_is_module_native(BeamCodeHeader* code_hdr)
 {
     Uint i, num_functions;
 
     /* Check NativeAdress of first real function in module */
-    if (code != NULL) {
-        num_functions = code[MI_NUM_FUNCTIONS];
+    if (code_hdr != NULL) {
+        num_functions = code_hdr->num_functions;
         for (i=0; i<num_functions; i++) {
-            BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+            BeamInstr* func_info = (BeamInstr *) code_hdr->functions[i];
             Eterm name = (Eterm) func_info[3];
             if (is_atom(name)) {
                 return func_info[1] != 0;
@@ -5554,7 +5565,7 @@ erts_is_module_native(BeamInstr* code)
  */
 
 static Eterm
-native_addresses(Process* p, BeamInstr* code)
+native_addresses(Process* p, BeamCodeHeader* code_hdr)
 {
     int i;
     Eterm* hp;
@@ -5563,12 +5574,12 @@ native_addresses(Process* p, BeamInstr* code)
     Eterm* hp_end;
     Eterm result = NIL;
 
-    num_functions = code[MI_NUM_FUNCTIONS];
+    num_functions = code_hdr->num_functions;
     need = (6+BIG_UINT_HEAP_SIZE)*num_functions;
     hp = HAlloc(p, need);
     hp_end = hp + need;
     for (i = num_functions-1; i >= 0 ; i--) {
-	BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+	BeamInstr* func_info = code_hdr->functions[i];
 	Eterm name = (Eterm) func_info[3];
 	int arity = (int) func_info[4];
 	Eterm tuple;
@@ -5634,15 +5645,15 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 
 Eterm
 attributes_for_module(Process* p, /* Process whose heap to use. */
-                      BeamInstr* code)
+                      BeamCodeHeader* code_hdr)
 {
     byte* ext;
     Eterm result = NIL;
 
-    ext = (byte *) code[MI_ATTR_PTR];
+    ext = code_hdr->attr_ptr;
     if (ext != NULL) {
 	ErtsHeapFactory factory;
-	erts_factory_proc_prealloc_init(&factory, p, code[MI_ATTR_SIZE_ON_HEAP]);
+	erts_factory_proc_prealloc_init(&factory, p, code_hdr->attr_size_on_heap);
 	result = erts_decode_ext(&factory, &ext);
 	if (is_value(result)) {
 	    erts_factory_close(&factory);
@@ -5657,15 +5668,15 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
 
 Eterm
 compilation_info_for_module(Process* p, /* Process whose heap to use. */
-                            BeamInstr* code)
+                            BeamCodeHeader* code_hdr)
 {
     byte* ext;
     Eterm result = NIL;
 
-    ext = (byte *) code[MI_COMPILE_PTR];
+    ext = code_hdr->compile_ptr;
     if (ext != NULL) {
 	ErtsHeapFactory factory;
-	erts_factory_proc_prealloc_init(&factory, p, code[MI_COMPILE_SIZE_ON_HEAP]);
+	erts_factory_proc_prealloc_init(&factory, p, code_hdr->compile_size_on_heap);
 	result = erts_decode_ext(&factory, &ext);
 	if (is_value(result)) {
 	    erts_factory_close(&factory);
@@ -5680,9 +5691,9 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 
 Eterm
 md5_of_module(Process* p, /* Process whose heap to use. */
-              BeamInstr* code)
+              BeamCodeHeader* code_hdr)
 {
-    return new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE);
+    return new_binary(p, code_hdr->md5_ptr, MD5_SIZE);
 }
 
 /*
@@ -5893,7 +5904,7 @@ static byte*
 stub_copy_info(LoaderState* stp,
 	       int chunk,	/* Chunk: ATTR_CHUNK or COMPILE_CHUNK */
 	       byte* info,	/* Where to store info. */
-	       BeamInstr* ptr_word,	/* Where to store pointer into info. */
+	       byte** ptr_word,	/* Where to store pointer into info. */
 	       BeamInstr* size_word,	/* Where to store size into info. */
 	       BeamInstr* size_on_heap_word) /* Where to store size on heap. */
 {
@@ -5901,7 +5912,7 @@ stub_copy_info(LoaderState* stp,
     Uint size = stp->chunks[chunk].size;
     if (size != 0) {
 	memcpy(info, stp->chunks[chunk].start, size);
-	*ptr_word = (BeamInstr) info;
+	*ptr_word = info;
 	decoded_size = erts_decode_ext_size(info, size);
 	if (decoded_size < 0) {
  	    return 0;
@@ -6156,11 +6167,10 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     BeamInstr Patchlist;
     Eterm MD5Bin;
     Eterm* tp;
-    BeamInstr* code = NULL;
-    BeamInstr* ptrs;
+    BeamCodeHeader* code_hdr;
+    BeamInstr* code_base;
     BeamInstr* fp;
     byte* info;
-    Uint ci;
     int n;
     int code_size;
     int rval;
@@ -6238,39 +6248,39 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Allocate memory for the stub module.
      */
 
-    code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr);
-    code_size += stp->chunks[ATTR_CHUNK].size;
-    code_size += stp->chunks[COMPILE_CHUNK].size;
-    code_size += MD5_SIZE;
-    code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size);
-    if (!code) {
+    code_size = (offsetof(BeamCodeHeader,functions)
+                 + ((n+1) * sizeof(BeamInstr*))
+                 + (WORDS_PER_FUNCTION*n + 1) * sizeof(BeamInstr)
+                 + stp->chunks[ATTR_CHUNK].size
+                 + stp->chunks[COMPILE_CHUNK].size
+                 + MD5_SIZE);
+    code_hdr = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size);
+    if (!code_hdr) {
 	goto error;
     }
 
     /*
-     * Initialize code area.
+     * Initialize code header.
      */
 
-    code[MI_NUM_FUNCTIONS] = n;
-    code[MI_ATTR_PTR] = 0;
-    code[MI_ATTR_SIZE] = 0;
-    code[MI_ATTR_SIZE_ON_HEAP] = 0;
-    code[MI_COMPILE_PTR] = 0;
-    code[MI_COMPILE_SIZE] = 0;
-    code[MI_COMPILE_SIZE_ON_HEAP] = 0;
-    code[MI_LITERALS_START] = 0;
-    code[MI_LITERALS_END] = 0;
-    code[MI_LITERALS_OFF_HEAP] = 0;
-    code[MI_ON_LOAD_FUNCTION_PTR] = 0;
-    code[MI_MD5_PTR] = 0;
-    ci = MI_FUNCTIONS + n + 1;
+    code_hdr->num_functions = n;
+    code_hdr->attr_ptr = NULL;
+    code_hdr->attr_size = 0;
+    code_hdr->attr_size_on_heap = 0;
+    code_hdr->compile_ptr = NULL;
+    code_hdr->compile_size = 0;
+    code_hdr->compile_size_on_heap = 0;
+    code_hdr->literals_start = NULL;
+    code_hdr->literals_end = NULL;
+    code_hdr->literals_off_heap = 0;
+    code_hdr->on_load_function_ptr = NULL;
+    code_hdr->md5_ptr = NULL;
 
     /*
      * Make stubs for all functions.
      */
 
-    ptrs = code + MI_FUNCTIONS;
-    fp = code + ci;
+    fp = code_base = (BeamInstr*) &code_hdr->functions[n+1];
     for (i = 0; i < n; i++) {
 	Eterm* listp;
 	Eterm tuple;
@@ -6313,7 +6323,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	 * Set the pointer and make the stub. Put a return instruction
 	 * as the body until we know what kind of trap we should put there.
 	 */
-	ptrs[i] = (BeamInstr) fp;
+	code_hdr->functions[i] = fp;
 #ifdef HIPE
 	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
 #else
@@ -6326,7 +6336,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Insert the last pointer and the int_code_end instruction.
      */
 
-    ptrs[i] = (BeamInstr) fp;
+    code_hdr->functions[i] = fp;
     *fp++ = (BeamInstr) BeamOp(op_int_code_end);
 
     /*
@@ -6335,16 +6345,16 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 
     info = (byte *) fp;
     info = stub_copy_info(stp, ATTR_CHUNK, info,
-			  code+MI_ATTR_PTR,
-			  code+MI_ATTR_SIZE,
-			  code+MI_ATTR_SIZE_ON_HEAP);
+			  &code_hdr->attr_ptr,
+			  &code_hdr->attr_size,
+			  &code_hdr->attr_size_on_heap);
     if (info == NULL) {
 	goto error;
     }
     info = stub_copy_info(stp, COMPILE_CHUNK, info,
-			  code+MI_COMPILE_PTR,
-			  code+MI_COMPILE_SIZE,
-			  code+MI_COMPILE_SIZE_ON_HEAP);
+			  &code_hdr->compile_ptr,
+			  &code_hdr->compile_size,
+			  &code_hdr->compile_size_on_heap);
     if (info == NULL) {
 	goto error;
     }
@@ -6353,7 +6363,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
       byte *md5 = NULL;
       if ((md5 = erts_get_aligned_binary_bytes(MD5Bin, &tmp)) != NULL) {
         sys_memcpy(info, md5, MD5_SIZE);
-        code[MI_MD5_PTR] = (BeamInstr) info;
+        code_hdr->md5_ptr = info;
       }
       erts_free_aligned_binary_bytes(tmp);
     }
@@ -6362,7 +6372,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Insert the module in the module table.
      */
 
-    rval = insert_new_code(p, 0, p->group_leader, Mod, code, code_size);
+    rval = insert_new_code(p, 0, p->group_leader, Mod, code_hdr, code_size);
     if (rval != NIL) {
 	goto error;
     }
@@ -6371,7 +6381,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Export all stub functions and insert the correct type of HiPE trap.
      */
 
-    fp = code + ci;
+    fp = code_base;
     for (i = 0; i < n; i++) {
 	stub_final_touch(stp, fp);
 	fp += WORDS_PER_FUNCTION;
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index eedb5ee4cd..c86ac65521 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -24,7 +24,6 @@
 #include "beam_opcodes.h"
 #include "erl_process.h"
 
-int erts_is_module_native(BeamInstr* code);
 Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks,
 			    Eterm module);
 
@@ -61,63 +60,66 @@ extern BeamInstr* em_call_nif;
 
 /* Total code size in bytes */
 extern Uint erts_total_code_size;
-/*
- * Index into start of code chunks which contains additional information
- * about the loaded module.
- *
- * First number of functions.
- */
-
-#define MI_NUM_FUNCTIONS     0
-
-/*
- * The attributes retrieved by Mod:module_info(attributes).
- */
-
-#define MI_ATTR_PTR          1
-#define MI_ATTR_SIZE	     2
-#define MI_ATTR_SIZE_ON_HEAP 3
-
-/*
- * The compilation information retrieved by Mod:module_info(compile).
- */
-
-#define MI_COMPILE_PTR          4
-#define MI_COMPILE_SIZE         5
-#define MI_COMPILE_SIZE_ON_HEAP 6
-
-/*
- * Literal area (constant pool).
- */
-#define MI_LITERALS_START	7
-#define MI_LITERALS_END		8
-#define MI_LITERALS_OFF_HEAP	9
-
-/*
- * Pointer to the on_load function (or NULL if none).
- */
-#define MI_ON_LOAD_FUNCTION_PTR 10
-
-/*
- * Pointer to the line table (or NULL if none).
- */
-#define MI_LINE_TABLE 11
-
-/*
- * Pointer to the module MD5 sum (16 bytes)
- */
-#define MI_MD5_PTR 12
 
 /*
- * Start of function pointer table.  This table contains pointers to
- * all functions in the module plus an additional pointer just beyond
- * the end of the last function.
- *
- * The actual loaded code (for the first function) start just beyond
- * this table.
+ * Header of code chunks which contains additional information
+ * about the loaded module.
  */
-
-#define MI_FUNCTIONS         13
+typedef struct beam_code_header {
+    /*
+     * Number of functions.
+     */
+    UWord num_functions;
+
+    /*
+     * The attributes retrieved by Mod:module_info(attributes).
+     */
+    byte* attr_ptr;
+    UWord attr_size;
+    UWord attr_size_on_heap;
+
+    /*
+     * The compilation information retrieved by Mod:module_info(compile).
+     */
+    byte* compile_ptr;
+    UWord compile_size;
+    UWord compile_size_on_heap;
+
+    /*
+     * Literal area (constant pool).
+     */
+    Eterm* literals_start;
+    Eterm* literals_end;
+    struct erl_off_heap_header* literals_off_heap;
+
+    /*
+     * Pointer to the on_load function (or NULL if none).
+     */
+    BeamInstr* on_load_function_ptr;
+
+    /*
+     * Pointer to the line table (or NULL if none).
+     */
+    Eterm* line_table;
+
+    /*
+     * Pointer to the module MD5 sum (16 bytes)
+     */
+    byte* md5_ptr;
+
+    /*
+     * Start of function pointer table.  This table contains pointers to
+     * all functions in the module plus an additional pointer just beyond
+     * the end of the last function.
+     *
+     * The actual loaded code (for the first function) start just beyond
+     * this table.
+     */
+    BeamInstr* functions[1];
+
+}BeamCodeHeader;
+
+int erts_is_module_native(BeamCodeHeader* code);
 
 /*
  * Layout of the line table.
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 19079ba150..e6cf43446e 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -38,7 +38,7 @@ typedef struct {
 
 static Range* find_range(BeamInstr* pc);
 static void lookup_loc(FunctionInfo* fi, BeamInstr* pc,
-		       BeamInstr* modp, int idx);
+		       BeamCodeHeader*, int idx);
 
 /*
  * The following variables keep a sorted list of address ranges for
@@ -241,6 +241,7 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
     BeamInstr** high;
     BeamInstr** mid;
     Range* rp;
+    BeamCodeHeader* hdr;
 
     fi->current = NULL;
     fi->needed = 5;
@@ -249,9 +250,10 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
     if (rp == 0) {
 	return;
     }
+    hdr = (BeamCodeHeader*) rp->start;
 
-    low = (BeamInstr **) (rp->start + MI_FUNCTIONS);
-    high = low + rp->start[MI_NUM_FUNCTIONS];
+    low = hdr->functions;
+    high = low + hdr->num_functions;
     while (low < high) {
 	mid = low + (high-low) / 2;
 	if (pc < mid[0]) {
@@ -259,10 +261,9 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
 	} else if (pc < mid[1]) {
 	    fi->current = mid[0]+2;
 	    if (full_info) {
-		BeamInstr** fp = (BeamInstr **) (rp->start +
-						 MI_FUNCTIONS);
+		BeamInstr** fp = hdr->functions;
 		int idx = mid - fp;
-		lookup_loc(fi, pc, rp->start, idx);
+		lookup_loc(fi, pc, hdr, idx);
 	    }
 	    return;
 	} else {
@@ -295,9 +296,9 @@ find_range(BeamInstr* pc)
 }
 
 static void
-lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx)
+lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamCodeHeader* code_hdr, int idx)
 {
-    Eterm* line = (Eterm *) modp[MI_LINE_TABLE];
+    Eterm* line = code_hdr->line_table;
     Eterm* low;
     Eterm* high;
     Eterm* mid;
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 64c8bc5e58..c7e7411935 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -381,7 +381,7 @@ loaded(int to, void *to_arg)
     int i;
     int old = 0;
     int cur = 0;
-    BeamInstr* code;
+    BeamCodeHeader* code;
     Module* modp;
     ErtsCodeIndex code_ix;
 
@@ -439,30 +439,30 @@ loaded(int to, void *to_arg)
 		erts_print(to, to_arg, "\n");
 		erts_print(to, to_arg, "Current size: %d\n",
 			   modp->curr.code_length);
-		code = modp->curr.code;
-		if (code != NULL && code[MI_ATTR_PTR]) {
+		code = modp->curr.code_hdr;
+		if (code != NULL && code->attr_ptr) {
 		    erts_print(to, to_arg, "Current attributes: ");
-		    dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR],
-				    code[MI_ATTR_SIZE]);
+		    dump_attributes(to, to_arg, code->attr_ptr,
+				    code->attr_size);
 		}
-		if (code != NULL && code[MI_COMPILE_PTR]) {
+		if (code != NULL && code->compile_ptr) {
 		    erts_print(to, to_arg, "Current compilation info: ");
-		    dump_attributes(to, to_arg, (byte *) code[MI_COMPILE_PTR],
-				    code[MI_COMPILE_SIZE]);
+		    dump_attributes(to, to_arg, code->compile_ptr,
+				    code->compile_size);
 		}
 
 		if (modp->old.code_length != 0) {
 		    erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length);
-		    code = modp->old.code;
-		    if (code[MI_ATTR_PTR]) {
+		    code = modp->old.code_hdr;
+		    if (code->attr_ptr) {
 			erts_print(to, to_arg, "Old attributes: ");
-			dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR],
-					code[MI_ATTR_SIZE]);
+			dump_attributes(to, to_arg, code->attr_ptr,
+					code->attr_size);
 		    }
-		    if (code[MI_COMPILE_PTR]) {
+		    if (code->compile_ptr) {
 			erts_print(to, to_arg, "Old compilation info: ");
-			dump_attributes(to, to_arg, (byte *) code[MI_COMPILE_PTR],
-					code[MI_COMPILE_SIZE]);
+			dump_attributes(to, to_arg, code->compile_ptr,
+					code->compile_size);
 		    }
 		}
 	    }
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index add4a66f90..01414f326d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2230,17 +2230,17 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
  ***************************************************************************/
 
 
-static BeamInstr** get_func_pp(BeamInstr* mod_code, Eterm f_atom, unsigned arity)
+static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity)
 {
-    int n = (int) mod_code[MI_NUM_FUNCTIONS];
+    int n = (int) mod_code->num_functions;
     int j;
     for (j = 0; j < n; ++j) {
-	BeamInstr* code_ptr = (BeamInstr*) mod_code[MI_FUNCTIONS+j];
+	BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j];
 	ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
 	if (f_atom == ((Eterm) code_ptr[3])
 	    && arity == ((unsigned) code_ptr[4])) {
 
-	    return (BeamInstr**) &mod_code[MI_FUNCTIONS+j];
+	    return (BeamInstr**) &mod_code->functions[j];
 	}
     }
     return NULL;
@@ -2423,8 +2423,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     if (init_func != NULL)
       handle = init_func;
 
-    if (!in_area(caller, mod->curr.code, mod->curr.code_length)) {
-	ASSERT(in_area(caller, mod->old.code, mod->old.code_length));
+    if (!in_area(caller, mod->curr.code_hdr, mod->curr.code_length)) {
+	ASSERT(in_area(caller, mod->old.code_hdr, mod->old.code_length));
 
 	ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old "
 			     "module '%T' not allowed", mod_atom);
@@ -2478,7 +2478,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
 	    BeamInstr** code_pp;
 	    if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1)
-		|| (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) {
+		|| (code_pp = get_func_pp(mod->curr.code_hdr, f_atom, f->arity))==NULL) {
 		ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u",
 				     mod_atom, f->name, f->arity);
 	    }
@@ -2621,7 +2621,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	{
 	    BeamInstr* code_ptr;
 	    erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1);
-	    code_ptr = *get_func_pp(mod->curr.code, f_atom, f->arity);
+	    code_ptr = *get_func_pp(mod->curr.code_hdr, f_atom, f->arity);
 
 	    if (code_ptr[1] == 0) {
 		code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif);
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 86dd3b5aac..f6794c012f 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -74,8 +74,8 @@ static Module* module_alloc(Module* tmpl)
     erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module));
 
     obj->module = tmpl->module;
-    obj->curr.code = 0;
-    obj->old.code = 0;
+    obj->curr.code_hdr = 0;
+    obj->old.code_hdr = 0;
     obj->curr.code_length = 0;
     obj->old.code_length = 0;
     obj->slot.index = -1;
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index c8a6351b04..e66d628ca9 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -26,7 +26,7 @@
 #endif
 
 struct erl_module_instance {
-    BeamInstr* code;
+    BeamCodeHeader* code_hdr;
     int code_length;		/* Length of loaded code in bytes. */
     unsigned catches;
     struct erl_module_nif* nif;
-- 
cgit v1.2.3


From ec4a7e7150c47523a553cac806c86ec08ab2dae7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 16 Sep 2015 15:02:19 +0200
Subject: Refactor GC

---
 erts/emulator/beam/erl_gc.c | 620 ++++++++++++++++++++++----------------------
 erts/emulator/beam/sys.h    |  37 ++-
 2 files changed, 334 insertions(+), 323 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index e316ab95ab..2d7b7cafa4 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,11 +42,76 @@
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
 
+#define ERTS_CONTINUOUS_NEW_HEAP
+
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10
 #define ERTS_INACT_WR_PB_LEAVE_PERCENTAGE 10
 
+#if defined(DEBUG) || 0
+#define ERTS_GC_DEBUG
+#else
+#undef ERTS_GC_DEBUG
+#endif
+#ifdef ERTS_GC_DEBUG
+#  define ERTS_GC_ASSERT ASSERT
+#else
+#  define ERTS_GC_ASSERT(B) ((void) 1)
+#endif
+
+#ifdef ERTS_CONTINUOUS_NEW_HEAP
+#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    ErtsInArea((Ptr), (NhPtr), (NhSz))
+#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (!ErtsInArea((Ptr), (NhPtr), (NhSz))				\
+     && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
+
+#ifdef ERTS_GC_DEBUG
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr))			\
+		       && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1)	\
+     : (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))			\
+		       || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))), 1)		\
+     : (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr))), 0))
+#endif
+
+#else
+
+#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (!erts_is_literal((TPtr), (Ptr)) && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
+#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (erts_is_literal((TPtr), (Ptr)))
+
+#ifdef ERTS_GC_DEBUG
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz))), 1)		\
+     : (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz))), 0))
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz))		\
+		       && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1)	\
+     : (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz))		\
+		       || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
+#endif
+
+#endif
+
+#ifndef ERTS_IS_NEW_HEAP_PTR
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
+#endif
+
+#ifndef ERTS_IS_LITERAL_PTR
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
+#endif
+
 /*
  * Returns number of elements in an array.
  */
@@ -100,13 +165,27 @@ static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
 static Uint combined_message_size(Process* p);
 static void remove_message_buffers(Process* p);
+static Eterm *full_sweep_heaps(Process *p,
+			       int hibernate,
+			       Eterm *n_heap, Eterm* n_htop,
+			       char *h, Uint h_size,
+			       char *oh, Uint oh_size,
+			       Eterm *objv, int nobj);
 static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
 static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
 static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
-static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size);
-static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size);
-static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
-			     char* src, Uint src_size);
+static Eterm *sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
+			     char* new_heap, Uint new_heap_size,
+			     char* old_heap, Uint old_heap_size);
+static Eterm *sweep_heaps(Eterm *n_hp, Eterm *n_htop,
+			  char* new_heap, Uint new_heap_size,
+			  char* old_heap, Uint old_heap_size);
+static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop,
+				    char* new_heap, Uint new_heap_size,
+				    char* old_heap, Uint old_heap_size,
+				    char* src, Uint src_size);
+static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
+					 char* src, Uint src_size);
 static Eterm* collect_heap_frags(Process* p, Eterm* heap,
 				 Eterm* htop, Eterm* objv, int nobj);
 static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
@@ -514,9 +593,6 @@ erts_garbage_collect_hibernate(Process* p)
     Uint heap_size;
     Eterm* heap;
     Eterm* htop;
-    Rootset rootset;
-    char* src;
-    Uint src_size;
     Uint actual_size;
     char* area;
     Uint area_size;
@@ -544,41 +620,16 @@ erts_garbage_collect_hibernate(Process* p)
 				    sizeof(Eterm)*heap_size);
     htop = heap;
 
-    (void) setup_rootset(p, p->arg_reg, p->arity, &rootset);
-#if HIPE
-    hipe_empty_nstack(p);
-#endif
-
-    src = (char *) p->heap;
-    src_size = (char *) p->htop - src;
-    htop = sweep_rootset(&rootset, htop, src, src_size);
-    htop = sweep_one_area(heap, htop, src, src_size);
-
-    if (p->old_heap) {
-	src = (char *) p->old_heap;
-	src_size = (char *) p->old_htop - src;
-	htop = sweep_rootset(&rootset, htop, src, src_size);
-	htop = sweep_one_area(heap, htop, src, src_size);
-    }
-
-    cleanup_rootset(&rootset);
-
-    if (MSO(p).first) {
-        sweep_off_heap(p, 1);
-    }
-
-    /*
-     *  Update all pointers.
-     */
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (void*)HEAP_START(p),
-		   HEAP_SIZE(p) * sizeof(Eterm));
-    if (p->old_heap) {
-	ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP,
-		       (void*)p->old_heap,
-		       (p->old_hend - p->old_heap) * sizeof(Eterm));
-	p->old_heap = p->old_htop = p->old_hend = 0;
-    }
+    htop = full_sweep_heaps(p,
+			    1,
+			    heap,
+			    htop,
+			    (char *) p->heap,
+			    (char *) p->htop - (char *) p->heap,
+			    (char *) p->old_heap,
+			    (char *) p->old_htop - (char *) p->old_heap,
+			    p->arg_reg,
+			    p->arity);
 
     p->heap = heap;
     p->high_water = htop;
@@ -724,7 +775,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
                 if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
-		} else if (in_area(ptr, area, area_size)) {
+		} else if (ErtsInArea(ptr, area, area_size)) {
                     MOVE_BOXED(ptr,val,old_htop,g_ptr++);
 		} else {
 		    g_ptr++;
@@ -735,7 +786,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
                 val = *ptr;
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
-		} else if (in_area(ptr, area, area_size)) {
+		} else if (ErtsInArea(ptr, area, area_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
                 } else {
 		    g_ptr++;
@@ -755,8 +806,11 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
      * Now we'll have to go through all heaps updating all other references.
      */
 
-    old_htop = sweep_one_heap(p->heap, p->htop, old_htop, area, area_size);
-    old_htop = sweep_one_area(p->old_heap, old_htop, area, area_size);
+    old_htop = sweep_literals_to_old_heap(p->heap, p->htop, old_htop, area, area_size);
+    old_htop = sweep_literal_area(p->old_heap, old_htop,
+				  (char *) p->heap, sizeof(Eterm)*p->heap_sz,
+				  (char *) p->old_heap, sizeof(Eterm)*old_heap_size,
+				  area, area_size);
     ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend);
     p->old_htop = old_htop;
 
@@ -1034,15 +1088,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
-                } else if (in_area(ptr, heap, mature_size)) {
+                } else if (ErtsInArea(ptr, heap, mature_size)) {
                     MOVE_BOXED(ptr,val,old_htop,g_ptr++);
-                } else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+                } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
                     MOVE_BOXED(ptr,val,n_htop,g_ptr++);
                 } else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
                 break;
@@ -1053,15 +1105,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 val = *ptr;
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
-                } else if (in_area(ptr, heap, mature_size)) {
+                } else if (ErtsInArea(ptr, heap, mature_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
-                } else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+                } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
                     MOVE_CONS(ptr,val,n_htop,g_ptr++);
                 } else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
 		break;
@@ -1084,7 +1134,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
      */
 
     if (mature_size == 0) {
-	n_htop = sweep_one_area(n_heap, n_htop, heap, heap_size);
+	n_htop = sweep_new_heap(n_heap, n_htop, heap, heap_size,
+				oh, oh_size);
     } else {
 	Eterm* n_hp = n_heap;
 	Eterm* ptr;
@@ -1101,15 +1152,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
-		} else if (in_area(ptr, heap, mature_size)) {
+		} else if (ErtsInArea(ptr, heap, mature_size)) {
 		    MOVE_BOXED(ptr,val,old_htop,n_hp++);
-		} else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+		} else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
 		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1119,15 +1168,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		val = *ptr;
 		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
-		} else if (in_area(ptr, heap, mature_size)) {
+		} else if (ErtsInArea(ptr, heap, mature_size)) {
 		    MOVE_CONS(ptr,val,old_htop,n_hp++);
-		} else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+		} else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1145,19 +1192,15 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 			if (IS_MOVED_BOXED(val)) {
 			    *origptr = val;
 			    mb->base = binary_bytes(val);
-			} else if (in_area(ptr, heap, mature_size)) {
+			} else if (ErtsInArea(ptr, heap, mature_size)) {
 			    MOVE_BOXED(ptr,val,old_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
-			} else if (in_area(ptr, heap, heap_size)) {
-			    ASSERT(!erts_is_literal(*origptr, origptr)
-				   && !in_area(ptr, oh, oh_size));
+			} else if (ERTS_IS_NEW_HEAP_PTR(*origptr, ptr,
+							heap, heap_size,
+							oh, oh_size)) {
 			    MOVE_BOXED(ptr,val,n_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			}
-			else {
-			    ASSERT(erts_is_literal(*origptr, origptr)
-				   || in_area(ptr, oh, oh_size));
-			}
 		    }
 		    n_hp += (thing_arityval(gval)+1);
 		}
@@ -1176,7 +1219,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
      */
 
     if (OLD_HTOP(p) < old_htop) {
-	old_htop = sweep_one_area(OLD_HTOP(p), old_htop, heap, heap_size);
+	old_htop = sweep_new_heap(OLD_HTOP(p), old_htop,
+				  heap, heap_size,
+				  oh, oh_size);
     }
     OLD_HTOP(p) = old_htop;
     HIGH_WATER(p) = n_htop;
@@ -1229,8 +1274,6 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 static int
 major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 {
-    Rootset rootset;
-    Roots* roots;
     const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p))
                               + (OLD_HTOP(p) - OLD_HEAP(p))
                               + MBUF_SIZE(p));
@@ -1240,8 +1283,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     Uint src_size = (char *) HEAP_TOP(p) - src;
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
-    Uint n;
-    Uint new_sz;
+    Uint new_sz, stk_sz;
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
@@ -1272,13 +1314,93 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj);
     }
 
+    n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, src, src_size,
+			      oh, oh_size, objv, nobj);
+
+    /* Move the stack to the end of the heap */
+    stk_sz = HEAP_END(p) - p->stop;
+    sys_memcpy(n_heap + new_sz - stk_sz, p->stop, stk_sz * sizeof(Eterm));
+    p->stop = n_heap + new_sz - stk_sz;
+
+#ifdef USE_VM_PROBES
+    if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
+        DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, pidbuf);
+        DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
+    }
+#endif
+
+    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
+		   (void *) HEAP_START(p),
+		   (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
+    HEAP_START(p) = n_heap;
+    HEAP_TOP(p) = n_htop;
+    HEAP_SIZE(p) = new_sz;
+    HEAP_END(p) = n_heap + new_sz;
+    GEN_GCS(p) = 0;
+
+    HIGH_WATER(p) = HEAP_TOP(p);
+
+    ErtsGcQuickSanityCheck(p);
+
+    *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+
+    {
+	ErlMessage *msgp;
+
+	/*
+	 * Copy newly received message onto the end of the new heap.
+	 */
+        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+	    if (msgp->data.attached) {
+                ErtsHeapFactory factory;
+                erts_factory_proc_prealloc_init(&factory, p,
+                                                erts_msg_attached_data_size(msgp));
+		erts_move_msg_attached_data_to_heap(&factory, msgp);
+                erts_factory_close(&factory);
+		ErtsGcQuickSanityCheck(p);
+	    }
+	}
+    }
+
+    adjust_after_fullsweep(p, need, objv, nobj);
+
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p);
+#endif
+    remove_message_buffers(p);
+
+    ErtsGcQuickSanityCheck(p);
+    return 1;			/* We are done. */
+}
+
+static Eterm *
+full_sweep_heaps(Process *p,
+		 int hibernate,
+		 Eterm *n_heap, Eterm* n_htop,
+		 char *h, Uint h_size,
+		 char *oh, Uint oh_size,
+		 Eterm *objv, int nobj)
+{
+    Rootset rootset;
+    Roots *roots;
+    Uint n;
+
     /*
      * Copy all top-level terms directly referenced by the rootset to
      * the new new_heap.
      */
 
     n = setup_rootset(p, objv, nobj, &rootset);
-    n_htop = fullsweep_nstack(p, n_htop);
+
+#ifdef HIPE
+    if (hibernate)
+	hipe_empty_nstack(p);
+    else
+	n_htop = fullsweep_nstack(p, n_htop);
+#endif
+
     roots = rootset.roots;
     while (n--) {
 	Eterm* g_ptr = roots->v;
@@ -1298,11 +1420,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
+		} else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
+						h, h_size,
+						oh, oh_size)) {
 		    MOVE_BOXED(ptr,val,n_htop,g_ptr++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1313,11 +1435,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		val = *ptr;
 		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
+		} else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
+						h, h_size,
+						oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1340,82 +1462,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      * until all is copied.
      */
 
-    if (oh_size == 0) {
-	n_htop = sweep_one_area(n_heap, n_htop, src, src_size);
-    } else {
-	Eterm* n_hp = n_heap;
-
-	while (n_hp != n_htop) {
-	    Eterm* ptr;
-	    Eterm val;
-	    Eterm gval = *n_hp;
-
-	    switch (primary_tag(gval)) {
-	    case TAG_PRIMARY_BOXED: {
-		ptr = boxed_val(gval);
-		val = *ptr;
-		if (IS_MOVED_BOXED(val)) {
-		    ASSERT(is_boxed(val));
-		    *n_hp++ = val;
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
-		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
-		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
-		    n_hp++;
-		}
-		break;
-	    }
-	    case TAG_PRIMARY_LIST: {
-		ptr = list_val(gval);
-		val = *ptr;
-		if (IS_MOVED_CONS(val)) {
-		    *n_hp++ = ptr[1];
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
-		    MOVE_CONS(ptr,val,n_htop,n_hp++);
-		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
-		    n_hp++;
-		}
-		break;
-	    }
-	    case TAG_PRIMARY_HEADER: {
-		if (!header_is_thing(gval))
-		    n_hp++;
-		else {
-		    if (header_is_bin_matchstate(gval)) {
-			ErlBinMatchState *ms = (ErlBinMatchState*) n_hp;
-			ErlBinMatchBuffer *mb = &(ms->mb);
-			Eterm* origptr;	
-			origptr = &(mb->orig);
-			ptr = boxed_val(*origptr);
-			val = *ptr;
-			if (IS_MOVED_BOXED(val)) {
-			    *origptr = val;
-			    mb->base = binary_bytes(*origptr);
-			} else if (in_area(ptr, src, src_size) ||
--				   in_area(ptr, oh, oh_size)) {
-			    ASSERT(!erts_is_literal(*origptr, origptr));
-			    MOVE_BOXED(ptr,val,n_htop,origptr); 
-			    mb->base = binary_bytes(*origptr);
-			    ptr = boxed_val(*origptr);
-			    val = *ptr;
-			}
-			else {
-			    ASSERT(erts_is_literal(*origptr, origptr));
-			}
-		    }
-		    n_hp += (thing_arityval(gval)+1);
-		}
-		break;
-	    }
-	    default:
-		n_hp++;
-		break;
-	    }
-	}
-    }
+    n_htop = sweep_heaps(n_heap, n_htop, h, h_size, oh, oh_size);
 
     if (MSO(p).first) {
 	sweep_off_heap(p, 1);
@@ -1428,62 +1475,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	OLD_HEAP(p) = OLD_HTOP(p) = OLD_HEND(p) = NULL;
     }
 
-    /* Move the stack to the end of the heap */
-    n = HEAP_END(p) - p->stop;
-    sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm));
-    p->stop = n_heap + new_sz - n;
-
-#ifdef USE_VM_PROBES
-    if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
-        DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
-
-        dtrace_proc_str(p, pidbuf);
-        DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
-    }
-#endif
-
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (void *) HEAP_START(p),
-		   (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
-    HEAP_START(p) = n_heap;
-    HEAP_TOP(p) = n_htop;
-    HEAP_SIZE(p) = new_sz;
-    HEAP_END(p) = n_heap + new_sz;
-    GEN_GCS(p) = 0;
-
-    HIGH_WATER(p) = HEAP_TOP(p);
-
-    ErtsGcQuickSanityCheck(p);
-
-    *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
-
-    {
-	ErlMessage *msgp;
-
-	/*
-	 * Copy newly received message onto the end of the new heap.
-	 */
-        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-	    if (msgp->data.attached) {
-                ErtsHeapFactory factory;
-                erts_factory_proc_prealloc_init(&factory, p,
-                                                erts_msg_attached_data_size(msgp));
-		erts_move_msg_attached_data_to_heap(&factory, msgp);
-                erts_factory_close(&factory);
-		ErtsGcQuickSanityCheck(p);
-	    }
-	}
-    }
-
-    adjust_after_fullsweep(p, need, objv, nobj);
-
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-    remove_message_buffers(p);
-
-    ErtsGcQuickSanityCheck(p);
-    return 1;			/* We are done. */
+    return n_htop;
 }
 
 static void
@@ -1585,7 +1577,7 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 		objv++;
 	    } else {
  		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1601,7 +1593,7 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 		objv++;
 	    } else {
 		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1644,9 +1636,9 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	    ptr = _unchecked_boxed_val(val);
-	    if (!in_area(ptr, heap, heap_size)) {
+	    if (!ErtsInArea(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1654,9 +1646,9 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	    break;
 	case TAG_PRIMARY_LIST:
 	    ptr = _unchecked_list_val(val);
-	    if (!in_area(ptr, heap, heap_size)) {
+	    if (!ErtsInArea(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1699,12 +1691,12 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	    ptr = (Eterm *) val;
-	    if (!in_area(ptr, old_heap, old_heap_size)) {
-		if (in_area(ptr, new_heap, new_heap_size)) {
+	    if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
+		if (ErtsInArea(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1712,12 +1704,12 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	    break;
 	case TAG_PRIMARY_LIST:
 	    ptr = (Eterm *) val;
-	    if (!in_area(ptr, old_heap, old_heap_size)) {
-		if (in_area(ptr, new_heap, new_heap_size)) {
+	    if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
+		if (ErtsInArea(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1726,7 +1718,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	case TAG_PRIMARY_HEADER:
 	    if (header_is_thing(val)) {
 		hp += _unchecked_thing_arityval(val);
-		if (!in_area(hp, old_heap, old_heap_size+1)) {
+		if (!ErtsInArea(hp, old_heap, old_heap_size+1)) {
 		    abort();
 		}
 	    }
@@ -1736,66 +1728,35 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 }
 #endif
 
-static Eterm*
-sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
+typedef enum {
+    ErtsSweepNewHeap,
+    ErtsSweepHeaps,
+    ErtsSweepLiteralArea
+} ErtsSweepType;
+
+static ERTS_FORCE_INLINE Eterm *
+sweep(Eterm *n_hp, Eterm *n_htop,
+      ErtsSweepType type,
+      char *h, Uint hsz,
+      char *oh, Uint ohsz,
+      char *src, Uint src_size)
 {
-    Roots* roots = rootset->roots;
-    Uint n = rootset->num_roots;
     Eterm* ptr;
-    Eterm gval;
     Eterm val;
+    Eterm gval;
 
-    while (n--) {
-        Eterm* g_ptr = roots->v;
-        Uint g_sz = roots->sz;
-
-	roots++;
-        while (g_sz--) {
-            gval = *g_ptr;
-
-            switch (primary_tag(gval)) {
-	    case TAG_PRIMARY_BOXED: {
-		ptr = boxed_val(gval);
-                val = *ptr;
-                if (IS_MOVED_BOXED(val)) {
-		    ASSERT(is_boxed(val));
-                    *g_ptr++ = val;
-                } else if (in_area(ptr, src, src_size)) {
-                    MOVE_BOXED(ptr,val,htop,g_ptr++);
-                } else {
-		    g_ptr++;
-		}
-                break;
-	    }
-	    case TAG_PRIMARY_LIST: {
-                ptr = list_val(gval);
-                val = *ptr;
-                if (IS_MOVED_CONS(val)) {
-                    *g_ptr++ = ptr[1];
-                } else if (in_area(ptr, src, src_size)) {
-                    MOVE_CONS(ptr,val,htop,g_ptr++);
-                } else {
-		    g_ptr++;
-		}
-		break;
-	    }
-
-	    default:
-                g_ptr++;
-		break;
-            }
-        }
-    }
-    return htop;
-}
-
+#undef ERTS_IS_IN_SWEEP_AREA
 
-static Eterm*
-sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
-{
-    Eterm* ptr;
-    Eterm val;
-    Eterm gval;
+#define ERTS_IS_IN_SWEEP_AREA(TPtr, Ptr)				\
+    (type == ErtsSweepHeaps						\
+     ? !ERTS_IS_LITERAL_PTR((TPtr), (Ptr), h, hsz, oh, ohsz)		\
+     : (type == ErtsSweepNewHeap					\
+	? ERTS_IS_NEW_HEAP_PTR((TPtr), (Ptr), h, hsz, oh, ohsz)		\
+	: (ErtsInArea((Ptr), src, src_size)				\
+	   ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))		\
+			     && !ErtsInArea((Ptr), h, hsz)		\
+			     && !ErtsInArea((Ptr), oh, ohsz)), 1)	\
+	   : 0)))
 
     while (n_hp != n_htop) {
 	ASSERT(n_hp < n_htop);
@@ -1807,7 +1768,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*n_hp++ = val;
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
 		MOVE_BOXED(ptr,val,n_htop,n_hp++);
 	    } else {
 		n_hp++;
@@ -1819,7 +1780,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	    val = *ptr;
 	    if (IS_MOVED_CONS(val)) {
 		*n_hp++ = ptr[1];
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
 		MOVE_CONS(ptr,val,n_htop,n_hp++);
 	    } else {
 		n_hp++;
@@ -1840,7 +1801,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 		    if (IS_MOVED_BOXED(val)) {
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
-		    } else if (in_area(ptr, src, src_size)) {
+		    } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) {
 			MOVE_BOXED(ptr,val,n_htop,origptr); 
 			mb->base = binary_bytes(*origptr);
 		    }
@@ -1855,10 +1816,49 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	}
     }
     return n_htop;
+#undef ERTS_IS_IN_SWEEP_AREA
+}
+
+static Eterm *
+sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
+	       char* new_heap, Uint new_heap_size,
+	       char* old_heap, Uint old_heap_size)
+{
+    return sweep(n_hp, n_htop,
+		 ErtsSweepNewHeap,
+		 new_heap, new_heap_size,
+		 old_heap, old_heap_size,
+		 NULL, 0);
+}
+
+static Eterm *
+sweep_heaps(Eterm *n_hp, Eterm *n_htop,
+	    char* new_heap, Uint new_heap_size,
+	    char* old_heap, Uint old_heap_size)
+{
+    return sweep(n_hp, n_htop,
+		 ErtsSweepHeaps,
+		 new_heap, new_heap_size,
+		 old_heap, old_heap_size,
+		 NULL, 0);
+}
+
+static Eterm *
+sweep_literal_area(Eterm *n_hp, Eterm *n_htop,
+		   char* new_heap, Uint new_heap_size,
+		   char* old_heap, Uint old_heap_size,
+		   char* src, Uint src_size)
+{
+    return sweep(n_hp, n_htop,
+		 ErtsSweepLiteralArea,
+		 new_heap, new_heap_size,
+		 old_heap, old_heap_size,
+		 src, src_size);
 }
 
 static Eterm*
-sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size)
+sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
+			   char* src, Uint src_size)
 {
     while (heap_ptr < heap_end) {
 	Eterm* ptr;
@@ -1872,7 +1872,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*heap_ptr++ = val;
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ErtsInArea(ptr, src, src_size)) {
 		MOVE_BOXED(ptr,val,htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
@@ -1884,7 +1884,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	    val = *ptr;
 	    if (IS_MOVED_CONS(val)) {
 		*heap_ptr++ = ptr[1];
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ErtsInArea(ptr, src, src_size)) {
 		MOVE_CONS(ptr,val,htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
@@ -1905,7 +1905,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 		    if (IS_MOVED_BOXED(val)) {
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
-		    } else if (in_area(ptr, src, src_size)) {
+		    } else if (ErtsInArea(ptr, src, src_size)) {
 			MOVE_BOXED(ptr,val,htop,origptr);
 			mb->base = binary_bytes(*origptr);
 		    }
@@ -2340,11 +2340,11 @@ sweep_off_heap(Process *p, int fullsweep)
      */
     while (ptr) {
 	if (IS_MOVED_BOXED(ptr->thing_word)) {
-	    ASSERT(!in_area(ptr, oheap, oheap_sz));
+	    ASSERT(!ErtsInArea(ptr, oheap, oheap_sz));
 	    *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
 	    ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
 	    if (ptr->thing_word == HEADER_PROC_BIN) {
-		int to_new_heap = !in_area(ptr, oheap, oheap_sz);
+		int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz);
 		ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
 		if (to_new_heap) {
 		    bin_vheap += ptr->size / sizeof(Eterm);
@@ -2358,7 +2358,7 @@ sweep_off_heap(Process *p, int fullsweep)
 		ptr = ptr->next;
 	    }
 	}
-	else if (!in_area(ptr, oheap, oheap_sz)) {
+	else if (!ErtsInArea(ptr, oheap, oheap_sz)) {
 	    /* garbage */
 	    switch (thing_subtag(ptr->thing_word)) {
 	    case REFC_BINARY_SUBTAG:
@@ -2390,7 +2390,7 @@ sweep_off_heap(Process *p, int fullsweep)
      * generational collection - keep objects in list.
      */
     while (ptr) {
-	ASSERT(in_area(ptr, oheap, oheap_sz));
+	ASSERT(ErtsInArea(ptr, oheap, oheap_sz));
 	ASSERT(!IS_MOVED_BOXED(ptr->thing_word));       
 	if (ptr->thing_word == HEADER_PROC_BIN) {
 	    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
@@ -2480,7 +2480,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 	switch (primary_tag(val)) {
 	  case TAG_PRIMARY_LIST:
 	  case TAG_PRIMARY_BOXED:
-	      if (in_area(ptr_val(val), area, area_size)) {
+	      if (ErtsInArea(ptr_val(val), area, area_size)) {
 		  *hp = offset_ptr(val, offs);
 	      }
 	      hp++;
@@ -2502,7 +2502,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 		  {
 		      struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
 
-		      if (in_area(oh->next, area, area_size)) {
+		      if (ErtsInArea(oh->next, area, area_size)) {
 			  Eterm** uptr = (Eterm **) (void *) &oh->next;
 			  *uptr += offs; /* Patch the mso chain */
 		      }
@@ -2512,7 +2512,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 		{	
 		  ErlBinMatchState *ms = (ErlBinMatchState*) hp;
 		  ErlBinMatchBuffer *mb = &(ms->mb);
-		  if (in_area(ptr_val(mb->orig), area, area_size)) {
+		  if (ErtsInArea(ptr_val(mb->orig), area, area_size)) {
 		      mb->orig = offset_ptr(mb->orig, offs);
 		      mb->base = binary_bytes(mb->orig);
 		  }
@@ -2542,7 +2542,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_LIST:
 	case TAG_PRIMARY_BOXED:
-	    if (in_area(ptr_val(val), area, area_size)) {
+	    if (ErtsInArea(ptr_val(val), area, area_size)) {
 		*hp = offset_ptr(val, offs);
 	    }
 	    hp++;
@@ -2557,7 +2557,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 static void
 offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
 {
-    if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) {
+    if (MSO(p).first && ErtsInArea((Eterm *)MSO(p).first, area, area_size)) {
         Eterm** uptr = (Eterm**) (void *) &MSO(p).first;
         *uptr += offs;
     }
@@ -2577,19 +2577,19 @@ offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
 	    switch (primary_tag(mesg)) {
 	    case TAG_PRIMARY_LIST:
 	    case TAG_PRIMARY_BOXED:
-		if (in_area(ptr_val(mesg), area, area_size)) {
+		if (ErtsInArea(ptr_val(mesg), area, area_size)) {
 		    ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
 		}
 		break;
 	    }
 	}
 	mesg = ERL_MESSAGE_TOKEN(mp);
-	if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
+	if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
 	    ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
         }
 #ifdef USE_VM_PROBES
 	mesg = ERL_MESSAGE_DT_UTAG(mp);
-	if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
+	if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
 	    ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs);
         }
 #endif	
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d63e81cb5e..5db7ee6d7c 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -21,6 +21,19 @@
 #ifndef __SYS_H__
 #define __SYS_H__
 
+#if !defined(__GNUC__)
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
+#elif !defined(__GNUC_MINOR__)
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#elif !defined(__GNUC_PATCHLEVEL__)
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#else
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#endif
+
 #ifdef ERTS_INLINE
 #  ifndef ERTS_CAN_INLINE
 #    define ERTS_CAN_INLINE 1
@@ -38,6 +51,17 @@
 #  endif
 #endif
 
+#ifndef ERTS_FORCE_INLINE
+#  if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
+#    define ERTS_FORCE_INLINE __inline__ __attribute__((__always_inline__))
+#  elif defined(__WIN32__)
+#    define ERTS_FORCE_INLINE __forceinline
+#  endif
+#  ifndef ERTS_FORCE_INLINE
+#    define ERTS_FORCE_INLINE ERTS_INLINE
+#  endif
+#endif
+
 #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
 #  undef ERTS_CAN_INLINE
 #  define ERTS_CAN_INLINE 0
@@ -112,19 +136,6 @@ typedef int ErtsSysFdType;
 typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #endif
 
-#if !defined(__GNUC__)
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
-#elif !defined(__GNUC_MINOR__)
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
-  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#elif !defined(__GNUC_PATCHLEVEL__)
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
-  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#else
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
-  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#endif
-
 #if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
 #  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
 #  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
-- 
cgit v1.2.3


From 7858ca939f8bf2db918396616fee13364d150a1e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 5 Oct 2015 16:13:37 +0200
Subject: erts: Refactor line table in loaded beam code

to use real C struct with correct types
---
 erts/emulator/beam/beam_load.c   | 52 ++++++++++++++++++++--------------------
 erts/emulator/beam/beam_load.h   | 18 +++++++++-----
 erts/emulator/beam/beam_ranges.c | 37 +++++++++++++---------------
 3 files changed, 54 insertions(+), 53 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index c56e6732a0..1c598601c6 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4388,9 +4388,11 @@ freeze_code(LoaderState* stp)
     if (stp->line_instr == 0) {
 	line_size = 0;
     } else {
-	line_size = (MI_LINE_FUNC_TAB + (stp->num_functions + 1) +
-		     (stp->current_li+1) + stp->num_fnames) *
-	    sizeof(Eterm) + (stp->current_li+1) * stp->loc_size;
+	line_size = (offsetof(BeamCodeLineTab,func_tab)
+                     + (stp->num_functions + 1) * sizeof(BeamInstr**) /* func_tab */
+                     + (stp->current_li + 1) * sizeof(BeamInstr*)     /* line items */
+                     + stp->num_fnames * sizeof(Eterm)                /* fname table */
+                     + (stp->current_li + 1) * stp->loc_size);        /* loc_tab */
     }
     size = offsetof(BeamCodeHeader,functions) + (stp->ci * sizeof(BeamInstr)) +
 	strtab_size + attr_size + compile_size + MD5_SIZE + line_size;
@@ -4465,42 +4467,40 @@ freeze_code(LoaderState* stp)
 	code_hdr->line_table = NULL;
 	str_table = (byte *) (codev + stp->ci);
     } else {
-	Eterm* line_tab = (Eterm *) (codev+stp->ci);
-	Eterm* p;
-	int ftab_size = stp->num_functions;
-	int num_instrs = stp->current_li;
-	Eterm* first_line_item;
+	BeamCodeLineTab* const line_tab = (BeamCodeLineTab *) (codev+stp->ci);
+	const int ftab_size = stp->num_functions;
+        const int num_instrs = stp->current_li;
+        const BeamInstr** const line_items =
+            (const BeamInstr**) &line_tab->func_tab[ftab_size + 1];
 
 	code_hdr->line_table = line_tab;
-	p = line_tab + MI_LINE_FUNC_TAB;
 
-	first_line_item = (p + ftab_size + 1);
 	for (i = 0; i < ftab_size; i++) {
-	     *p++ = (Eterm) (BeamInstr) (first_line_item + stp->func_line[i]);
+            line_tab->func_tab[i] = line_items + stp->func_line[i];
 	}
-	*p++ = (Eterm) (BeamInstr) (first_line_item + num_instrs);
-	ASSERT(p == first_line_item);
+	line_tab->func_tab[i] = line_items + num_instrs;
+
 	for (i = 0; i < num_instrs; i++) {
-		*p++ = (Eterm) (BeamInstr) (codev + stp->line_instr[i].pos);
+            line_items[i] = codev + stp->line_instr[i].pos;
 	}
-	*p++ = (Eterm) (BeamInstr) (codev + stp->ci - 1);
+	line_items[i] = codev + stp->ci - 1;
 
-	line_tab[MI_LINE_FNAME_PTR] = (Eterm) (BeamInstr) p;
-	memcpy(p, stp->fname, stp->num_fnames*sizeof(Eterm));
-	p += stp->num_fnames;
+	line_tab->fname_ptr = (Eterm*) &line_items[i + 1];
+	memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm));
 
-	line_tab[MI_LINE_LOC_TAB] = (Eterm) (BeamInstr) p;
-	line_tab[MI_LINE_LOC_SIZE] = stp->loc_size;
+	line_tab->loc_size = stp->loc_size;
 	if (stp->loc_size == 2) {
-	    Uint16* locp = (Uint16 *) p;
-	    for (i = 0; i < num_instrs; i++) {
+            Uint16* locp = (Uint16 *) &line_tab->fname_ptr[stp->num_fnames];
+            line_tab->loc_tab.p2 = locp;
+            for (i = 0; i < num_instrs; i++) {
 		*locp++ = (Uint16) stp->line_instr[i].loc;
-	    }
-	    *locp++ = LINE_INVALID_LOCATION;
+            }
+            *locp++ = LINE_INVALID_LOCATION;
             str_table = (byte *) locp;
 	} else {
-	    Uint32* locp = (Uint32 *) p;
-	    ASSERT(stp->loc_size == 4);
+	    Uint32* locp = (Uint32 *) &line_tab->fname_ptr[stp->num_fnames];
+            ASSERT(stp->loc_size == 4);
+            line_tab->loc_tab.p4 = locp;
 	    for (i = 0; i < num_instrs; i++) {
 		*locp++ = stp->line_instr[i].loc;
 	    }
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index c86ac65521..22ab71c868 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -61,6 +61,8 @@ extern BeamInstr* em_call_nif;
 /* Total code size in bytes */
 extern Uint erts_total_code_size;
 
+typedef struct BeamCodeLineTab_ BeamCodeLineTab;
+
 /*
  * Header of code chunks which contains additional information
  * about the loaded module.
@@ -100,7 +102,7 @@ typedef struct beam_code_header {
     /*
      * Pointer to the line table (or NULL if none).
      */
-    Eterm* line_table;
+    BeamCodeLineTab* line_table;
 
     /*
      * Pointer to the module MD5 sum (16 bytes)
@@ -124,11 +126,15 @@ int erts_is_module_native(BeamCodeHeader* code);
 /*
  * Layout of the line table.
  */
-
-#define MI_LINE_FNAME_PTR 0
-#define MI_LINE_LOC_TAB 1
-#define MI_LINE_LOC_SIZE 2
-#define MI_LINE_FUNC_TAB 3
+struct BeamCodeLineTab_ {
+    Eterm* fname_ptr;
+    int loc_size;
+    union {
+        Uint16* p2;
+        Uint32* p4;
+    }loc_tab;
+    const BeamInstr** func_tab[1];
+};
 
 #define LINE_INVALID_LOCATION (0)
 
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index e6cf43446e..5a2b66727a 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -37,7 +37,7 @@ typedef struct {
 #define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end))
 
 static Range* find_range(BeamInstr* pc);
-static void lookup_loc(FunctionInfo* fi, BeamInstr* pc,
+static void lookup_loc(FunctionInfo* fi, const BeamInstr* pc,
 		       BeamCodeHeader*, int idx);
 
 /*
@@ -296,39 +296,34 @@ find_range(BeamInstr* pc)
 }
 
 static void
-lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamCodeHeader* code_hdr, int idx)
+lookup_loc(FunctionInfo* fi, const BeamInstr* pc,
+           BeamCodeHeader* code_hdr, int idx)
 {
-    Eterm* line = code_hdr->line_table;
-    Eterm* low;
-    Eterm* high;
-    Eterm* mid;
-    Eterm pc;
+    BeamCodeLineTab* lt = code_hdr->line_table;
+    const BeamInstr** low;
+    const BeamInstr** high;
+    const BeamInstr** mid;
 
-    if (line == 0) {
+    if (lt == NULL) {
 	return;
     }
 
-    pc = (Eterm) (BeamInstr) orig_pc;
-    fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR];
-    low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx];
-    high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1];
+    fi->fname_ptr = lt->fname_ptr;
+    low = lt->func_tab[idx];
+    high = lt->func_tab[idx+1];
     while (high > low) {
 	mid = low + (high-low) / 2;
 	if (pc < mid[0]) {
 	    high = mid;
 	} else if (pc < mid[1]) {
 	    int file;
-	    int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB];
+	    int index = mid - lt->func_tab[0];
 
-	    if (line[MI_LINE_LOC_SIZE] == 2) {
-		Uint16* loc_table =
-		    (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB];
-		fi->loc = loc_table[index];
+	    if (lt->loc_size == 2) {
+		fi->loc = lt->loc_tab.p2[index];
 	    } else {
-		Uint32* loc_table =
-		    (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB];
-		ASSERT(line[MI_LINE_LOC_SIZE] == 4);
-		fi->loc = loc_table[index];
+		ASSERT(lt->loc_size == 4);
+		fi->loc = lt->loc_tab.p4[index];
 	    }
 	    if (fi->loc == LINE_INVALID_LOCATION) {
 		return;
-- 
cgit v1.2.3


From 3ac08f9b668613a4292436979eacc61863c2ab94 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 16 Sep 2015 15:02:50 +0200
Subject: Fragmented young heap generation and off_heap_message_queue option

* The youngest generation of the heap can now consist of multiple
  blocks. Heap fragments and message fragments are added to the
  youngest generation when needed without triggering a GC. After
  a GC the youngest generation is contained in one single block.
* The off_heap_message_queue process flag has been added. When
  enabled all message data in the queue is kept off heap. When
  a message is selected from the queue, the message fragment (or
  heap fragment) containing the actual message is attached to the
  youngest generation. Messages stored off heap is not part of GC.
---
 erts/emulator/beam/atom.names         |    1 +
 erts/emulator/beam/beam_bif_load.c    |  216 +++--
 erts/emulator/beam/beam_emu.c         |  142 ++--
 erts/emulator/beam/beam_load.c        |    6 +-
 erts/emulator/beam/bif.c              |   39 +-
 erts/emulator/beam/bif.h              |   48 +-
 erts/emulator/beam/break.c            |    4 +-
 erts/emulator/beam/copy.c             |    4 +-
 erts/emulator/beam/dist.c             |   34 +-
 erts/emulator/beam/erl_alloc.c        |   21 +-
 erts/emulator/beam/erl_alloc.h        |    4 +-
 erts/emulator/beam/erl_alloc.types    |    2 +
 erts/emulator/beam/erl_bif_ddll.c     |   17 +-
 erts/emulator/beam/erl_bif_info.c     |  270 +++---
 erts/emulator/beam/erl_debug.c        |    6 +-
 erts/emulator/beam/erl_gc.c           |  688 ++++++++-------
 erts/emulator/beam/erl_gc.h           |   27 +-
 erts/emulator/beam/erl_hl_timer.c     |   85 +-
 erts/emulator/beam/erl_init.c         |   26 +-
 erts/emulator/beam/erl_message.c      | 1509 ++++++++++++++++++++-------------
 erts/emulator/beam/erl_message.h      |  277 ++++--
 erts/emulator/beam/erl_nif.c          |    7 +-
 erts/emulator/beam/erl_node_tables.c  |   68 +-
 erts/emulator/beam/erl_process.c      |  346 ++++----
 erts/emulator/beam/erl_process.h      |  106 ++-
 erts/emulator/beam/erl_process_dump.c |   13 +-
 erts/emulator/beam/erl_process_lock.h |    6 -
 erts/emulator/beam/erl_time_sup.c     |    7 +-
 erts/emulator/beam/erl_trace.c        |   68 +-
 erts/emulator/beam/global.h           |  123 +--
 erts/emulator/beam/io.c               |  113 +--
 erts/emulator/beam/sys.h              |    2 +
 erts/emulator/beam/utils.c            |   17 +-
 33 files changed, 2380 insertions(+), 1922 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 6328b3d18f..f142cf1142 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -423,6 +423,7 @@ atom notify
 atom notsup
 atom nouse_stdio
 atom objects
+atom off_heap_message_queue
 atom offset
 atom ok
 atom old_heap_block_size
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 2c275c4649..22b4e26c77 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -723,29 +723,50 @@ set_default_trace_pattern(Eterm module)
     }
 }
 
+static ERTS_INLINE int
+check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size)
+{
+    struct erl_off_heap_header* oh;
+    for (oh = off_heap->first; oh; oh = oh->next) {
+	if (thing_subtag(oh->thing_word) == FUN_SUBTAG) {
+	    ErlFunThing* funp = (ErlFunThing*) oh;
+	    if (ErtsInArea(funp->fe->address, area, area_size))
+		return !0;
+	}
+    }
+    return 0;
+}
+
+
 static Eterm
 check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 {
     BeamInstr* start;
     char* literals;
     Uint lit_bsize;
-    BeamInstr* end;
+    char* mod_start;
+    Uint mod_size;
     Eterm* sp;
-    struct erl_off_heap_header* oh;
     int done_gc = 0;
+    int need_gc = 0;
+    ErtsMessage *msgp;
+    ErlHeapFragment *hfrag;
 
-#define INSIDE(a) (start <= (a) && (a) < end)
+#define ERTS_ORDINARY_GC__ (1 << 0)
+#define ERTS_LITERAL_GC__  (1 << 1)
 
     /*
      * Pick up limits for the module.
      */
     start = (BeamInstr*) modp->old.code_hdr;
-    end = (BeamInstr *)((char *)start + modp->old.code_length);
+    mod_start = (char *) start;
+    mod_size = modp->old.code_length;
 
     /*
      * Check if current instruction or continuation pointer points into module.
      */
-    if (INSIDE(rp->i) || INSIDE(rp->cp)) {
+    if (ErtsInArea(rp->i, mod_start, mod_size)
+	|| ErtsInArea(rp->cp, mod_start, mod_size)) {
 	return am_true;
     }
 
@@ -753,7 +774,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
      * Check all continuation pointers stored on the stack.
      */
     for (sp = rp->stop; sp < STACK_START(rp); sp++) {
-	if (is_CP(*sp) && INSIDE(cp_val(*sp))) {
+	if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) {
 	    return am_true;
 	}
     }
@@ -767,15 +788,15 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	struct StackTrace *s;
 	ASSERT(is_list(rp->ftrace));
 	s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace)));
-	if ((s->pc && INSIDE(s->pc)) ||
-	    (s->current && INSIDE(s->current))) {
+	if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) ||
+	    (s->current && ErtsInArea(s->current, mod_start, mod_size))) {
 	    rp->freason = EXC_NULL;
 	    rp->fvalue = NIL;
 	    rp->ftrace = NIL;
 	} else {
 	    int i;
 	    for (i = 0;  i < s->depth;  i++) {
-		if (INSIDE(s->trace[i])) {
+		if (ErtsInArea(s->trace[i], mod_start, mod_size)) {
 		    rp->freason = EXC_NULL;
 		    rp->fvalue = NIL;
 		    rp->ftrace = NIL;
@@ -796,108 +817,141 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
     }
 
     /*
-     * See if there are funs that refer to the old version of the module.
+     * Message queue can contains funs, but (at least currently) no
+     * constants. If we got references to this module from the message
+     * queue, a GC cannot remove these...
      */
 
- rescan:
-    for (oh = MSO(rp).first; oh; oh = oh->next) {
-	if (thing_subtag(oh->thing_word) == FUN_SUBTAG) {
-	    ErlFunThing* funp = (ErlFunThing*) oh;
+    erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
+    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
+    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
 
-	    if (INSIDE((BeamInstr *) funp->fe->address)) {
-		if (done_gc) {
-		    return am_true;
-		} else {
-		    if (!allow_gc)
-			return am_aborted;
-		    /*
-		    * Try to get rid of this fun by garbage collecting.
-		    * Clear both fvalue and ftrace to make sure they
-		    * don't hold any funs.
-		    */
-		    rp->freason = EXC_NULL;
-		    rp->fvalue = NIL;
-		    rp->ftrace = NIL;
-		    done_gc = 1;
-		    FLAGS(rp) |= F_NEED_FULLSWEEP;
-		    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
-		    goto rescan;
-		}
-	    }
+    for (msgp = rp->msg.first; msgp; msgp = msgp->next) {
+	if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
+	    hfrag = &msgp->hfrag;
+	else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag)
+	    hfrag = msgp->data.heap_frag;
+	else
+	    continue;
+	for (; hfrag; hfrag = hfrag->next) {
+	    if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
+		return am_true;
+	    /* Should not contain any constants... */
+	    ASSERT(!any_heap_ref_ptrs(&hfrag->mem[0],
+				      &hfrag->mem[hfrag->used_size],
+				      mod_start,
+				      mod_size));
 	}
     }
 
-    /*
-     * See if there are constants inside the module referenced by the process.
-     */
-    done_gc = 0;
     literals = (char*) modp->old.code_hdr->literals_start;
     lit_bsize = (char*) modp->old.code_hdr->literals_end - literals;
-    for (;;) {
-	ErlMessage* mp;
 
+    while (1) {
+
+	/* Check heap, stack etc... */
+	if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size))
+	    goto try_gc;
 	if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) {
 	    rp->freason = EXC_NULL;
 	    rp->fvalue = NIL;
 	    rp->ftrace = NIL;
 	}
-	if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) {
-	    goto need_gc;
-	}
-	if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) {
-	    goto need_gc;
-	}
+	if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize))
+	    goto try_literal_gc;
+	if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize))
+	    goto try_literal_gc;
+	if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize))
+	    goto try_literal_gc;
+
+	/* Check dictionary */
+	if (rp->dictionary) {
+	    Eterm* start = rp->dictionary->data;
+	    Eterm* end = start + rp->dictionary->used;
 
-	if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) {
-	    goto need_gc;
+	    if (any_heap_ref_ptrs(start, end, literals, lit_bsize))
+		goto try_literal_gc;
 	}
 
-	if (rp->dictionary != NULL) {
-	    Eterm* start = rp->dictionary->data;
-	    Eterm* end = start + rp->dictionary->used;
+	/* Check heap fragments */
+	for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) {
+	    Eterm *hp, *hp_end;
+	    /* Off heap lists should already have been moved into process */
+	    ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size));
 
-	    if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) {
-		goto need_gc;
-	    }
+	    hp = &hfrag->mem[0];
+	    hp_end = &hfrag->mem[hfrag->used_size];
+	    if (any_heap_ref_ptrs(hp, hp_end, mod_start, lit_bsize))
+		goto try_literal_gc;
 	}
 
-	for (mp = rp->msg.first; mp != NULL; mp = mp->next) {
-	    if (any_heap_ref_ptrs(mp->m, mp->m+2, literals, lit_bsize)) {
-		goto need_gc;
+#ifdef DEBUG
+	/*
+	 * Message buffer fragments should not have any references
+	 * to constants, and off heap lists should already have
+	 * been moved into process off heap structure.
+	 */
+	for (msgp = rp->msg_frag; msgp; msgp = msgp->next) {
+	    if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
+		hfrag = &msgp->hfrag;
+	    else
+		hfrag = msgp->data.heap_frag;
+	    for (; hfrag; hfrag = hfrag->next) {
+		Eterm *hp, *hp_end;
+		ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size));
+
+		hp = &hfrag->mem[0];
+		hp_end = &hfrag->mem[hfrag->used_size];
+		ASSERT(!any_heap_ref_ptrs(hp, hp_end, mod_start, lit_bsize));
 	    }
 	}
-	break;
 
-    need_gc:
-	if (done_gc) {
+#endif
+
+	return am_false;
+
+    try_literal_gc:
+	need_gc |= ERTS_LITERAL_GC__;
+
+    try_gc:
+	need_gc |= ERTS_ORDINARY_GC__;
+
+	if ((done_gc & need_gc) == need_gc)
 	    return am_true;
-	} else {
-	    struct erl_off_heap_header* oh;
 
-	    if (!allow_gc)
-		return am_aborted;
+	if (!allow_gc)
+	    return am_aborted;
 
-	    /*
-	     * Try to get rid of constants by by garbage collecting.
-	     * Clear both fvalue and ftrace.
-	     */
-	    rp->freason = EXC_NULL;
-	    rp->fvalue = NIL;
-	    rp->ftrace = NIL;
-	    done_gc = 1;
+	need_gc &= ~done_gc;
+
+	/*
+	 * Try to get rid of constants by by garbage collecting.
+	 * Clear both fvalue and ftrace.
+	 */
+
+	rp->freason = EXC_NULL;
+	rp->fvalue = NIL;
+	rp->ftrace = NIL;
+
+	if (need_gc & ERTS_ORDINARY_GC__) {
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
 	    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+	    done_gc |= ERTS_ORDINARY_GC__;
+	}
+	if (need_gc & ERTS_LITERAL_GC__) {
+	    struct erl_off_heap_header* oh;
 	    oh = modp->old.code_hdr->literals_off_heap;
 	    *redsp += lit_bsize / 64; /* Need, better value... */
 	    erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh);
+	    done_gc |= ERTS_LITERAL_GC__;
 	}
+	need_gc = 0;
     }
-    return am_false;
-#undef INSIDE
-}
 
-#define in_area(ptr,start,nbytes) \
-    ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
+#undef ERTS_ORDINARY_GC__
+#undef ERTS_LITERAL_GC__
+
+}
 
 static int
 any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
@@ -910,7 +964,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(val, mod_start, mod_size)) {
+	    if (ErtsInArea(val, mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -930,7 +984,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(val, mod_start, mod_size)) {
+	    if (ErtsInArea(val, mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -940,7 +994,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
                 if (header_is_bin_matchstate(val)) {
                     ErlBinMatchState *ms = (ErlBinMatchState*) p;
                     ErlBinMatchBuffer *mb = &(ms->mb);
-                    if (in_area(mb->orig, mod_start, mod_size)) {
+                    if (ErtsInArea(mb->orig, mod_start, mod_size)) {
                         return 1;
                     }
                 }
@@ -953,8 +1007,6 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
     return 0;
 }
 
-#undef in_area
-
 BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 {
     ErtsCodeIndex code_ix;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4d19f52a52..1dd56ff989 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1654,10 +1654,6 @@ void process_main(void)
      ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
-     if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
-	 result = erts_gc_after_bif_call(c_p, result, reg, 2);
-	 E = c_p->stop;
-     }
      HTOP = HEAP_TOP(c_p);
      FCALLS = c_p->fcalls;
      if (is_value(result)) {
@@ -1745,8 +1741,7 @@ void process_main(void)
 		 SWAPIN;
 	     }
 	     /* only x(2) is included in the rootset here */
-	     if (E - HTOP < 3 || c_p->mbuf) {	/* Force GC in case add_stacktrace()
-						 * created heap fragments */
+	     if (E - HTOP < 3) {
 		 SWAPOUT;
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
@@ -1833,10 +1828,17 @@ void process_main(void)
  OpCase(i_loop_rec_f):
  {
      BeamInstr *next;
-     ErlMessage* msgp;
+     ErtsMessage* msgp;
 
- loop_rec__:
+     /*
+      * We need to disable GC while matching messages
+      * in the queue. This since messages with data outside
+      * the heap will be corrupted by a GC.
+      */
+     ASSERT(!(c_p->flags & F_DISABLE_GC));
+     c_p->flags |= F_DISABLE_GC;
 
+ loop_rec__:
      PROCESS_MAIN_CHK_LOCKS(c_p);
 
      msgp = PEEK_MESSAGE(c_p);
@@ -1848,6 +1850,7 @@ void process_main(void)
 	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	     SWAPOUT;
+	     c_p->flags &= ~F_DISABLE_GC;
 	     goto do_schedule; /* Will be rescheduled for exit */
 	 }
 	 ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
@@ -1857,30 +1860,27 @@ void process_main(void)
 	 else
 #endif
 	 {
+	     c_p->flags &= ~F_DISABLE_GC;
 	     SET_I((BeamInstr *) Arg(0));
 	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
 	 }
      }
-     ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS,
-				   {
-				       SWAPOUT;
-				       PROCESS_MAIN_CHK_LOCKS(c_p);
-				   },
-				   {
-				       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-				       PROCESS_MAIN_CHK_LOCKS(c_p);
-				       SWAPIN;
-				   });
      if (is_non_value(ERL_MESSAGE_TERM(msgp))) {
-	 /*
-	  * A corrupt distribution message that we weren't able to decode;
-	  * remove it...
-	  */
-	 ASSERT(!msgp->data.attached);
-         /* TODO: Add DTrace probe for this bad message situation? */
-	 UNLINK_MESSAGE(c_p, msgp);
-	 free_message(msgp);
-	 goto loop_rec__;
+	 SWAPOUT; /* erts_decode_dist_message() may write to heap... */
+	 if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
+	     /*
+	      * A corrupt distribution message that we weren't able to decode;
+	      * remove it...
+	      */
+	     /* No swapin should be needed */
+	     ASSERT(HTOP == c_p->htop && E == c_p->stop);
+	     /* TODO: Add DTrace probe for this bad message situation? */
+	     UNLINK_MESSAGE(c_p, msgp);
+	     msgp->next = NULL;
+	     erts_cleanup_messages(msgp);
+	     goto loop_rec__;
+	 }
+	 SWAPIN;
      }
      PreFetch(1, next);
      r(0) = ERL_MESSAGE_TERM(msgp);
@@ -1892,8 +1892,7 @@ void process_main(void)
   */
  OpCase(remove_message): {
      BeamInstr *next;
-     ErlMessage* msgp;
-
+     ErtsMessage* msgp;
      PROCESS_MAIN_CHK_LOCKS(c_p);
 
      PreFetch(0, next);
@@ -1988,11 +1987,21 @@ void process_main(void)
      UNLINK_MESSAGE(c_p, msgp);
      JOIN_MESSAGE(c_p);
      CANCEL_TIMER(c_p);
-     free_message(msgp);
+
+     erts_save_message_in_proc(c_p, msgp);
+     c_p->flags &= ~F_DISABLE_GC;
+
+     if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) {
+	 /*
+	  * We want to GC soon but we leave a few
+	  * reductions giving the message some time
+	  * to turn into garbage.
+	  */
+	 ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
+     }
 
      ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
-
      NextPF(0, next);
  }
 
@@ -2001,9 +2010,22 @@ void process_main(void)
      * message didn't match), then jump to the loop_rec instruction.
      */
  OpCase(loop_rec_end_f): {
+
+     ASSERT(c_p->flags & F_DISABLE_GC);
+
      SET_I((BeamInstr *) Arg(0));
      SAVE_MESSAGE(c_p);
-     goto loop_rec__;
+     if (FCALLS > 0 || FCALLS > neg_o_reds) {
+	 FCALLS--;
+	 goto loop_rec__;
+     }
+
+     c_p->flags &= ~F_DISABLE_GC;
+     c_p->i = I;
+     SWAPOUT;
+     c_p->arity = 0;
+     c_p->current = NULL;
+     goto do_schedule;
  }
     /*
      * Prepare to wait for a message or a timeout, whichever occurs first.
@@ -2733,6 +2755,7 @@ do {						\
 	Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
 	Eterm result;
 	BeamInstr *next;
+	ErlHeapFragment *live_hf_end;
 
 	PRE_BIF_SWAPOUT(c_p);
 	c_p->fcalls = FCALLS - 1;
@@ -2742,17 +2765,18 @@ do {						\
 	PreFetch(1, next);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	live_hf_end = c_p->mbuf;
 	result = (*bf)(c_p, reg, I);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
+	if (ERTS_IS_GC_DESIRED(c_p)) {
 	    Uint arity = ((Export *)Arg(0))->code[2];
-	    result = erts_gc_after_bif_call(c_p, result, reg, arity);
+	    result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity);
 	    E = c_p->stop;
 	}
+	PROCESS_MAIN_CHK_LOCKS(c_p);
 	HTOP = HEAP_TOP(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
@@ -3414,9 +3438,6 @@ do {						\
 	 goto do_schedule;
      } else {
 	 ASSERT(!is_value(r(0)));
-	 if (c_p->mbuf) {
-	     erts_garbage_collect(c_p, 0, reg+1, 3);
-	 }
 	 SWAPIN;
 	 Goto(*I);
      }
@@ -3440,6 +3461,7 @@ do {						\
 	     * I[3]: Function pointer to dirty NIF
 	     */
 	    BifFunction vbf;
+	    ErlHeapFragment *live_hf_end;
 
 	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
@@ -3455,6 +3477,7 @@ do {						\
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
+		live_hf_end = c_p->mbuf;
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
 		    nif_bif_result = THE_NON_VALUE;
@@ -3497,6 +3520,7 @@ do {						\
 	    {
 		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+		live_hf_end = c_p->mbuf;
 		nif_bif_result = (*bf)(c_p, reg, I);
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
 		       is_non_value(nif_bif_result));
@@ -3509,9 +3533,17 @@ do {						\
 	apply_bif_or_nif_epilogue:
 	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_HOLE_CHECK(c_p);
-	    if (c_p->mbuf) {
-		nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result,
-						  reg, bif_nif_arity);
+	    /*
+	     * We want to test with ERTS_IS_GC_DESIRED(c_p) in order
+	     * to trigger gc due to binaries based on same conditions
+	     * regardless of how the bif is called. This change will
+	     * however be introduced in a separate commit in order to
+	     * easier identify why the characteristics changed.
+	     */
+	    if (c_p->stop - c_p->htop < c_p->mbuf_sz) {
+		nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end,
+							    nif_bif_result,
+							    reg, bif_nif_arity);
 	    }
 	    SWAPIN;  /* There might have been a garbage collection. */
 	    FCALLS = c_p->fcalls;
@@ -6340,13 +6372,6 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
         erts_factory_proc_init(&factory, p);
         res = erts_hashmap_from_array(&factory, thp, n/2, 0);
         erts_factory_close(&factory);
-        if (p->mbuf) {
-            Uint live = Arg(2);
-            reg[live] = res;
-            erts_garbage_collect(p, 0, reg, live+1);
-            res       = reg[live];
-            E = p->stop;
-        }
         return res;
     }
 
@@ -6412,13 +6437,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 	    hx = hashmap_make_hash(new_key);
 
 	    res = erts_hashmap_insert(p, hx, new_key, val, res,  0);
-	    if (p->mbuf) {
-		Uint live = Arg(3);
-		reg[live] = res;
-		erts_garbage_collect(p, 0, reg, live+1);
-		res       = reg[live];
-		E = p->stop;
-	    }
 
 	    new_p += 2;
 	}
@@ -6578,12 +6596,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     /* The expensive case, need to build a hashmap */
     if (n > MAP_SMALL_MAP_LIMIT) {
 	res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n);
-	if (p->mbuf) {
-	    Uint live = Arg(3);
-	    reg[live] = res;
-	    erts_garbage_collect(p, 0, reg, live+1);
-	    res       = reg[live];
-	}
     }
     return res;
 }
@@ -6639,14 +6651,6 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 		return res;
 	    }
 
-	    if (p->mbuf) {
-		Uint live = Arg(3);
-		reg[live] = res;
-		erts_garbage_collect(p, 0, reg, live+1);
-		res       = reg[live];
-		E = p->stop;
-	    }
-
 	    new_p += 2;
 	}
 	return res;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7a1a563be2..5db971b6af 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -902,7 +902,7 @@ static ErlHeapFragment* new_literal_fragment(Uint size)
     ErlHeapFragment* bp;
     bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE,
 					    ERTS_HEAP_FRAG_SIZE(size));
-    ERTS_INIT_HEAP_FRAG(bp, size);
+    ERTS_INIT_HEAP_FRAG(bp, size, size);
     return bp;
 }
 
@@ -1528,8 +1528,8 @@ read_literal_table(LoaderState* stp)
 	}
 
         if (heap_size > 0) {
-            erts_factory_message_init(&factory, NULL, NULL,
-                                      new_literal_fragment(heap_size));
+            erts_factory_heap_frag_init(&factory,
+					new_literal_fragment(heap_size));
 	    factory.alloc_type = ERTS_ALC_T_PREPARED_CODE;
             val = erts_decode_ext(&factory, &p);
 
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 5ec1840c7b..e4283ac945 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -72,7 +72,7 @@ BIF_RETTYPE spawn_3(BIF_ALIST_3)
     ErlSpawnOpts so;
     Eterm pid;
 
-    so.flags = 0;
+    so.flags = erts_default_spo_flags;
     pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so);
     if (is_non_value(pid)) {
 	BIF_ERROR(BIF_P, so.error_code);
@@ -589,7 +589,7 @@ erts_queue_monitor_message(Process *p,
     Eterm reason_copy, ref_copy, item_copy;
     Uint reason_size, ref_size, item_size, heap_size;
     ErlOffHeap *ohp;
-    ErlHeapFragment *bp;
+    ErtsMessage *msgp;
 
     reason_size = IS_CONST(reason) ? 0 : size_object(reason);
     item_size   = IS_CONST(item) ? 0 : size_object(item);
@@ -597,11 +597,8 @@ erts_queue_monitor_message(Process *p,
 
     heap_size = 6+reason_size+ref_size+item_size;
 
-    hp = erts_alloc_message_heap(heap_size,
-				 &bp,
-				 &ohp,
-				 p,
-				 p_locksp);
+    msgp = erts_alloc_message_heap(p, p_locksp, heap_size,
+				   &hp, &ohp);
 
     reason_copy = (IS_CONST(reason)
 		   ? reason
@@ -612,7 +609,7 @@ erts_queue_monitor_message(Process *p,
     ref_copy    = copy_struct(ref, ref_size, &hp, ohp);
 
     tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy);
-    erts_queue_message(p, p_locksp, bp, tup, NIL);
+    erts_queue_message(p, p_locksp, msgp, tup, NIL);
 }
 
 static BIF_RETTYPE
@@ -841,7 +838,7 @@ BIF_RETTYPE spawn_link_3(BIF_ALIST_3)
     ErlSpawnOpts so;
     Eterm pid;
 
-    so.flags = SPO_LINK;
+    so.flags = erts_default_spo_flags|SPO_LINK;
     pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so);
     if (is_non_value(pid)) {
 	BIF_ERROR(BIF_P, so.error_code);
@@ -878,7 +875,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
     /*
      * Store default values for options.
      */
-    so.flags          = SPO_USE_ARGS;
+    so.flags          = erts_default_spo_flags|SPO_USE_ARGS;
     so.min_heap_size  = H_MIN_SIZE;
     so.min_vheap_size = BIN_VH_MIN_SIZE;
     so.priority       = PRIORITY_NORMAL;
@@ -913,6 +910,13 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
 		    so.priority = PRIORITY_LOW;
 		else
 		    goto error;
+	    } else if (arg == am_off_heap_message_queue) {
+		if (val == am_true)
+		    so.flags |= SPO_OFF_HEAP_MSGQ;
+		else if (val == am_false)
+		    so.flags &= ~SPO_OFF_HEAP_MSGQ;
+		else
+		    goto error;
 	    } else if (arg == am_min_heap_size && is_small(val)) {
 		Sint min_heap_size = signed_val(val);
 		if (min_heap_size < 0) {
@@ -1691,6 +1695,17 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        }
        BIF_RET(old_value);
    }
+   else if (BIF_ARG_1 == am_off_heap_message_queue) {
+       int enable;
+       if (BIF_ARG_2 == am_true)
+	   enable = 1;
+       else if (BIF_ARG_2 == am_false)
+	   enable = 0;
+       else
+	   goto error;
+       old_value = erts_change_off_heap_message_queue_state(BIF_P, enable);
+       BIF_RET(old_value);
+   }
    else if (BIF_ARG_1 == am_sensitive) {
        Uint is_sensitive;
        if (BIF_ARG_2 == am_true) {
@@ -1931,7 +1946,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
     } else if (is_atom(to)) {
 	Eterm id = erts_whereis_name_to_id(p, to);
 
-	rp = erts_proc_lookup(id);
+	rp = erts_proc_lookup_raw(id);
 	if (rp) {
 	    if (IS_TRACED(p))
 		trace_send(p, to, msg);
@@ -4479,7 +4494,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	}
     } else if (BIF_ARG_1 == make_small(1)) {
 	int i, max;
-	ErlMessage* mp;
+	ErtsMessage* mp;
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_smp_thr_progress_block();
 
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index c6ed60376a..a62eddf36b 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -54,22 +54,24 @@ extern Export *erts_convert_time_unit_trap;
 	(p)->fcalls = -CONTEXT_REDS;		\
 } while(0)
 
-
-#define ERTS_VBUMP_ALL_REDS(p)						\
+#define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls)				\
 do {									\
     if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) {				\
-	if ((p)->fcalls > 0)						\
-	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (p)->fcalls;	\
-	(p)->fcalls = 0;						\
+	if ((fcalls) > 0)						\
+	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (fcalls);	\
+	(fcalls) = 0;							\
     }									\
     else {								\
-	if ((p)->fcalls > -CONTEXT_REDS)				\
+	if ((fcalls) > -CONTEXT_REDS)					\
 	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds			\
-		+= ((p)->fcalls - (-CONTEXT_REDS));			\
-	(p)->fcalls = -CONTEXT_REDS;					\
+		+= ((fcalls) - (-CONTEXT_REDS));			\
+	(fcalls) = -CONTEXT_REDS;					\
     }									\
 } while(0)
 
+#define ERTS_VBUMP_ALL_REDS(p) \
+    ERTS_VBUMP_ALL_REDS_INTERNAL((p), (p)->fcalls)
+
 #define BUMP_REDS(p, gc) do {			   \
      ASSERT(p);		 			   \
      ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\
@@ -110,10 +112,34 @@ do {									\
     }									\
 } while(0)
 
-#define ERTS_BIF_REDS_LEFT(p)						\
+#define ERTS_VBUMP_LEAVE_REDS_INTERNAL(P, Reds, FCalls)			\
+    do {								\
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) {			\
+	    int nreds__ = ((int)(Reds)) - CONTEXT_REDS;			\
+	    if ((FCalls) > nreds__) {					\
+		ERTS_PROC_GET_SCHDATA((P))->virtual_reds		\
+		    += (FCalls) - nreds__;				\
+		(FCalls) = nreds__;					\
+	    }								\
+	}								\
+	else {								\
+	    if ((FCalls) > (Reds)) {					\
+		ERTS_PROC_GET_SCHDATA((P))->virtual_reds		\
+		    += (FCalls) - (Reds);				\
+		(FCalls) = (Reds);					\
+	    }								\
+	}								\
+    } while (0)
+
+#define ERTS_VBUMP_LEAVE_REDS(P, Reds)					\
+    ERTS_VBUMP_LEAVE_REDS_INTERNAL(P, Reds, (P)->fcalls)
+
+#define ERTS_REDS_LEFT(p, FCalls)					\
   (ERTS_PROC_GET_SAVED_CALLS_BUF((p))					\
-   ? ((p)->fcalls > -CONTEXT_REDS ? ((p)->fcalls - (-CONTEXT_REDS)) : 0)\
-   : ((p)->fcalls > 0 ? (p)->fcalls : 0))
+   ? ((FCalls) > -CONTEXT_REDS ? ((FCalls) - (-CONTEXT_REDS)) : 0)	\
+   : ((FCalls) > 0 ? (FCalls) : 0))
+
+#define ERTS_BIF_REDS_LEFT(p) ERTS_REDS_LEFT(p, p->fcalls)
 
 #define BIF_RET2(x, gc) do {			\
     BUMP_REDS(BIF_P, (gc));			\
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index c7e7411935..aa5ec123a7 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -252,7 +252,7 @@ print_process_info(int to, void *to_arg, Process *p)
 
     /* display the message queue only if there is anything in it */
     if (!ERTS_IS_CRASH_DUMPING && p->msg.first != NULL && !garbing) {
-	ErlMessage* mp;
+	ErtsMessage* mp;
 	erts_print(to, to_arg, "Message queue: [");
 	for (mp = p->msg.first; mp; mp = mp->next)
 	    erts_print(to, to_arg, mp->next ? "%T," : "%T", ERL_MESSAGE_TERM(mp));
@@ -323,7 +323,7 @@ print_process_info(int to, void *to_arg, Process *p)
     erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop));
     erts_print(to, to_arg, "OldHeap unused: %bpu\n",
 	       (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) );
-    erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p));
+    erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0));
 
     if (garbing) {
 	print_garb_info(to, to_arg, p);
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index b185758b1d..f27c526413 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -279,7 +279,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    break;
 	case TAG_PRIMARY_LIST:
 	    objp = list_val(obj);
-	    if (in_area(objp,hstart,hsize)) {
+	    if (ErtsInArea(objp,hstart,hsize)) {
 		hp++;
 		break;
 	    }
@@ -318,7 +318,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    }
 	    
 	case TAG_PRIMARY_BOXED:
-	    if (in_area(boxed_val(obj),hstart,hsize)) {
+	    if (ErtsInArea(boxed_val(obj),hstart,hsize)) {
 		hp++;
 		break;
 	    }
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index efd5109269..bfddcadca3 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -373,10 +373,11 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
     ASSERT(lnk->type == LINK_NODE);
     if (is_internal_pid(lnk->pid)) {
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
-	rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks);
-	if (!rp) {
+	ErlOffHeap *ohp;
+	rp = erts_proc_lookup(lnk->pid);
+	if (!rp)
 	    goto done;
-	}
+	erts_smp_proc_lock(rp, rp_locks);
 	rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name);
 	if (rlnk != NULL) {
 	    ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE));
@@ -384,12 +385,14 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	}
 	n = ERTS_LINK_REFC(lnk);
 	for (i = 0; i < n; ++i) {
-	    ErlHeapFragment* bp;
-	    ErlOffHeap *ohp;
 	    Eterm tup;
-	    Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks);
+	    Eterm *hp;
+	    ErtsMessage *msgp;
+
+	    msgp = erts_alloc_message_heap(rp, &rp_locks,
+					   3, &hp, &ohp);
 	    tup = TUPLE2(hp, am_nodedown, name);
-	    erts_queue_message(rp, &rp_locks, bp, tup, NIL);
+	    erts_queue_message(rp, &rp_locks, msgp, tup, NIL);
 	}
 	erts_smp_proc_unlock(rp, rp_locks);
     }
@@ -1458,7 +1461,7 @@ int erts_net_message(Port *prt,
 		ErlOffHeap *ohp;
 		ASSERT(xsize);
 		heap_frag = erts_dist_ext_trailer(ede_copy);
-		ERTS_INIT_HEAP_FRAG(heap_frag, token_size);
+		ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size);
 		hp = heap_frag->mem;
 		ohp = &heap_frag->off_heap;
 		token = tuple[5];
@@ -1507,7 +1510,7 @@ int erts_net_message(Port *prt,
 		ErlOffHeap *ohp;
 		ASSERT(xsize);
 		heap_frag = erts_dist_ext_trailer(ede_copy);
-		ERTS_INIT_HEAP_FRAG(heap_frag, token_size);
+		ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size);
 		hp = heap_frag->mem;
 		ohp = &heap_frag->off_heap;
 		token = tuple[4];
@@ -3267,11 +3270,16 @@ send_nodes_mon_msg(Process *rp,
 		   Uint sz)
 {
     Eterm msg;
-    ErlHeapFragment* bp;
+    Eterm *hp;
+    ErtsMessage *mp;
     ErlOffHeap *ohp;
-    Eterm *hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, rp_locksp);
 #ifdef DEBUG
-    Eterm *hend = hp + sz;
+    Eterm *hend;
+#endif
+
+    mp = erts_alloc_message_heap(rp, rp_locksp, sz, &hp, &ohp);
+#ifdef DEBUG
+    hend = hp + sz;
 #endif
 
     if (!nmp->opts) {
@@ -3317,7 +3325,7 @@ send_nodes_mon_msg(Process *rp,
     }
 
     ASSERT(hend == hp);
-    erts_queue_message(rp, rp_locksp, bp, msg, NIL);
+    erts_queue_message(rp, rp_locksp, mp, msg, NIL);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 3e300f88ea..019aa0f16c 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -582,7 +582,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
 	= sizeof(ErtsDrvSelectDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
-	= sizeof(ErlMessage);
+	= sizeof(ErtsMessageRef);
 #ifdef ERTS_SMP
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
 	= sizeof(ErtsThrQElement_t);
@@ -2916,12 +2916,12 @@ reply_alloc_info(void *vair)
     int global_instances = air->req_sched == sched_id;
     ErtsProcLocks rp_locks;
     Process *rp = air->proc;
-    Eterm ref_copy = NIL, ai_list, msg;
-    Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL;
+    Eterm ref_copy = NIL, ai_list, msg = NIL;
+    Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL;
     Eterm **hpp;
     Uint sz, *szp;
     ErlOffHeap *ohp = NULL;
-    ErlHeapFragment *bp = NULL;
+    ErtsMessage *mp = NULL;
     struct erts_mmap_info_struct emis;
     int i;
     Eterm (*info_func)(Allctr_t *,
@@ -3123,20 +3123,17 @@ reply_alloc_info(void *vair)
 	if (hpp)
 	    break;
 
-	hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+	mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
 	hp_start = hp;
 	hp_end = hp + sz;
 	szp = NULL;
 	hpp = &hp;
     }
-    if (bp)
-	bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1);
-    else {
-	ASSERT(hp);
-	HRelease(rp, hp_end, hp);	    
-    }
 
-    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+    if (hp != hp_end)
+	erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1);
+
+    erts_queue_message(rp, &rp_locks, mp, msg, NIL);
 
     if (air->req_sched == sched_id)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 9da9c823f7..14e80960f5 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -44,9 +44,11 @@
 #if ERTS_CAN_INLINE && ERTS_ALC_WANT_INLINE
 #  define ERTS_ALC_DO_INLINE 1
 #  define ERTS_ALC_INLINE static ERTS_INLINE
+#  define ERTS_ALC_FORCE_INLINE static ERTS_FORCE_INLINE
 #else
 #  define ERTS_ALC_DO_INLINE 0
 #  define ERTS_ALC_INLINE
+#  define ERTS_ALC_FORCE_INLINE
 #endif
 
 #define ERTS_ALC_NO_FIXED_SIZES \
@@ -293,7 +295,7 @@ int erts_is_allctr_wrapper_prelocked(void)
 
 #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
 
-ERTS_ALC_INLINE
+ERTS_ALC_FORCE_INLINE
 int erts_is_in_literal_range(void* ptr)
 {
 #if defined(ARCH_32)
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 7d519c1be4..75b4913012 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -152,6 +152,8 @@ type	OLD_HEAP	EHEAP		PROCESSES	old_heap
 type	HEAP_FRAG	EHEAP		PROCESSES	heap_frag
 type	TMP_HEAP	TEMPORARY	PROCESSES	tmp_heap
 type	MSG_REF		FIXED_SIZE	PROCESSES	msg_ref
+type	MSG		EHEAP		PROCESSES	message
+type	MSGQ_CHNG	SHORT_LIVED	PROCESSES	messages_queue_change
 type	MSG_ROOTS	TEMPORARY	PROCESSES	msg_roots
 type	ROOTSET		TEMPORARY	PROCESSES	root_set
 type	LOADER_TMP	TEMPORARY	CODE		loader_tmp
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 28bec6325c..2b1d875bfe 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1707,18 +1707,19 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
     Eterm mess;
     Eterm r;
     Eterm *hp;
-    ErlHeapFragment *bp;
-    ErlOffHeap *ohp;
+    ErtsMessage *mp;
     ErtsProcLocks rp_locks = 0;
+    ErlOffHeap *ohp;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     assert_drv_list_rwlocked();
     if (errcode != 0) {
 	int need = load_error_need(errcode);
 	Eterm e;
-	hp = erts_alloc_message_heap(6 /* tuple */ + 3 /* Error tuple */ + 
-				     REF_THING_SIZE + need, &bp, &ohp, 
-				     proc, &rp_locks);
+	mp = erts_alloc_message_heap(proc, &rp_locks,
+				     (6 /* tuple */ + 3 /* Error tuple */ + 
+				      REF_THING_SIZE + need),
+				     &hp, &ohp);
 	r = copy_ref(ref,hp);
 	hp += REF_THING_SIZE;
 	e = build_load_error_hp(hp, errcode);
@@ -1727,12 +1728,14 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	hp += 3;
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,mess);
     } else {	
-	hp = erts_alloc_message_heap(6 /* tuple */ + REF_THING_SIZE, &bp, &ohp, proc, &rp_locks);
+	mp = erts_alloc_message_heap(proc, &rp_locks,
+				     6 /* tuple */ + REF_THING_SIZE,
+				     &hp, &ohp);
 	r = copy_ref(ref,hp);
 	hp += REF_THING_SIZE;
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
     }
-    erts_queue_message(proc, &rp_locks, bp, mess, am_undefined);
+    erts_queue_message(proc, &rp_locks, mp, mess, am_undefined);
     erts_smp_proc_unlock(proc, rp_locks);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 }
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a684c81445..1eb106a551 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -589,6 +589,7 @@ static Eterm pi_args[] = {
     am_min_bin_vheap_size,
     am_current_location,
     am_current_stacktrace,
+    am_off_heap_message_queue
 };
 
 #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
@@ -636,6 +637,7 @@ pi_arg2ix(Eterm arg)
     case am_min_bin_vheap_size:			return 28;
     case am_current_location:			return 29;
     case am_current_stacktrace:			return 30;
+    case am_off_heap_message_queue:		return 31;
     default:					return -1;
     }
 }
@@ -718,9 +720,10 @@ pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks)
 
 
 
-BIF_RETTYPE
+static BIF_RETTYPE
 process_info_aux(Process *BIF_P,
 		 Process *rp,
+		 ErtsProcLocks rp_locks,
 		 Eterm rpid,
 		 Eterm item,
 		 int always_wrap);
@@ -811,10 +814,31 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
 	*fail_type = ERTS_PI_FAIL_TYPE_AWAIT_EXIT;
 	goto done;
     }
-    else if (!(locks & ERTS_PROC_LOCK_STATUS)) {
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+    else {
+	ErtsProcLocks unlock_locks = 0;
+
+	if (c_p == rp)
+	    locks |= ERTS_PROC_LOCK_MAIN;
+
+	if (!(locks & ERTS_PROC_LOCK_STATUS))
+	    unlock_locks |= ERTS_PROC_LOCK_STATUS;
+
+	if (locks & ERTS_PROC_LOCK_MSGQ) {
+	    /*
+	     * Move in queue into private queue and
+	     * release msgq lock, enabling others to
+	     * send messages to the process while it
+	     * is being inspected...
+	     */
+	    ASSERT(locks & ERTS_PROC_LOCK_MAIN);
+	    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
+	    locks &= ~ERTS_PROC_LOCK_MSGQ;
+	    unlock_locks |= ERTS_PROC_LOCK_MSGQ;
+	}
+
+	if (unlock_locks)
+	    erts_smp_proc_unlock(rp, unlock_locks);
     }
-	
 
     /*
      * We always handle 'messages' first if it should be part
@@ -826,7 +850,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
     if (want_messages) {
 	ix = pi_arg2ix(am_messages);
 	ASSERT(part_res[ix] == THE_NON_VALUE);
-	part_res[ix] = process_info_aux(c_p, rp, pid, am_messages, always_wrap);
+	part_res[ix] = process_info_aux(c_p, rp, locks, pid, am_messages, always_wrap);
 	ASSERT(part_res[ix] != THE_NON_VALUE);
     }
 
@@ -834,7 +858,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
 	ix = res_elem_ix[res_elem_ix_ix];
 	if (part_res[ix] == THE_NON_VALUE) {
 	    arg = pi_ix2arg(ix);
-	    part_res[ix] = process_info_aux(c_p, rp, pid, arg, always_wrap);
+	    part_res[ix] = process_info_aux(c_p, rp, locks, pid, arg, always_wrap);
 	    ASSERT(part_res[ix] != THE_NON_VALUE);
 	}
     }
@@ -965,9 +989,31 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
 	ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
     }
     else {
+	ErtsProcLocks unlock_locks = 0;
+
+	if (BIF_P == rp)
+	    info_locks |= ERTS_PROC_LOCK_MAIN;
+
 	if (!(info_locks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-	res = process_info_aux(BIF_P, rp, pid, BIF_ARG_2, 0);
+	    unlock_locks |= ERTS_PROC_LOCK_STATUS;
+
+	if (info_locks & ERTS_PROC_LOCK_MSGQ) {
+	    /*
+	     * Move in queue into private queue and
+	     * release msgq lock, enabling others to
+	     * send messages to the process while it
+	     * is being inspected...
+	     */
+	    ASSERT(info_locks & ERTS_PROC_LOCK_MAIN);
+	    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
+	    info_locks &= ~ERTS_PROC_LOCK_MSGQ;
+	    unlock_locks |= ERTS_PROC_LOCK_MSGQ;
+	}
+
+	if (unlock_locks)
+	    erts_smp_proc_unlock(rp, unlock_locks);
+
+	res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2, 0);
     }
     ASSERT(is_value(res));
 
@@ -985,6 +1031,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
 Eterm
 process_info_aux(Process *BIF_P,
 		 Process *rp,
+		 ErtsProcLocks rp_locks,
 		 Eterm rpid,
 		 Eterm item,
 		 int always_wrap)
@@ -1056,171 +1103,55 @@ process_info_aux(Process *BIF_P,
 	break;
 
     case am_messages: {
-	ErlMessage* mp;
-	int n;
 
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
-	n = rp->msg.len;
-
-	if (n == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) {
+	if (rp->msg.len == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) {
 	    hp = HAlloc(BIF_P, 3);
 	} else {
-	    int remove_bad_messages = 0;
-	    struct {
-		Uint copy_struct_size;
-		ErlMessage* msgp;
-	    } *mq = erts_alloc(ERTS_ALC_T_TMP, n*sizeof(*mq));
-	    Sint i = 0;
-	    Uint heap_need = 3;
+	    ErtsMessageInfo *mip;
+	    Sint i;
+	    Uint heap_need;
+#ifdef DEBUG
 	    Eterm *hp_end;
+#endif
 
-	    for (mp = rp->msg.first; mp; mp = mp->next) {
-		heap_need += 2;
-		mq[i].msgp = mp;
-		if (rp != BIF_P) {
-		    Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
-		    if (is_value(msg)) {
-			mq[i].copy_struct_size = (is_immed(msg)? 0 :
-						  size_object(msg));
-		    }
-		    else if (mq[i].msgp->data.attached) {
-			mq[i].copy_struct_size
-			    = erts_msg_attached_data_size(mq[i].msgp);
-		    }
-		    else {
-			/* Bad distribution message; ignore */
-			remove_bad_messages = 1;
-			mq[i].copy_struct_size = 0;
-		    }
-		    heap_need += mq[i].copy_struct_size;
-		}
-		else {
-		    mq[i].copy_struct_size = mp->data.attached ?
-                        erts_msg_attached_data_size(mp) : 0;
-		}
-		i++;
-	    }
+	    mip = erts_alloc(ERTS_ALC_T_TMP,
+			     rp->msg.len*sizeof(ErtsMessageInfo));
 
-            if (rp != BIF_P) {
-                hp = HAlloc(BIF_P, heap_need);
-                hp_end = hp + heap_need;
-                ASSERT(i == n);
-                for (i--; i >= 0; i--) {
-                    Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
-		    if (is_value(msg)) {
-			if (mq[i].copy_struct_size)
-			    msg = copy_struct(msg,
-					      mq[i].copy_struct_size,
-					      &hp,
-					      &MSO(BIF_P));
-		    }
-		    else if (mq[i].msgp->data.attached) {
-			ErlHeapFragment *hfp;
-			/*
-			 * Decode it into a message buffer and attach it
-			 * to the message instead of the attached external
-			 * term.
-			 *
-			 * Note that we may not pass a process pointer
-			 * to erts_msg_distext2heap(), since it would then
-			 * try to alter locks on that process.
-			 */
-			msg = erts_msg_distext2heap(
-			    NULL, NULL, &hfp, &ERL_MESSAGE_TOKEN(mq[i].msgp),
-			    mq[i].msgp->data.dist_ext);
-
-			ERL_MESSAGE_TERM(mq[i].msgp) = msg;
-			mq[i].msgp->data.heap_frag = hfp;
-
-			if (is_non_value(msg)) {
-			    ASSERT(!mq[i].msgp->data.heap_frag);
-			    /* Bad distribution message; ignore */
-			    remove_bad_messages = 1;
-			    continue;
-			}
-			else {
-			    /* Make our copy of the message */
-			    ASSERT(size_object(msg) == erts_used_frag_sz(hfp));
-			    msg = copy_struct(msg,
-					      erts_used_frag_sz(hfp),
-					      &hp,
-					      &MSO(BIF_P));
-			}
-		    }
-		    else {
-			/* Bad distribution message; ignore */
-			remove_bad_messages = 1;
-			continue;
-		    }
-                    res = CONS(hp, msg, res);
-                    hp += 2;
-		}
-                HRelease(BIF_P, hp_end, hp+3);
-            }
-	    else {
-                for (i--; i >= 0; i--) {
-                    ErtsHeapFactory factory;
-                    Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
-
-                    erts_factory_proc_prealloc_init(&factory, BIF_P,
-                                                    mq[i].copy_struct_size+2);
-		    if (mq[i].msgp->data.attached) {
-			/* Decode it on the heap */
-			erts_move_msg_attached_data_to_heap(&factory,
-							    mq[i].msgp);
-			msg = ERL_MESSAGE_TERM(mq[i].msgp);
-			ASSERT(!mq[i].msgp->data.attached);
-                    }
-                    if (is_value(msg)) {
-                        hp = erts_produce_heap(&factory, 2, 0);
-                        res = CONS(hp, msg, res);
-                    }
-                    else {
-                        /* Bad distribution message; ignore */
-                        remove_bad_messages = 1;
-                        continue;
-                    }
-                    erts_factory_close(&factory);
-		}
-                hp = HAlloc(BIF_P, 3);
-	    }
-	    erts_free(ERTS_ALC_T_TMP, mq);
-	    if (remove_bad_messages) {
-		ErlMessage **mpp;
-		/*
-		 * We need to remove bad distribution messages from
-		 * the queue, so that the value returned for
-		 * 'message_queue_len' is consistent with the value
-		 * returned for 'messages'.
-		 */
-		mpp = &rp->msg.first;
-		mp = rp->msg.first;
-		while (mp) {
-		    if (is_value(ERL_MESSAGE_TERM(mp))) {
-			mpp = &mp->next;
-			mp = mp->next;
-		    }
-		    else {
-			ErlMessage* bad_mp = mp;
-			ASSERT(!mp->data.attached);
-			if (rp->msg.save == &mp->next)
-			    rp->msg.save = mpp;
-			if (rp->msg.last == &mp->next)
-			    rp->msg.last = mpp;
-			*mpp = mp->next;
-			mp = mp->next;
-			rp->msg.len--;
-			free_message(bad_mp);
-		    }
-		}
+	    /*
+	     * Note that message queue may shrink when calling
+	     * erts_prep_msgq_for_inspection() since it removes
+	     * corrupt distribution messages.
+	     */
+	    heap_need = erts_prep_msgq_for_inspection(BIF_P, rp, rp_locks, mip);
+	    heap_need += 3; /* top 2-tuple */
+	    heap_need += rp->msg.len*2; /* Cons cells */
+
+	    hp = HAlloc(BIF_P, heap_need); /* heap_need is exact */
+#ifdef DEBUG
+	    hp_end = hp + heap_need;
+#endif
+
+	    /* Build list of messages... */
+	    for (i = rp->msg.len - 1, res = NIL; i >= 0; i--) {
+		Eterm msg = ERL_MESSAGE_TERM(mip[i].msgp);
+		Uint sz = mip[i].size;
+
+		if (sz != 0)
+		    msg = copy_struct(msg, sz, &hp, &BIF_P->off_heap);
+
+		res = CONS(hp, msg, res);
+		hp += 2;
 	    }
+
+	    ASSERT(hp_end == hp + 3);
+
+	    erts_free(ERTS_ALC_T_TMP, mip);
 	}
 	break;
     }
 
     case am_message_queue_len:
 	hp = HAlloc(BIF_P, 3);
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
 	res = make_small(rp->msg.len);
 	break;
 
@@ -1408,7 +1339,7 @@ process_info_aux(Process *BIF_P,
     }
 
     case am_total_heap_size: {
-	ErlMessage *mp;
+	ErtsMessage *mp;
 	Uint total_heap_size;
 	Uint hsz = 3;
 
@@ -1418,8 +1349,6 @@ process_info_aux(Process *BIF_P,
 
 	total_heap_size += rp->mbuf_sz;
 
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
-
 	for (mp = rp->msg.first; mp; mp = mp->next)
 	    if (mp->data.attached)
 		total_heap_size += erts_msg_attached_data_size(mp);
@@ -1441,7 +1370,7 @@ process_info_aux(Process *BIF_P,
 
     case am_memory: { /* Memory consumed in bytes */
 	Uint hsz = 3;
-	Uint size = erts_process_memory(rp);
+	Uint size = erts_process_memory(rp, 0);
 	(void) erts_bld_uint(NULL, &hsz, size);
 	hp = HAlloc(BIF_P, hsz);
 	res = erts_bld_uint(&hp, NULL, size);
@@ -1567,6 +1496,11 @@ process_info_aux(Process *BIF_P,
 	break;
     }
 
+    case am_off_heap_message_queue:
+	res = BIF_P->flags & F_OFF_HEAP_MSGQ ? am_true : am_false;
+	hp = HAlloc(BIF_P, 3);
+	break;
+
     default:
 	return THE_NON_VALUE; /* will produce badarg */
 
@@ -2728,6 +2662,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(am_true);
     }
 #endif
+    else if (BIF_ARG_1 == am_off_heap_message_queue) {
+	BIF_RET(erts_default_spo_flags & SPO_OFF_HEAP_MSGQ
+		? am_true : am_false);
+    }
     else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) {
 	Uint  sz;
 	Eterm res = NIL, tup, text;
@@ -3865,9 +3803,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		BIF_RET(am_false);
 	    }
 	    else {
-		FLAGS(rp) |= F_FORCE_GC;
-		if (BIF_P != rp)
-		    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+		ERTS_FORCE_GC(BIF_P);
 		BIF_RET(am_true);
 	    }
 	}
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 434b6c6d7a..6652ef9bb3 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -309,6 +309,8 @@ void erts_check_for_holes(Process* p)
     p->last_htop = HEAP_TOP(p);
 
     for (hf = MBUF(p); hf != 0; hf = hf->next) {
+	if (hf == p->heap_hfrag)
+	    continue;
 	if (hf == p->last_mbuf) {
 	    break;
 	}
@@ -399,7 +401,7 @@ void verify_process(Process *p)
         erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); }
 
 
-    ErlMessage* mp = p->msg.first;
+    ErtsMessage* mp = p->msg.first;
 
     VERBOSE(DEBUG_MEMORY,("Verify process: %T...\n",p->common.id));
 
@@ -528,7 +530,7 @@ static void print_process_memory(Process *p)
                 PTR_SIZE, "PCB", dashes, dashes, dashes, dashes);
 
     if (p->msg.first != NULL) {
-        ErlMessage* mp;
+        ErtsMessage* mp;
         erts_printf("  Message Queue:\n");
         mp = p->msg.first;
         while (mp != NULL) {
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 2d7b7cafa4..6a52e1a890 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,8 +42,6 @@
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
 
-#define ERTS_CONTINUOUS_NEW_HEAP
-
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10
@@ -60,58 +58,6 @@
 #  define ERTS_GC_ASSERT(B) ((void) 1)
 #endif
 
-#ifdef ERTS_CONTINUOUS_NEW_HEAP
-#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    ErtsInArea((Ptr), (NhPtr), (NhSz))
-#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    (!ErtsInArea((Ptr), (NhPtr), (NhSz))				\
-     && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
-
-#ifdef ERTS_GC_DEBUG
-#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
-     ? (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr))			\
-		       && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1)	\
-     : (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))			\
-		       || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
-#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
-     ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))), 1)		\
-     : (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr))), 0))
-#endif
-
-#else
-
-#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    (!erts_is_literal((TPtr), (Ptr)) && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
-#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    (erts_is_literal((TPtr), (Ptr)))
-
-#ifdef ERTS_GC_DEBUG
-#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
-     ? (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz))), 1)		\
-     : (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz))), 0))
-#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
-     ? (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz))		\
-		       && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1)	\
-     : (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz))		\
-		       || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
-#endif
-
-#endif
-
-#ifndef ERTS_IS_NEW_HEAP_PTR
-#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
-#endif
-
-#ifndef ERTS_IS_LITERAL_PTR
-#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
-    ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
-#endif
-
 /*
  * Returns number of elements in an array.
  */
@@ -132,10 +78,10 @@
 #define ErtsGcQuickSanityCheck(P)					\
 do {									\
     ASSERT((P)->heap < (P)->hend);					\
-    ASSERT((P)->heap_sz == (P)->hend - (P)->heap);			\
+    ASSERT((p)->abandoned_heap || (P)->heap_sz == (P)->hend - (P)->heap); \
     ASSERT((P)->heap <= (P)->htop && (P)->htop <= (P)->hend);		\
     ASSERT((P)->heap <= (P)->stop && (P)->stop <= (P)->hend);		\
-    ASSERT((P)->heap <= (P)->high_water && (P)->high_water <= (P)->hend);\
+    ASSERT((p)->abandoned_heap || ((P)->heap <= (P)->high_water && (P)->high_water <= (P)->hend)); \
     OverRunCheck((P));							\
 } while (0)
 #else
@@ -163,31 +109,32 @@ typedef struct {
 
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
-static Uint combined_message_size(Process* p);
 static void remove_message_buffers(Process* p);
 static Eterm *full_sweep_heaps(Process *p,
 			       int hibernate,
 			       Eterm *n_heap, Eterm* n_htop,
-			       char *h, Uint h_size,
 			       char *oh, Uint oh_size,
 			       Eterm *objv, int nobj);
-static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
-static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
-static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
+static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
+			   int need, Eterm* objv, int nobj);
+static int major_collection(Process* p, ErlHeapFragment *live_hf_end,
+			    int need, Eterm* objv, int nobj, Uint *recl);
+static int minor_collection(Process* p, ErlHeapFragment *live_hf_end,
+			    int need, Eterm* objv, int nobj, Uint *recl);
+static void do_minor(Process *p, ErlHeapFragment *live_hf_end,
+		     char *mature, Uint mature_size,
+		     Uint new_sz, Eterm* objv, int nobj);
 static Eterm *sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
-			     char* new_heap, Uint new_heap_size,
 			     char* old_heap, Uint old_heap_size);
 static Eterm *sweep_heaps(Eterm *n_hp, Eterm *n_htop,
-			  char* new_heap, Uint new_heap_size,
 			  char* old_heap, Uint old_heap_size);
 static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop,
-				    char* new_heap, Uint new_heap_size,
-				    char* old_heap, Uint old_heap_size,
-				    char* src, Uint src_size);
+				 char* old_heap, Uint old_heap_size,
+				 char* src, Uint src_size);
 static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 					 char* src, Uint src_size);
-static Eterm* collect_heap_frags(Process* p, Eterm* heap,
-				 Eterm* htop, Eterm* objv, int nobj);
+static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
+				      Eterm* heap, Eterm* htop, Eterm* objv, int nobj);
 static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
@@ -204,7 +151,6 @@ static void init_gc_info(ErtsGCInfo *gcip);
 #ifdef HARDDEBUG
 static void disallow_heap_frag_ref_in_heap(Process* p);
 static void disallow_heap_frag_ref_in_old_heap(Process* p);
-static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj);
 #endif
 
 #if defined(ARCH_64)
@@ -411,10 +357,19 @@ erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high)
 #undef ptr_within
 
 Eterm
-erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity)
+erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
+			   Eterm result, Eterm* regs, Uint arity)
 {
     int cost;
 
+    if (p->flags & F_HIBERNATE_SCHED) {
+	/*
+	 * We just hibernated. We do *not* want to mess
+	 * up the hibernation by an ordinary GC...
+	 */
+	return result;
+    }
+
     if (is_non_value(result)) {
 	if (p->freason == TRAP) {
 	  #if HIPE
@@ -422,21 +377,28 @@ erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity)
 		regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array;
 	    }
 	  #endif
-	    cost = erts_garbage_collect(p, 0, regs, p->arity);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity);
 	} else {
-	    cost = erts_garbage_collect(p, 0, regs, arity);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, arity);
 	}
     } else {
 	Eterm val[1];
 
 	val[0] = result;
-	cost = erts_garbage_collect(p, 0, val, 1);
+	cost = garbage_collect(p, live_hf_end, 0, val, 1);
 	result = val[0];
     }
     BUMP_REDS(p, cost);
     return result;
 }
 
+Eterm
+erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity)
+{
+    return erts_gc_after_bif_call_lhf(p, ERTS_INVALID_HFRAG_PTR,
+				      result, regs, arity);
+}
+
 static ERTS_INLINE void reset_active_writer(Process *p)
 {
     struct erl_off_heap_header* ptr;
@@ -450,6 +412,117 @@ static ERTS_INLINE void reset_active_writer(Process *p)
     }
 }
 
+#define ERTS_DELAY_GC_EXTRA_FREE 40
+
+static int
+delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
+{
+    ErlHeapFragment *hfrag;
+    Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
+    Eterm *stop, *hend;
+    Uint hsz, ssz;
+
+    ERTS_HOLE_CHECK(p);
+
+    if (p->live_hf_end == ERTS_INVALID_HFRAG_PTR)
+	p->live_hf_end = live_hf_end;
+
+    if (need == 0)
+	return 1;
+
+    /*
+     * Satisfy need in a heap fragment...
+     */
+    ASSERT(need > 0);
+
+    orig_heap = p->heap;
+    orig_hend = p->hend;
+    orig_htop = p->htop;
+    orig_stop = p->stop;
+
+    ssz = orig_hend - orig_stop;
+    hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE;
+
+    hfrag = new_message_buffer(hsz);
+    hfrag->next = p->mbuf;
+    p->mbuf = hfrag;
+    p->mbuf_sz += hsz;
+    p->heap = p->htop = &hfrag->mem[0];
+    p->hend = hend = &hfrag->mem[hsz];
+    p->stop = stop = hend - ssz;
+    sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm));
+
+    if (p->abandoned_heap) {
+	/* Active heap already in a fragment; adjust it... */
+	ErlHeapFragment *hfrag = ((ErlHeapFragment *)
+				  (((char *) orig_heap)
+				   - offsetof(ErlHeapFragment, mem)));
+	Uint unused = orig_hend - orig_htop;
+	ASSERT(hfrag->used_size == hfrag->alloc_size);
+	ASSERT(hfrag->used_size >= unused);
+	hfrag->used_size -= unused;
+	p->mbuf_sz -= unused;
+    }
+    else {
+	/* Do not leave a hole in the abandoned heap... */
+	if (orig_htop < orig_hend) {
+	    *orig_htop = make_pos_bignum_header(orig_hend-orig_htop-1);
+	    if (orig_htop + 1 < orig_hend) {
+		orig_hend[-1] = (Uint) (orig_htop - orig_heap);
+		p->flags |= F_ABANDONED_HEAP_USE;
+	    }
+	}
+	p->abandoned_heap = orig_heap;
+    }
+
+#ifdef CHECK_FOR_HOLES
+    p->last_htop = p->htop;
+    p->heap_hfrag = hfrag;
+#endif
+
+    /* Make sure that we do a proper GC as soon as possible... */
+    p->flags |= F_FORCE_GC;
+    return CONTEXT_REDS;
+}
+
+static ERTS_FORCE_INLINE Uint
+young_gen_usage(Process *p)
+{
+    Uint hsz;
+    Eterm *aheap;
+
+    hsz = p->mbuf_sz;
+    aheap = p->abandoned_heap;
+    if (!aheap)
+	hsz += p->htop - p->heap;
+    else {
+	/* used in orig heap */
+	if (p->flags & F_ABANDONED_HEAP_USE)
+	    hsz += aheap[p->heap_sz-1];
+	else
+	    hsz += p->heap_sz;
+	/* Remove unused part in latest fragment */
+	hsz -= p->hend - p->htop;
+    }
+    return hsz;
+}
+
+#define ERTS_GET_ORIG_HEAP(Proc, Heap, HTop)			\
+    do {							\
+	Eterm *aheap__ = (Proc)->abandoned_heap;		\
+	if (!aheap__) {						\
+	    (Heap) = (Proc)->heap;				\
+	    (HTop) = (Proc)->htop;				\
+	}							\
+	else {							\
+	    (Heap) = aheap__;					\
+	    if ((Proc)->flags & F_ABANDONED_HEAP_USE)		\
+		(HTop) = aheap__ + aheap__[(Proc)->heap_sz-1];	\
+	    else						\
+		(HTop) = aheap__ + (Proc)->heap_sz;		\
+	}							\
+    } while (0)
+
 /*
  * Garbage collect a process.
  *
@@ -458,8 +531,9 @@ static ERTS_INLINE void reset_active_writer(Process *p)
  * objv: Array of terms to add to rootset; that is to preserve.
  * nobj: Number of objects in objv.
  */
-int
-erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+static int
+garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
+		int need, Eterm* objv, int nobj)
 {
     Uint reclaimed_now = 0;
     int done = 0;
@@ -469,10 +543,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
-    if (p->flags & F_DISABLE_GC) {
-	ASSERT(need == 0);
-	return 1;
-    }
+    if (p->flags & F_DISABLE_GC)
+	return delay_garbage_collection(p, live_hf_end, need);
+
+    if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
+	live_hf_end = p->live_hf_end;
 
     esdp = erts_get_scheduler_data();
 
@@ -480,7 +555,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
         trace_gc(p, am_gc_start);
     }
 
-    (void) erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
+    erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     if (erts_system_monitor_long_gc != 0)
 	start_time = erts_get_monotonic_time(esdp);
 
@@ -505,11 +580,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     while (!done) {
 	if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) {
 	    DTRACE2(gc_major_start, pidbuf, need);
-	    done = major_collection(p, need, objv, nobj, &reclaimed_now);
+	    done = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
 	    DTRACE2(gc_major_end, pidbuf, reclaimed_now);
 	} else {
 	    DTRACE2(gc_minor_start, pidbuf, need);
-	    done = minor_collection(p, need, objv, nobj, &reclaimed_now);
+	    done = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
 	    DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
 	}
     }
@@ -551,6 +626,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     esdp->gc_info.reclaimed += reclaimed_now;
     
     FLAGS(p) &= ~F_FORCE_GC;
+    p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
 
 #ifdef CHECK_FOR_HOLES
     /*
@@ -583,6 +659,12 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     }
 }
 
+int
+erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+{
+    return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+}
+
 /*
  * Place all living data on a the new heap; deallocate any old heap.
  * Meant to be used by hibernate/3.
@@ -606,16 +688,16 @@ erts_garbage_collect_hibernate(Process* p)
      */
     erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     ErtsGcQuickSanityCheck(p);
-    ASSERT(p->mbuf_sz == 0);
-    ASSERT(p->mbuf == 0);
+    ASSERT(p->mbuf == NULL);
     ASSERT(p->stop == p->hend);	/* Stack must be empty. */
+    ASSERT(!p->abandoned_heap);
 
     /*
      * Do it.
      */
 
 
-    heap_size = p->heap_sz + (p->old_htop - p->old_heap);
+    heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz;
     heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP,
 				    sizeof(Eterm)*heap_size);
     htop = heap;
@@ -624,8 +706,6 @@ erts_garbage_collect_hibernate(Process* p)
 			    1,
 			    heap,
 			    htop,
-			    (char *) p->heap,
-			    (char *) p->htop - (char *) p->heap,
 			    (char *) p->old_heap,
 			    (char *) p->old_htop - (char *) p->old_heap,
 			    p->arg_reg,
@@ -644,6 +724,7 @@ erts_garbage_collect_hibernate(Process* p)
     }
 
     FLAGS(p) &= ~F_FORCE_GC;
+    p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
 
     /*
      * Move the heap to its final destination.
@@ -663,6 +744,8 @@ erts_garbage_collect_hibernate(Process* p)
     sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm));
     ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm));
 
+    remove_message_buffers(p);
+
     p->stop = p->hend = heap + heap_size;
 
     offs = heap - p->heap;
@@ -808,7 +891,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 
     old_htop = sweep_literals_to_old_heap(p->heap, p->htop, old_htop, area, area_size);
     old_htop = sweep_literal_area(p->old_heap, old_htop,
-				  (char *) p->heap, sizeof(Eterm)*p->heap_sz,
 				  (char *) p->old_heap, sizeof(Eterm)*old_heap_size,
 				  area, area_size);
     ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend);
@@ -869,15 +951,18 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 }
 
 static int
-minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
+minor_collection(Process* p, ErlHeapFragment *live_hf_end,
+		 int need, Eterm* objv, int nobj, Uint *recl)
 {
-    Uint mature = HIGH_WATER(p) - HEAP_START(p);
+    Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap;
+    Uint mature_size = p->high_water - mature;
+    Uint size_before = young_gen_usage(p);
 
     /*
      * Allocate an old heap if we don't have one and if we'll need one.
      */
 
-    if (OLD_HEAP(p) == NULL && mature != 0) {
+    if (OLD_HEAP(p) == NULL && mature_size != 0) {
         Eterm* n_old;
 
         /* Note: We choose a larger heap size than strictly needed,
@@ -885,7 +970,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
          * This improved Estone by more than 1200 estones on my computer
          * (Ultra Sparc 10).
          */
-        Uint new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1);
+        Uint new_sz = erts_next_heap_size(size_before, 1);
 
         /* Create new, empty old_heap */
         n_old = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP,
@@ -901,41 +986,25 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      */
 
     if (OLD_HEAP(p) &&
-	    ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
-	    ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
-	    ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
-	ErlMessage *msgp;
-	Uint size_after;
-	Uint need_after;
-	const Uint stack_size = STACK_SZ_ON_HEAP(p);
-	const Uint size_before = MBUF_SIZE(p) + (HEAP_TOP(p) - HEAP_START(p));
-	Uint new_sz = HEAP_SIZE(p) + MBUF_SIZE(p) + combined_message_size(p);
+	((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) &&
+	 ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
+	 ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
+	Uint stack_size, size_after, need_after, new_sz;
+
+	stack_size = p->hend - p->stop;
+	new_sz = stack_size + size_before;
         new_sz = next_heap_size(p, new_sz, 0);
 
-        do_minor(p, new_sz, objv, nobj);
+        do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm),
+		 new_sz, objv, nobj);
 
         size_after = HEAP_TOP(p) - HEAP_START(p);
         *recl += (size_before - size_after);
 
-        /*
-	 * Copy newly received message onto the end of the new heap.
-	 */
-        ErtsGcQuickSanityCheck(p);
-        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-            if (msgp->data.attached) {
-                ErtsHeapFactory factory;
-                erts_factory_proc_prealloc_init(&factory, p,
-                                                erts_msg_attached_data_size(msgp));
-                erts_move_msg_attached_data_to_heap(&factory, msgp);
-                erts_factory_close(&factory);
-                ErtsGcQuickSanityCheck(p);
-            }
-        }
 	ErtsGcQuickSanityCheck(p);
 
         GEN_GCS(p)++;
         need_after = ((HEAP_TOP(p) - HEAP_START(p))
-                      + erts_used_frag_sz(MBUF(p))
                       + need
                       + stack_size);
 	
@@ -984,10 +1053,13 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
             ASSERT(MBUF(p) == NULL);
             return 1;
 	}
+
+        grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
+	return 1;
     }
 
     /*
-     * Still not enough room after minor collection. Must force a major collection.
+     * Not enough room for a minor collection. Must force a major collection.
      */
     FLAGS(p) |= F_NEED_FULLSWEEP;
     return 0;
@@ -1044,7 +1116,9 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs,
 #endif /* HIPE */
 
 static void
-do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
+do_minor(Process *p, ErlHeapFragment *live_hf_end,
+	 char *mature, Uint mature_size,
+	 Uint new_sz, Eterm* objv, int nobj)
 {
     Rootset rootset;            /* Rootset for GC (stack, dictionary, etc). */
     Roots* roots;
@@ -1053,9 +1127,6 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
     Eterm* ptr;
     Eterm val;
     Eterm gval;
-    char* heap = (char *) HEAP_START(p);
-    Uint heap_size = (char *) HEAP_TOP(p) - heap;
-    Uint mature_size = (char *) HIGH_WATER(p) - heap;
     Eterm* old_htop = OLD_HTOP(p);
     Eterm* n_heap;
     char* oh = (char *) OLD_HEAP(p);
@@ -1064,8 +1135,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
     n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 					       sizeof(Eterm)*new_sz);
 
-    if (MBUF(p) != NULL) {
-	n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj);
+    if (live_hf_end != ERTS_INVALID_HFRAG_PTR) {
+	/*
+	 * Move heap frags that we know are completely live
+	 * directly into the new young heap generation.
+	 */
+	n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop,
+					 objv, nobj);
     }
 
     n = setup_rootset(p, objv, nobj, &rootset);
@@ -1088,11 +1164,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
-                } else if (ErtsInArea(ptr, heap, mature_size)) {
+                } else if (ErtsInArea(ptr, mature, mature_size)) {
                     MOVE_BOXED(ptr,val,old_htop,g_ptr++);
-                } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
-						heap, heap_size,
-						oh, oh_size)) {
+                } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
                     MOVE_BOXED(ptr,val,n_htop,g_ptr++);
                 } else {
 		    g_ptr++;
@@ -1105,11 +1179,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 val = *ptr;
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
-                } else if (ErtsInArea(ptr, heap, mature_size)) {
+                } else if (ErtsInArea(ptr, mature, mature_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
-                } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
-						heap, heap_size,
-						oh, oh_size)) {
+                } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
                     MOVE_CONS(ptr,val,n_htop,g_ptr++);
                 } else {
 		    g_ptr++;
@@ -1134,8 +1206,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
      */
 
     if (mature_size == 0) {
-	n_htop = sweep_new_heap(n_heap, n_htop, heap, heap_size,
-				oh, oh_size);
+	n_htop = sweep_new_heap(n_heap, n_htop, oh, oh_size);
     } else {
 	Eterm* n_hp = n_heap;
 	Eterm* ptr;
@@ -1152,11 +1223,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
-		} else if (ErtsInArea(ptr, heap, mature_size)) {
+		} else if (ErtsInArea(ptr, mature, mature_size)) {
 		    MOVE_BOXED(ptr,val,old_htop,n_hp++);
-		} else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
-						heap, heap_size,
-						oh, oh_size)) {
+		} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
 		} else {
 		    n_hp++;
@@ -1168,11 +1237,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		val = *ptr;
 		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
-		} else if (ErtsInArea(ptr, heap, mature_size)) {
+		} else if (ErtsInArea(ptr, mature, mature_size)) {
 		    MOVE_CONS(ptr,val,old_htop,n_hp++);
-		} else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
-						heap, heap_size,
-						oh, oh_size)) {
+		} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
 		} else {
 		    n_hp++;
@@ -1192,12 +1259,10 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 			if (IS_MOVED_BOXED(val)) {
 			    *origptr = val;
 			    mb->base = binary_bytes(val);
-			} else if (ErtsInArea(ptr, heap, mature_size)) {
+			} else if (ErtsInArea(ptr, mature, mature_size)) {
 			    MOVE_BOXED(ptr,val,old_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
-			} else if (ERTS_IS_NEW_HEAP_PTR(*origptr, ptr,
-							heap, heap_size,
-							oh, oh_size)) {
+			} else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) {
 			    MOVE_BOXED(ptr,val,n_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			}
@@ -1218,11 +1283,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
      * may point to the old (soon to be deleted) new_heap.
      */
 
-    if (OLD_HTOP(p) < old_htop) {
-	old_htop = sweep_new_heap(OLD_HTOP(p), old_htop,
-				  heap, heap_size,
-				  oh, oh_size);
-    }
+    if (OLD_HTOP(p) < old_htop)
+	old_htop = sweep_new_heap(OLD_HTOP(p), old_htop, oh, oh_size);
     OLD_HTOP(p) = old_htop;
     HIGH_WATER(p) = n_htop;
 
@@ -1254,8 +1316,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 #endif
 
     ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (void*)HEAP_START(p),
+		   (p->abandoned_heap
+		    ? p->abandoned_heap
+		    : HEAP_START(p)),
 		   HEAP_SIZE(p) * sizeof(Eterm));
+    p->abandoned_heap = NULL;
+    p->flags &= ~F_ABANDONED_HEAP_USE;
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
     HEAP_SIZE(p) = new_sz;
@@ -1272,15 +1338,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
  */
 
 static int
-major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
+major_collection(Process* p, ErlHeapFragment *live_hf_end,
+		 int need, Eterm* objv, int nobj, Uint *recl)
 {
-    const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p))
-                              + (OLD_HTOP(p) - OLD_HEAP(p))
-                              + MBUF_SIZE(p));
+    Uint size_before, stack_size;
     Eterm* n_heap;
     Eterm* n_htop;
-    char* src = (char *) HEAP_START(p);
-    Uint src_size = (char *) HEAP_TOP(p) - src;
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
     Uint new_sz, stk_sz;
@@ -1290,9 +1353,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      * to receive all live data.
      */
 
-    new_sz = (HEAP_SIZE(p) + MBUF_SIZE(p)
-              + combined_message_size(p)
-              + (OLD_HTOP(p) - OLD_HEAP(p)));
+    size_before = young_gen_usage(p);
+    size_before += p->old_htop - p->old_heap;
+    stack_size = p->hend - p->stop;
+
+    new_sz = stack_size + size_before;
     new_sz = next_heap_size(p, new_sz, 0);
 
     /*
@@ -1306,16 +1371,16 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 						sizeof(Eterm)*new_sz);
 
-    /*
-     * Get rid of heap fragments.
-     */
-
-    if (MBUF(p) != NULL) {
-	n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj);
+    if (live_hf_end != ERTS_INVALID_HFRAG_PTR) {
+	/*
+	 * Move heap frags that we know are completely live
+	 * directly into the heap.
+	 */
+	n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop,
+					 objv, nobj);
     }
 
-    n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, src, src_size,
-			      oh, oh_size, objv, nobj);
+    n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj);
 
     /* Move the stack to the end of the heap */
     stk_sz = HEAP_END(p) - p->stop;
@@ -1332,8 +1397,12 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 #endif
 
     ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (void *) HEAP_START(p),
+		   (p->abandoned_heap
+		    ? p->abandoned_heap
+		    : HEAP_START(p)),
 		   (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
+    p->abandoned_heap = NULL;
+    p->flags &= ~F_ABANDONED_HEAP_USE;
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
     HEAP_SIZE(p) = new_sz;
@@ -1346,24 +1415,6 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 
     *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
 
-    {
-	ErlMessage *msgp;
-
-	/*
-	 * Copy newly received message onto the end of the new heap.
-	 */
-        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-	    if (msgp->data.attached) {
-                ErtsHeapFactory factory;
-                erts_factory_proc_prealloc_init(&factory, p,
-                                                erts_msg_attached_data_size(msgp));
-		erts_move_msg_attached_data_to_heap(&factory, msgp);
-                erts_factory_close(&factory);
-		ErtsGcQuickSanityCheck(p);
-	    }
-	}
-    }
-
     adjust_after_fullsweep(p, need, objv, nobj);
 
 #ifdef HARDDEBUG
@@ -1379,7 +1430,6 @@ static Eterm *
 full_sweep_heaps(Process *p,
 		 int hibernate,
 		 Eterm *n_heap, Eterm* n_htop,
-		 char *h, Uint h_size,
 		 char *oh, Uint oh_size,
 		 Eterm *objv, int nobj)
 {
@@ -1420,9 +1470,7 @@ full_sweep_heaps(Process *p,
 		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
-		} else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
-						h, h_size,
-						oh, oh_size)) {
+		} else if (!erts_is_literal(gval, ptr)) {
 		    MOVE_BOXED(ptr,val,n_htop,g_ptr++);
 		} else {
 		    g_ptr++;
@@ -1435,9 +1483,7 @@ full_sweep_heaps(Process *p,
 		val = *ptr;
 		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
-		} else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
-						h, h_size,
-						oh, oh_size)) {
+		} else if (!erts_is_literal(gval, ptr)) {
 		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
 		} else {
 		    g_ptr++;
@@ -1462,7 +1508,7 @@ full_sweep_heaps(Process *p,
      * until all is copied.
      */
 
-    n_htop = sweep_heaps(n_heap, n_htop, h, h_size, oh, oh_size);
+    n_htop = sweep_heaps(n_heap, n_htop, oh, oh_size);
 
     if (MSO(p).first) {
 	sweep_off_heap(p, 1);
@@ -1513,23 +1559,6 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
     }
 }
 
-/*
- * Return the size of all message buffers that are NOT linked in the
- * mbuf list.
- */
-static Uint
-combined_message_size(Process* p)
-{
-    Uint sz;
-    ErlMessage *msgp;
-
-    for (sz = 0, msgp = p->msg.first; msgp; msgp = msgp->next) {
-	if (msgp->data.attached)
-	    sz += erts_msg_attached_data_size(msgp);
-    }
-    return sz;
-}
-
 /*
  * Remove all message buffers.
  */
@@ -1540,6 +1569,10 @@ remove_message_buffers(Process* p)
 	free_message_buffer(MBUF(p));
 	MBUF(p) = NULL;
     }    
+    if (p->msg_frag) {
+	erts_cleanup_messages(p->msg_frag);
+	p->msg_frag = NULL;
+    }
     MBUF_SIZE(p) = 0;    
 }
 #ifdef HARDDEBUG
@@ -1551,64 +1584,6 @@ remove_message_buffers(Process* p)
  * For performance reasons, we use _unchecked_list_val(), _unchecked_boxed_val(),
  * and so on to avoid a function call.
  */
- 
-static void
-disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
-{
-    ErlHeapFragment* mbuf;
-    ErlHeapFragment* qb;
-    Eterm gval;
-    Eterm* ptr;
-    Eterm val;
-
-    ASSERT(p->htop != NULL);
-    mbuf = MBUF(p);
-
-    while (nobj--) {
-	gval = *objv;
-	
-	switch (primary_tag(gval)) {
-
-	case TAG_PRIMARY_BOXED: {
-	    ptr = _unchecked_boxed_val(gval);
-	    val = *ptr;
-	    if (IS_MOVED_BOXED(val)) {
-		ASSERT(is_boxed(val));
-		objv++;
-	    } else {
- 		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
-			abort();
-		    }
-		}
-		objv++;
-	    }
-	    break;
-	}
-
-	case TAG_PRIMARY_LIST: {
-	    ptr = _unchecked_list_val(gval);
-	    val = *ptr;
-	    if (IS_MOVED_CONS(val)) {
-		objv++;
-	    } else {
-		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
-			abort();
-		    }
-		}
-		objv++;
-	    }
-	    break;
-	}
-
-	default: {
-	    objv++;
-	    break;
-	}
-	}
-    }
-}
 
 static void
 disallow_heap_frag_ref_in_heap(Process* p)
@@ -1737,7 +1712,6 @@ typedef enum {
 static ERTS_FORCE_INLINE Eterm *
 sweep(Eterm *n_hp, Eterm *n_htop,
       ErtsSweepType type,
-      char *h, Uint hsz,
       char *oh, Uint ohsz,
       char *src, Uint src_size)
 {
@@ -1749,14 +1723,10 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 
 #define ERTS_IS_IN_SWEEP_AREA(TPtr, Ptr)				\
     (type == ErtsSweepHeaps						\
-     ? !ERTS_IS_LITERAL_PTR((TPtr), (Ptr), h, hsz, oh, ohsz)		\
+     ? !erts_is_literal((TPtr), (Ptr))					\
      : (type == ErtsSweepNewHeap					\
-	? ERTS_IS_NEW_HEAP_PTR((TPtr), (Ptr), h, hsz, oh, ohsz)		\
-	: (ErtsInArea((Ptr), src, src_size)				\
-	   ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))		\
-			     && !ErtsInArea((Ptr), h, hsz)		\
-			     && !ErtsInArea((Ptr), oh, ohsz)), 1)	\
-	   : 0)))
+	? ErtsInYoungGen((TPtr), (Ptr), oh, ohsz)			\
+	: ErtsInArea((Ptr), src, src_size)))
 
     while (n_hp != n_htop) {
 	ASSERT(n_hp < n_htop);
@@ -1820,38 +1790,30 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 }
 
 static Eterm *
-sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
-	       char* new_heap, Uint new_heap_size,
-	       char* old_heap, Uint old_heap_size)
+sweep_new_heap(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size)
 {
     return sweep(n_hp, n_htop,
 		 ErtsSweepNewHeap,
-		 new_heap, new_heap_size,
 		 old_heap, old_heap_size,
 		 NULL, 0);
 }
 
 static Eterm *
-sweep_heaps(Eterm *n_hp, Eterm *n_htop,
-	    char* new_heap, Uint new_heap_size,
-	    char* old_heap, Uint old_heap_size)
+sweep_heaps(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size)
 {
     return sweep(n_hp, n_htop,
 		 ErtsSweepHeaps,
-		 new_heap, new_heap_size,
 		 old_heap, old_heap_size,
 		 NULL, 0);
 }
 
 static Eterm *
 sweep_literal_area(Eterm *n_hp, Eterm *n_htop,
-		   char* new_heap, Uint new_heap_size,
 		   char* old_heap, Uint old_heap_size,
 		   char* src, Uint src_size)
 {
     return sweep(n_hp, n_htop,
 		 ErtsSweepLiteralArea,
-		 new_heap, new_heap_size,
 		 old_heap, old_heap_size,
 		 src, src_size);
 }
@@ -1956,32 +1918,21 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size)
  */
 
 static Eterm*
-collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
-		   Eterm* objv, int nobj)
+collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
+			Eterm* n_hstart, Eterm* n_htop,
+			Eterm* objv, int nobj)
 {
     ErlHeapFragment* qb;
     char* frag_begin;
     Uint frag_size;
 
-    /*
-     * We don't allow references to a heap fragments from the stack, heap,
-     * or process dictionary.
-     */
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref(p, n_htop, p->stop, STACK_START(p) - p->stop);
-    if (p->dictionary != NULL) {
-	disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used);
-    }
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-
     /*
      * Move the heap fragments to the new heap. Note that no GC is done on
      * the heap fragments. Any garbage will thus be moved as well and survive
      * until next GC.  
      */ 
     qb = MBUF(p);
-    while (qb != NULL) {
+    while (qb != live_hf_end) {
         ASSERT(!qb->off_heap.first);  /* process fragments use the MSO(p) list */
 	frag_size = qb->used_size * sizeof(Eterm);
 	if (frag_size != 0) {
@@ -1996,9 +1947,7 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
 static Uint
 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 {
-    Uint avail;
     Roots* roots;
-    ErlMessage* mp;
     Uint n;
 
     n = 0;
@@ -2076,31 +2025,48 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 
     ASSERT(n <= rootset->size);
 
-    mp = p->msg.first;
-    avail = rootset->size - n;
-    while (mp != NULL) {
-	if (avail == 0) {
-	    Uint new_size = 2*rootset->size;
-	    if (roots == rootset->def) {
-		roots = erts_alloc(ERTS_ALC_T_ROOTSET,
-				   new_size*sizeof(Roots));
-		sys_memcpy(roots, rootset->def, sizeof(rootset->def));
-	    } else {
-		roots = erts_realloc(ERTS_ALC_T_ROOTSET,
-				     (void *) roots,
-				     new_size*sizeof(Roots));
-	    }
+    switch (p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) {
+    case F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG:
+	(void) erts_move_messages_off_heap(p);
+    case F_OFF_HEAP_MSGQ:
+	break;
+    case F_OFF_HEAP_MSGQ_CHNG:
+    case 0: {
+	/*
+	 * Off heap message queue disabled, i.e. we may
+	 * have references from the message queue to the
+	 * heap...
+	 */
+	ErtsMessage *mp;
+
+	/* Ensure large enough rootset... */
+	if (n + p->msg.len > rootset->size) {
+	    Uint new_size = n + p->msg.len;
+	    ERTS_GC_ASSERT(roots == rootset->def);
+	    roots = erts_alloc(ERTS_ALC_T_ROOTSET,
+			       new_size*sizeof(Roots));
+	    sys_memcpy(roots, rootset->def, n*sizeof(Roots));
 	    rootset->size = new_size;
-	    avail = new_size - n;
 	}
-	if (mp->data.attached == NULL) {
-	    roots[n].v = mp->m;
-	    roots[n].sz = 2;
-	    n++;
-	    avail--;
+
+	for (mp = p->msg.first; mp; mp = mp->next) {
+
+	    if (!mp->data.attached) {
+		/*
+		 * Message may refer data on heap;
+		 * add it to rootset...
+		 */
+		roots[n].v = mp->m;
+		roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ;
+		n++;
+	    }
 	}
-        mp = mp->next;
+	break;
+    }
     }
+
+    ASSERT(rootset->size >= n);
+
     rootset->roots = roots;
     rootset->num_roots = n;
     return n;
@@ -2569,7 +2535,7 @@ offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
 static void
 offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
 {
-    ErlMessage* mp = p->msg.first;
+    ErtsMessage* mp = p->msg.first;
 
     while (mp != NULL) {
         Eterm mesg = ERL_MESSAGE_TERM(mp);
@@ -2656,7 +2622,7 @@ reply_gc_info(void *vgcirp)
     Eterm **hpp;
     Uint sz, *szp;
     ErlOffHeap *ohp = NULL;
-    ErlHeapFragment *bp = NULL;
+    ErtsMessage *mp = NULL;
 
     ASSERT(esdp);
 
@@ -2682,12 +2648,13 @@ reply_gc_info(void *vgcirp)
 	if (hpp)
 	  break;
 	
-	hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+	mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+
 	szp = NULL;
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+    erts_queue_message(rp, &rp_locks, mp, msg, NIL);
 
     if (gcirp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -2739,36 +2706,49 @@ erts_gc_info_request(Process *c_p)
 static int
 within2(Eterm *ptr, Process *p, Eterm *real_htop)
 {
-    ErlHeapFragment* bp = MBUF(p);
-    ErlMessage* mp = p->msg.first;
-    Eterm *htop = real_htop ? real_htop : HEAP_TOP(p);
+    ErlHeapFragment* bp;
+    ErtsMessage* mp;
+    Eterm *htop, *heap;
+
+    if (p->abandoned_heap)
+	ERTS_GET_ORIG_HEAP(p, heap, htop);
+    else {
+	heap = p->heap;
+	htop = real_htop ? real_htop : HEAP_TOP(p);
+    }
 
     if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) {
         return 1;
     }
-    if (HEAP_START(p) <= ptr && ptr < htop) {
+    if (heap <= ptr && ptr < htop) {
         return 1;
     }
-    while (bp != NULL) {
-        if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) {
-            return 1;
-        }
-        bp = bp->next;
-    }
+
+    mp = p->msg_frag;
+    bp = p->mbuf;
+
+    if (bp)
+	goto search_heap_frags;
+
     while (mp) {
-	if (mp->data.attached) {
-	    ErlHeapFragment *hfp;
-	    if (is_value(ERL_MESSAGE_TERM(mp)))
-		hfp = mp->data.heap_frag;
-	    else if (is_not_nil(ERL_MESSAGE_TOKEN(mp)))
-		hfp = erts_dist_ext_trailer(mp->data.dist_ext);
-	    else
-		hfp = NULL;
-	    if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size)
+
+	if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG)
+	    bp = &mp->hfrag;
+	else
+	    bp = mp->data.heap_frag;
+
+	mp = mp->next;
+
+    search_heap_frags:
+
+	while (bp) {
+	    if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) {
 		return 1;
+	    }
+	    bp = bp->next;
 	}
-        mp = mp->next;
     }
+
     return 0;
 }
 
@@ -2790,11 +2770,11 @@ do {							\
 		 __FILE__, __LINE__, #EXP);		\
 } while (0)
 
+
 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
 #  define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31)
 #endif
 
-
 void
 erts_check_off_heap2(Process *p, Eterm *htop)
 {
@@ -2823,7 +2803,7 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	}
 	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
-	ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT));
+	ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_OFFHEAP_VISITED_BIT));
 	u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT;
 #endif
 	if (old) {
@@ -2836,7 +2816,7 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	}
     }
 
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
+#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
     for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next)
 	u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT;
 #endif
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index fdbf948f9d..a496c5f008 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -69,13 +69,14 @@ do {                                                                    \
     while (nelts--) *HTOP++ = *PTR++;                                   \
 } while(0)
 
-#define in_area(ptr,start,nbytes) \
- ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
-
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 int within(Eterm *ptr, Process *p);
 #endif
 
+#define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz)			\
+    (!erts_is_literal((TPtr), (Ptr))					\
+     & !ErtsInArea((Ptr), (OldHeap), (OldHeapSz)))
+
 ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -98,6 +99,7 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag)
     }
     return term;
 }
+
 #endif
 
 #endif /* ERL_GC_C__ || HIPE_GC_C__ */
@@ -106,6 +108,23 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag)
  * Global exported
  */
 
+#define ERTS_IS_GC_DESIRED_INTERNAL(Proc, HTop, STop)			\
+    ((((STop) - (HTop) < (Proc)->mbuf_sz))				\
+     | ((Proc)->off_heap.overhead > (Proc)->bin_vheap_sz)		\
+     | !!((Proc)->flags & F_FORCE_GC))
+
+#define ERTS_IS_GC_DESIRED(Proc)					\
+    ERTS_IS_GC_DESIRED_INTERNAL((Proc), (Proc)->htop, (Proc)->stop)
+
+#define ERTS_FORCE_GC_INTERNAL(Proc, FCalls)				\
+    do {								\
+	(Proc)->flags |= F_FORCE_GC;					\
+	ERTS_VBUMP_ALL_REDS_INTERNAL((Proc), (FCalls));			\
+    } while (0)
+
+#define ERTS_FORCE_GC(Proc)						\
+    ERTS_FORCE_GC_INTERNAL((Proc), (Proc)->fcalls)
+
 extern Uint erts_test_long_gc_sleep;
 
 typedef struct {
@@ -117,6 +136,8 @@ void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
 int erts_garbage_collect(struct process*, int, Eterm*, int);
 void erts_garbage_collect_hibernate(struct process* p);
+Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
+				 Eterm result, Eterm* regs, Uint arity);
 Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity);
 void erts_garbage_collect_literals(struct process* p, Eterm* literals,
 				   Uint lit_size,
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 51a0d68247..6853278828 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1245,7 +1245,9 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
 	 * the middle of tree destruction).
 	 */
 	if (!ERTS_PROC_IS_EXITING(proc)) {
-	    erts_queue_message(proc, &proc_locks, tmr->btm.bp,
+	    ErtsMessage *mp = erts_alloc_message(0, NULL);
+	    mp->data.heap_frag = tmr->btm.bp;
+	    erts_queue_message(proc, &proc_locks, mp,
 			       tmr->btm.message, NIL);
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND);
 	    queued_message = 1;
@@ -1926,36 +1928,31 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 
     if (proc) {
 	Uint hsz;
-	ErlOffHeap *ohp;
-	ErlHeapFragment* bp;
+	ErtsMessage *mp;
 	Eterm *hp, msg, ref, result;
+	ErlOffHeap *ohp;
+	Uint32 *refn;
 #ifdef ERTS_HLT_DEBUG
 	Eterm *hp_end;
 #endif
 
-	hsz = 3; /* 2-tuple */
-	if (!async)
-	    hsz += REF_THING_SIZE;
+	hsz = REF_THING_SIZE;
+	if (async) {
+	    refn = trefn; /* timer ref */
+	    hsz += 4; /* 3-tuple */
+	}
 	else {
-	    if (is_non_value(tref) || proc != c_p)
-		hsz += REF_THING_SIZE;
-	    hsz += 1; /* upgrade to 3-tuple */
+	    refn = rrefn; /* request ref */
+	    hsz += 3; /* 2-tuple */
 	}
+
+	ERTS_HLT_ASSERT(refn);
+
 	if (time_left > (Sint64) MAX_SMALL)
 	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
 
-	if (proc == c_p) {
-	    bp = NULL;
-	    ohp = NULL;
-	    hp = HAlloc(c_p, hsz);
-	}
-	else {
-	    hp = erts_alloc_message_heap(hsz,
-					 &bp,
-					 &ohp,
-					 proc,
-					 &proc_locks);
-	}
+	mp = erts_alloc_message_heap(proc, &proc_locks,
+				     hsz, &hp, &ohp);
 
 #ifdef ERTS_HLT_DEBUG
 	hp_end = hp + hsz;
@@ -1968,35 +1965,22 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 	else
 	    result = erts_sint64_to_big(time_left, &hp);
 
-	if (!async) {
-	    write_ref_thing(hp,
-			    rrefn[0],
-			    rrefn[1],
-			    rrefn[2]);
-	    ref = make_internal_ref(hp);
-	    hp += REF_THING_SIZE;
-	    msg = TUPLE2(hp, ref, result);
+	write_ref_thing(hp,
+			refn[0],
+			refn[1],
+			refn[2]);
+	ref = make_internal_ref(hp);
+	hp += REF_THING_SIZE;
 
-	    ERTS_HLT_ASSERT(hp + 3 == hp_end);
-	}
-	else {
-	    Eterm tag = cancel ? am_cancel_timer : am_read_timer;
-	    if (is_value(tref) && proc == c_p)
-		ref = tref;
-	    else {
-		write_ref_thing(hp,
-				trefn[0],
-				trefn[1],
-				trefn[2]);
-		ref = make_internal_ref(hp);
-		hp += REF_THING_SIZE;
-	    }
-	    msg = TUPLE3(hp, tag, ref, result);
+	msg = (async
+	       ? TUPLE3(hp, (cancel
+			     ? am_cancel_timer
+			     : am_read_timer), ref, result)
+	       : TUPLE2(hp, ref, result));
 
-	    ERTS_HLT_ASSERT(hp + 4 == hp_end);
+	ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end);
 
-	}
-	erts_queue_message(proc, &proc_locks, bp, msg, NIL);
+	erts_queue_message(proc, &proc_locks, mp, msg, NIL);
 
 	if (c_p)
 	    proc_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -2093,16 +2077,19 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 	}
     }
     else {
+	ErtsMessage *mp;
 	Eterm tag, res, msg;
 	Uint hsz;
 	Eterm *hp;
 	ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
+	ErlOffHeap *ohp;
 
 	hsz = 4;
 	if (time_left > (Sint64) MAX_SMALL)
 	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
 
-	hp = HAlloc(c_p, hsz);
+	mp = erts_alloc_message_heap(c_p, &proc_locks,
+				     hsz, &hp, &ohp);
 	if (cancel)
 	    tag = am_cancel_timer;
 	else
@@ -2117,7 +2104,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 
 	msg = TUPLE3(hp, tag, tref, res);
 
-	erts_queue_message(c_p, &proc_locks, NULL, msg, NIL);
+	erts_queue_message(c_p, &proc_locks, mp, msg, NIL);
 
 	proc_locks &= ~ERTS_PROC_LOCK_MAIN;
 	if (proc_locks)
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 5c209a4af2..f396a0a156 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -431,7 +431,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
     hp += 2;
     args = CONS(hp, env, args);
 
-    so.flags = SPO_SYSTEM_PROC;
+    so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC;
     res = erl_create_process(&parent, start_mod, am_start, args, &so);
     erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN);
     erts_cleanup_empty_process(&parent);
@@ -630,6 +630,7 @@ void erts_usage(void)
 
     erts_fprintf(stderr, "-W<i|w|e>      set error logger warnings mapping,\n");
     erts_fprintf(stderr, "               see error_logger documentation for details\n");
+    erts_fprintf(stderr, "-xohmq  bool   set default off_heap_message_queue flag for processes\n");
     erts_fprintf(stderr, "-zdbbl size    set the distribution buffer busy limit in kilobytes\n");
     erts_fprintf(stderr, "               valid range is [1-%d]\n", INT_MAX/1024);
     erts_fprintf(stderr, "-zdntgc time   set delayed node table gc in seconds\n");
@@ -2015,6 +2016,26 @@ erl_start(int argc, char **argv)
 	    }
 	    break;
 
+	case 'x': {
+	    char *sub_param = argv[i]+2;
+	    if (has_prefix("ohmq", sub_param)) {
+		arg = get_arg(sub_param+4, argv[i+1], &i);
+		if (sys_strcmp(arg, "true") == 0)
+		    erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ;
+		else if (sys_strcmp(arg, "false") == 0)
+		    erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ;
+		else {
+		    erts_fprintf(stderr,
+				 "Invalid off_heap_message_queue flag: %s\n", arg);
+		    erts_usage();
+		}
+	    } else {
+		erts_fprintf(stderr, "bad -x option %s\n", argv[i]);
+		erts_usage();
+	    }
+	    break;
+        }
+
 	case 'z': {
 	    char *sub_param = argv[i]+2;
 
@@ -2068,7 +2089,8 @@ erl_start(int argc, char **argv)
 				 "Invalid ets busy wait threshold: %s\n", arg);
 		    erts_usage();
 		}
-	    } else {
+	    }
+	    else {
 		erts_fprintf(stderr, "bad -z option %s\n", argv[i]);
 		erts_usage();
 	    }
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index e23c79d301..79739501a8 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -33,8 +33,8 @@
 #include "erl_binary.h"
 #include "dtrace-wrapper.h"
 
-ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message,
-				 ErlMessage,
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref,
+				 ErtsMessageRef,
 				 ERL_MESSAGE_BUF_SZ,
 				 ERTS_ALC_T_MSG_REF)
 
@@ -44,27 +44,20 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message,
 #undef HARD_DEBUG
 #endif
 
-
-
-
-#ifdef DEBUG
-static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp)
+void
+init_message(void)
 {
-    return ((unsigned)(ptr - bp->mem) < bp->used_size);
+    init_message_ref_alloc();
 }
-#endif
 
-
-void
-init_message(void)
+void *erts_alloc_message_ref(void)
 {
-    init_message_alloc();
+    return (void *) message_ref_alloc();
 }
 
-void
-free_message(ErlMessage* mp)
+void erts_free_message_ref(void *mp)
 {
-    message_free(mp);
+    message_ref_free((ErtsMessageRef *) mp);
 }
 
 /* Allocate message buffer (size in words) */
@@ -74,7 +67,7 @@ new_message_buffer(Uint size)
     ErlHeapFragment* bp;
     bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG,
 					    ERTS_HEAP_FRAG_SIZE(size));
-    ERTS_INIT_HEAP_FRAG(bp, size);
+    ERTS_INIT_HEAP_FRAG(bp, size, size);
     return bp;
 }
 
@@ -203,83 +196,87 @@ free_message_buffer(ErlHeapFragment* bp)
     }while (bp != NULL);
 }
 
-static ERTS_INLINE void
-link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
+void
+erts_cleanup_messages(ErtsMessage *msgp)
 {
-    if (bp) {
-	/* Link the message buffer */
-	bp->next = MBUF(proc);
-	MBUF(proc) = bp;
-	MBUF_SIZE(proc) += bp->used_size;
-	FLAGS(proc) |= F_FORCE_GC;
-
-	/* Move any off_heap's into the process */
-	if (bp->off_heap.first != NULL) {
-	    struct erl_off_heap_header** next_p = &bp->off_heap.first;
-	    while (*next_p != NULL) {
-		next_p = &((*next_p)->next);
+    ErtsMessage *mp = msgp;
+    while (mp) {
+	ErtsMessage *fmp;
+	ErlHeapFragment *bp;
+	if (is_non_value(ERL_MESSAGE_TERM(mp))) {
+	    if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) {
+		bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp;
+		erts_cleanup_offheap(&bp->off_heap);
 	    }
-	    *next_p = MSO(proc).first;
-	    MSO(proc).first = bp->off_heap.first;
-	    bp->off_heap.first = NULL;
-	    OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
+	    if (mp->data.dist_ext)
+		erts_free_dist_ext_copy(mp->data.dist_ext);
 	}
+	else {
+	    if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
+		bp = mp->data.heap_frag;
+	    else {
+		bp = mp->hfrag.next;
+		erts_cleanup_offheap(&mp->hfrag.off_heap);
+	    }
+	    if (bp)
+		free_message_buffer(bp);
+	}
+	fmp = mp;
+	mp = mp->next;
+	erts_free_message(fmp);
     }
 }
 
-Eterm
-erts_msg_distext2heap(Process *pp,
-		      ErtsProcLocks *plcksp,
-		      ErlHeapFragment **bpp,
-		      Eterm *tokenp,
-		      ErtsDistExternal *dist_extp)
+ErtsMessage *
+erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size)
 {
-    Eterm msg;
-    Uint tok_sz = 0;
-    Eterm *hp = NULL;
-    ErtsHeapFactory factory;
-    Sint sz;
-
-    *bpp = NULL;
-    sz = erts_decode_dist_ext_size(dist_extp);
-    if (sz < 0)
-	goto decode_error;
-    if (is_not_nil(*tokenp)) {
-	ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
-	tok_sz = heap_frag->used_size;
-	sz += tok_sz;
-    }
-    if (pp) {
-	ErlOffHeap *ohp;
-	hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp);
-    }
-    else {
-	*bpp = new_message_buffer(sz);
-	hp = (*bpp)->mem;
-    }
-    erts_factory_message_init(&factory, pp, hp, *bpp);
-    msg = erts_decode_dist_ext(&factory, dist_extp);
-    if (is_non_value(msg))
-	goto decode_error;
-    if (is_not_nil(*tokenp)) {
-	ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
-	hp = erts_produce_heap(&factory, tok_sz, 0);
-	*tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap);
-	erts_cleanup_offheap(&heap_frag->off_heap);
+    ErtsMessage *nmp = erts_realloc(ERTS_ALC_T_MSG, mp,
+				    sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm));
+    if (nmp != mp) {
+	Eterm *sp = &mp->hfrag.mem[0];
+	Eterm *ep = sp + sz;
+	Sint offs = &nmp->hfrag.mem[0] - sp;
+	erts_offset_off_heap(&nmp->hfrag.off_heap, offs, sp, ep);
+	erts_offset_heap(&nmp->hfrag.mem[0], sz, offs, sp, ep);
+	if (brefs && brefs_size)
+	    erts_offset_heap_ptr(brefs, brefs_size, offs, sp, ep);
     }
-    erts_free_dist_ext_copy(dist_extp);
-    erts_factory_close(&factory);
-    return msg;
 
- decode_error:
-    if (is_not_nil(*tokenp)) {
-	ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
-	erts_cleanup_offheap(&heap_frag->off_heap);
+    nmp->hfrag.used_size = sz;
+    nmp->hfrag.alloc_size = sz;
+
+    return nmp;
+}
+
+void
+erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp)
+{
+    if (first_bp) {
+	ErlHeapFragment *bp = first_bp;
+
+	while (1) {
+	    /* Move any off_heap's into the process */
+	    if (bp->off_heap.first != NULL) {
+		struct erl_off_heap_header** next_p = &bp->off_heap.first;
+		while (*next_p != NULL) {
+		    next_p = &((*next_p)->next);
+		}
+		*next_p = MSO(proc).first;
+		MSO(proc).first = bp->off_heap.first;
+		bp->off_heap.first = NULL;
+		OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
+	    }
+	    MBUF_SIZE(proc) += bp->used_size;
+	    if (!bp->next)
+		break;
+	    bp = bp->next;
+	}
+
+	/* Link the message buffer */
+	bp->next = MBUF(proc);
+	MBUF(proc) = first_bp;
     }
-    erts_free_dist_ext_copy(dist_extp);
-    *bpp = NULL;
-    return THE_NON_VALUE;
- }
+}
 
 void
 erts_queue_dist_message(Process *rcvr,
@@ -287,7 +284,7 @@ erts_queue_dist_message(Process *rcvr,
 			ErtsDistExternal *dist_ext,
 			Eterm token)
 {
-    ErlMessage* mp;
+    ErtsMessage* mp;
 #ifdef USE_VM_PROBES
     Sint tok_label = 0;
     Sint tok_lastcnt = 0;
@@ -299,7 +296,17 @@ erts_queue_dist_message(Process *rcvr,
 
     ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
 
-    mp = message_alloc();
+    mp = erts_alloc_message(0, NULL);
+    mp->data.dist_ext = dist_ext;
+
+    ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
+#ifdef USE_VM_PROBES
+    ERL_MESSAGE_DT_UTAG(mp) = NIL;
+    if (token == am_have_dt_utag)
+	ERL_MESSAGE_TOKEN(mp) = NIL;
+    else
+#endif
+	ERL_MESSAGE_TOKEN(mp) = token;
 
 #ifdef ERTS_SMP
     if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
@@ -318,58 +325,40 @@ erts_queue_dist_message(Process *rcvr,
 	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 	/* Drop message if receiver is exiting or has a pending exit ... */
-	if (is_not_nil(token)) {
-	    ErlHeapFragment *heap_frag;
-	    heap_frag = erts_dist_ext_trailer(mp->data.dist_ext);
-	    erts_cleanup_offheap(&heap_frag->off_heap);
-	}
-	erts_free_dist_ext_copy(dist_ext);
-	message_free(mp);
+	erts_cleanup_messages(mp);
     }
     else
 #endif
     if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
 	/* Ahh... need to decode it in order to trace it... */
-	ErlHeapFragment *mbuf;
-	Eterm msg;
 	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
-	message_free(mp);
-	msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
-	if (is_value(msg))
+	if (!erts_decode_dist_message(rcvr, *rcvr_locks, mp, 0))
+	    erts_free_message(mp);
+	else {
+	    Eterm msg = ERL_MESSAGE_TERM(mp);
+	    token = ERL_MESSAGE_TOKEN(mp);
 #ifdef USE_VM_PROBES
-            if (DTRACE_ENABLED(message_queued)) {
-                DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
-
-                dtrace_proc_str(rcvr, receiver_name);
-                if (token != NIL && token != am_have_dt_utag) {
-                    tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
-                    tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
-                    tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
-                }
-                DTRACE6(message_queued,
-                        receiver_name, size_object(msg), rcvr->msg.len,
-                        tok_label, tok_lastcnt, tok_serial);
-            }
+	    if (DTRACE_ENABLED(message_queued)) {
+		DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
+
+		dtrace_proc_str(rcvr, receiver_name);
+		if (token != NIL && token != am_have_dt_utag) {
+		    tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+		    tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
+		    tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
+		}
+		DTRACE6(message_queued,
+			receiver_name, size_object(msg), rcvr->msg.len,
+			tok_label, tok_lastcnt, tok_serial);
+	    }
 #endif
-	    erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
+	    erts_queue_message(rcvr, rcvr_locks, mp, msg, token);
+	}
     }
     else {
 	/* Enqueue message on external format */
 
-	ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
-#ifdef USE_VM_PROBES
-	ERL_MESSAGE_DT_UTAG(mp) = NIL;
-	if (token == am_have_dt_utag) {
-	    ERL_MESSAGE_TOKEN(mp) = NIL;
-	} else {
-#endif
-	    ERL_MESSAGE_TOKEN(mp) = token;
-#ifdef USE_VM_PROBES
-	}
-#endif
-	mp->next = NULL;
-
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(message_queued)) {
             DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
@@ -388,7 +377,7 @@ erts_queue_dist_message(Process *rcvr,
                     tok_label, tok_lastcnt, tok_serial);
         }
 #endif
-	mp->data.dist_ext = dist_ext;
+
 	LINK_MESSAGE(rcvr, mp);
 
 	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
@@ -408,9 +397,9 @@ erts_queue_dist_message(Process *rcvr,
 static Sint
 queue_message(Process *c_p,
 	      Process* receiver,
-	      ErtsProcLocks *receiver_locks,
 	      erts_aint32_t *receiver_state,
-	      ErlHeapFragment* bp,
+	      ErtsProcLocks *receiver_locks,
+	      ErtsMessage* mp,
 	      Eterm message,
 	      Eterm seq_trace_token
 #ifdef USE_VM_PROBES
@@ -419,31 +408,24 @@ queue_message(Process *c_p,
     )
 {
     Sint res;
-    ErlMessage* mp;
     int locked_msgq = 0;
-    erts_aint_t state;
-
-#ifndef ERTS_SMP
-    ASSERT(bp != NULL || receiver->mbuf == NULL);
-#endif
+    erts_aint32_t state;
 
     ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
 
-    mp = message_alloc();
-
-    if (receiver_state)
-	state = *receiver_state;
-    else
-	state = erts_smp_atomic32_read_acqb(&receiver->state);
-
 #ifdef ERTS_SMP
 
-    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
-	goto exiting;
-
     if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
 	if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 	    ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
+
+	    if (receiver_state)
+		state = *receiver_state;
+	    else
+		state = erts_smp_atomic32_read_nob(&receiver->state);
+	    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+		goto exiting;
+
 	    if (*receiver_locks & ERTS_PROC_LOCK_STATUS) {
 		erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
 		need_locks |= ERTS_PROC_LOCK_STATUS;
@@ -451,13 +433,12 @@ queue_message(Process *c_p,
 	    erts_smp_proc_lock(receiver, need_locks);
 	}
 	locked_msgq = 1;
-	state = erts_smp_atomic32_read_nob(&receiver->state);
-	if (receiver_state)
-	    *receiver_state = state;
     }
 
 #endif
 
+    state = erts_smp_atomic32_read_nob(&receiver->state);
+
     if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
 #ifdef ERTS_SMP
     exiting:
@@ -465,9 +446,7 @@ queue_message(Process *c_p,
 	/* Drop message if receiver is exiting or has a pending exit... */
 	if (locked_msgq)
 	    erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
-	if (bp)
-	    free_message_buffer(bp);
-	message_free(mp);
+	erts_cleanup_messages(mp);
 	return 0;
     }
 
@@ -476,13 +455,9 @@ queue_message(Process *c_p,
 #ifdef USE_VM_PROBES
     ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
 #endif
-    mp->next = NULL;
-    mp->data.heap_frag = bp;
 
-#ifndef ERTS_SMP
     res = receiver->msg.len;
-#else
-    res = receiver->msg_inq.len;
+#ifdef ERTS_SMP
     if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
 	/*
 	 * We move 'in queue' to 'private queue' and place
@@ -492,7 +467,7 @@ queue_message(Process *c_p,
 	 * we don't need to include the 'in queue' in
 	 * the root set when garbage collecting.
 	 */
-	res += receiver->msg.len;
+	res += receiver->msg_inq.len;
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
 	LINK_MESSAGE_PRIVQ(receiver, mp);
     }
@@ -544,19 +519,19 @@ queue_message(Process *c_p,
 void
 #ifdef USE_VM_PROBES
 erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks,
-                         ErlHeapFragment* bp,
+                         ErtsMessage* mp,
                          Eterm message, Eterm seq_trace_token, Eterm dt_utag)
 #else
 erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks,
-                   ErlHeapFragment* bp,
+                   ErtsMessage* mp,
                    Eterm message, Eterm seq_trace_token)
 #endif
 {
     queue_message(NULL,
 		  receiver,
-		  receiver_locks,
 		  NULL,
-		  bp,
+		  receiver_locks,
+		  mp,
 		  message,
 		  seq_trace_token
 #ifdef USE_VM_PROBES
@@ -565,246 +540,8 @@ erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks,
 	);
 }
 
-void
-erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp)
-{
-    Eterm* htop = HEAP_TOP(proc);
-
-    link_mbuf_to_proc(proc, bp);
-    if (htop < HEAP_LIMIT(proc)) {
-	*htop = make_pos_bignum_header(HEAP_LIMIT(proc)-htop-1);
-	HEAP_TOP(proc) = HEAP_LIMIT(proc);
-    }
-}
-
-/*
- * Moves content of message buffer attached to a message into a heap.
- * The message buffer is deallocated.
- */
-void
-erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
-{
-    struct erl_off_heap_header* oh;
-    Eterm term, token, *fhp, *hp;
-    Sint offs;
-    Uint sz;
-    ErlHeapFragment *bp;
-#ifdef USE_VM_PROBES
-    Eterm utag;
-#endif
-
-#ifdef HARD_DEBUG
-    struct erl_off_heap_header* dbg_oh_start = off_heap->first;
-    Eterm dbg_term, dbg_token;
-    ErlHeapFragment *dbg_bp;
-    Uint *dbg_hp, *dbg_thp_start;
-    Uint dbg_term_sz, dbg_token_sz;
-#ifdef USE_VM_PROBES
-    Eterm dbg_utag;
-    Uint dbg_utag_sz;
-#endif
-#endif
-
-    bp = msg->data.heap_frag;
-    term = ERL_MESSAGE_TERM(msg);
-    token = ERL_MESSAGE_TOKEN(msg);
-#ifdef USE_VM_PROBES
-    utag = ERL_MESSAGE_DT_UTAG(msg);
-#endif
-    if (!bp) {
-#ifdef USE_VM_PROBES
-	ASSERT(is_immed(term) && is_immed(token) && is_immed(utag));
-#else
-	ASSERT(is_immed(term) && is_immed(token));
-#endif
-	return;
-    }
-
-#ifdef HARD_DEBUG
-    dbg_term_sz = size_object(term);
-    dbg_token_sz = size_object(token);
-    dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz);
-#ifdef USE_VM_PROBES
-    dbg_utag_sz = size_object(utag);
-    dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz );
-#endif
-    /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg));
-      Copied size may be smaller due to removed SubBins's or garbage.
-      Copied size may be larger due to duplicated shared terms.
-    */
-    dbg_hp = dbg_bp->mem;
-    dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
-    dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
-#ifdef USE_VM_PROBES
-    dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap);
-#endif
-   dbg_thp_start = *hpp;
-#endif
-
-    if (bp->next != NULL) {
-	erts_move_multi_frags(hpp, off_heap, bp, msg->m, 
-#ifdef USE_VM_PROBES
-			      3,
-#else
-			      2,
-#endif
-			      0);
-	goto copy_done;
-    }
-
-    OH_OVERHEAD(off_heap, bp->off_heap.overhead);
-    sz = bp->used_size;
-
-    ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
-    ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp));
-
-    fhp = bp->mem;
-    hp = *hpp;
-    offs = hp - fhp;
-
-    oh = NULL;
-    while (sz--) {
-	Uint cpy_sz;
-	Eterm val = *fhp++;
-
-	switch (primary_tag(val)) {
-	case TAG_PRIMARY_IMMED1:
-	    *hp++ = val;
-	    break;
-	case TAG_PRIMARY_LIST:
-	case TAG_PRIMARY_BOXED:
-	    ASSERT(in_heapfrag(ptr_val(val), bp));
-	    *hp++ = offset_ptr(val, offs);
-	    break;
-	case TAG_PRIMARY_HEADER:
-	    *hp++ = val;
-	    switch (val & _HEADER_SUBTAG_MASK) {
-	    case ARITYVAL_SUBTAG:
-		break;
-	    case REFC_BINARY_SUBTAG:
-	    case FUN_SUBTAG:
-	    case EXTERNAL_PID_SUBTAG:
-	    case EXTERNAL_PORT_SUBTAG:
-	    case EXTERNAL_REF_SUBTAG:
-		oh = (struct erl_off_heap_header*) (hp-1);
-		cpy_sz = thing_arityval(val);
-		goto cpy_words;
-	    default:
-		cpy_sz = header_arity(val);
-
-	    cpy_words:
-		ASSERT(sz >= cpy_sz);
-		sz -= cpy_sz;
-		while (cpy_sz >= 8) {
-		    cpy_sz -= 8;
-		    *hp++ = *fhp++;
-		    *hp++ = *fhp++;
-		    *hp++ = *fhp++;
-		    *hp++ = *fhp++;
-		    *hp++ = *fhp++;
-		    *hp++ = *fhp++;
-		    *hp++ = *fhp++;
-		    *hp++ = *fhp++;
-		}
-		switch (cpy_sz) {
-		case 7: *hp++ = *fhp++;
-		case 6: *hp++ = *fhp++;
-		case 5: *hp++ = *fhp++;
-		case 4: *hp++ = *fhp++;
-		case 3: *hp++ = *fhp++;
-		case 2: *hp++ = *fhp++;
-		case 1: *hp++ = *fhp++;
-		default: break;
-		}
-		if (oh) {
-		    /* Add to offheap list */
-		    oh->next = off_heap->first;
-		    off_heap->first = oh;
-		    ASSERT(*hpp <= (Eterm*)oh);
-		    ASSERT(hp > (Eterm*)oh);
-		    oh = NULL;
-		}
-		break;
-	    }
-	    break;
-	}
-    }
-
-    ASSERT(bp->used_size == hp - *hpp);
-    *hpp = hp;
-
-    if (is_not_immed(token)) {
-	ASSERT(in_heapfrag(ptr_val(token), bp));
-	ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
-#ifdef HARD_DEBUG
-	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
-	ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg)));
-#endif
-    }
-
-    if (is_not_immed(term)) {
-	ASSERT(in_heapfrag(ptr_val(term),bp));
-	ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
-#ifdef HARD_DEBUG
-	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
-	ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg)));
-#endif
-    }
-#ifdef USE_VM_PROBES
-    if (is_not_immed(utag)) {
-	ASSERT(in_heapfrag(ptr_val(utag), bp));
-	ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs);
-#ifdef HARD_DEBUG
-	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
-	ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
-#endif
-    }
-#endif
-
-copy_done:
-
-#ifdef HARD_DEBUG
-    {
-	int i, j;
-	ErlHeapFragment* frag;
-	{
-	    struct erl_off_heap_header* dbg_oh = off_heap->first;
-	    i = j = 0;
-	    while (dbg_oh != dbg_oh_start) {
-		dbg_oh = dbg_oh->next;
-		i++;
-	    }
-	    for (frag=bp; frag; frag=frag->next) {
-		dbg_oh = frag->off_heap.first;
-		while (dbg_oh) {
-		    dbg_oh = dbg_oh->next;
-		    j++;
-		}
-	    }
-	    ASSERT(i == j);
-	}
-    }
-#endif
-	    
-
-    bp->off_heap.first = NULL;
-    free_message_buffer(bp);
-    msg->data.heap_frag = NULL;
-
-#ifdef HARD_DEBUG
-    ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
-    ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token));
-#ifdef USE_VM_PROBES
-    ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag));
-#endif
-    free_message_buffer(dbg_bp);
-#endif
-
-}
-
-
 Uint
-erts_msg_attached_data_size_aux(ErlMessage *msg)
+erts_msg_attached_data_size_aux(ErtsMessage *msg)
 {
     Sint sz;
     ASSERT(is_non_value(ERL_MESSAGE_TERM(msg)));
@@ -833,29 +570,72 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
     return sz;
 }
 
-void
-erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory,
-				    ErlMessage *msg)
+ErtsMessage *
+erts_try_alloc_message_on_heap(Process *pp,
+			       erts_aint32_t *psp,
+			       ErtsProcLocks *plp,
+			       Uint sz,
+			       Eterm **hpp,
+			       ErlOffHeap **ohpp,
+			       int *on_heap_p)
 {
-    if (is_value(ERL_MESSAGE_TERM(msg)))
-	erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg);
-    else if (msg->data.dist_ext) {
-	ASSERT(msg->data.dist_ext->heap_size >= 0);
-	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
-	    ErlHeapFragment *heap_frag;
-	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
-	    ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg),
-						 heap_frag->used_size,
-						 &factory->hp,
-						 factory->off_heap);
-	    erts_cleanup_offheap(&heap_frag->off_heap);
+#ifdef ERTS_SMP
+    int locked_main = 0;
+#endif
+    ErtsMessage *mp;
+
+    ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ));
+
+    if (
+#if defined(ERTS_SMP)
+	*plp & ERTS_PROC_LOCK_MAIN
+#else
+	1
+#endif
+	) {
+#ifdef ERTS_SMP
+    try_on_heap:
+#endif
+	if ((*psp & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+	    || (pp->flags & F_DISABLE_GC)
+	    || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) {
+	    /*
+	     * The heap is either potentially in an inconsistent
+	     * state, or not large enough.
+	     */
+#ifdef ERTS_SMP
+	    if (locked_main) {
+		*plp &= ~ERTS_PROC_LOCK_MAIN;
+		erts_smp_proc_unlock(pp, ERTS_PROC_LOCK_MAIN);
+	    }
+#endif
+	    goto in_message_fragment;
 	}
-	ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory,
-						     msg->data.dist_ext);
-	erts_free_dist_ext_copy(msg->data.dist_ext);
-	msg->data.dist_ext = NULL;
+
+	*hpp = HEAP_TOP(pp);
+	HEAP_TOP(pp) = *hpp + sz;
+	*ohpp = &MSO(pp);
+	mp = erts_alloc_message(0, NULL);
+	mp->data.attached = NULL;
+	*on_heap_p = !0;
+    }
+#ifdef ERTS_SMP
+    else if (erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
+	locked_main = 1;
+	*psp = erts_smp_atomic32_read_nob(&pp->state);
+	*plp |= ERTS_PROC_LOCK_MAIN;
+	goto try_on_heap;
+    }
+#endif
+    else {
+    in_message_fragment:
+
+	mp = erts_alloc_message(sz, hpp);
+	*ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
+	*on_heap_p = 0;
     }
-    /* else: bad external detected when calculating size */
+
+    return mp;
 }
 
 /*
@@ -870,7 +650,8 @@ erts_send_message(Process* sender,
 		  unsigned flags)
 {
     Uint msize;
-    ErlHeapFragment* bp = NULL;
+    ErtsMessage* mp;
+    ErlOffHeap *ohp;
     Eterm token = NIL;
     Sint res = 0;
 #ifdef USE_VM_PROBES
@@ -879,27 +660,31 @@ erts_send_message(Process* sender,
     Sint tok_label = 0;
     Sint tok_lastcnt = 0;
     Sint tok_serial = 0;
+    Eterm utag = NIL;
 #endif
+    erts_aint32_t receiver_state;
     BM_STOP_TIMER(system);
     BM_MESSAGE(message,sender,receiver);
     BM_START_TIMER(send);
 
  #ifdef USE_VM_PROBES
     *sender_name = *receiver_name = '\0';
-   if (DTRACE_ENABLED(message_send)) {
+    if (DTRACE_ENABLED(message_send)) {
         erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
 		      "%T", sender->common.id);
         erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
 		      "%T", receiver->common.id);
     }
 #endif
+
+    receiver_state = erts_smp_atomic32_read_nob(&receiver->state);
+
     if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
         Eterm* hp;
 	Eterm stoken = SEQ_TRACE_TOKEN(sender);
 	Uint seq_trace_size = 0;
 #ifdef USE_VM_PROBES
 	Uint dt_utag_size = 0;
-	Eterm utag = NIL;
 #endif
 
 	BM_SWAP_TIMER(send,size);
@@ -923,23 +708,32 @@ erts_send_message(Process* sender,
 	}
 #endif
 
-	bp = new_message_buffer(msize + seq_trace_size 
+	mp = erts_alloc_message_heap_state(receiver,
+					   &receiver_state,
+					   receiver_locks,
+					   (msize
 #ifdef USE_VM_PROBES
-				+ dt_utag_size
+					    + dt_utag_size
 #endif
-				);
-	hp = bp->mem;
+					    + seq_trace_size),
+					   &hp,
+					   &ohp);
 
         BM_SWAP_TIMER(send,copy);
-	token = copy_struct(stoken,
-			    seq_trace_size,
-			    &hp,
-			    &bp->off_heap);
+	if (is_immed(stoken))
+	    token = stoken;
+	else
+	    token = copy_struct(stoken, seq_trace_size, &hp, ohp);
+
+	if (is_not_immed(message))
+	    message = copy_struct(message, msize, &hp, ohp);
 
-	message = copy_struct(message, msize, &hp, &bp->off_heap);
 #ifdef USE_VM_PROBES
 	if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
-	    utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap);
+	    if (is_immed(DT_UTAG(sender)))
+		utag = DT_UTAG(sender);
+	    else
+		utag = copy_struct(DT_UTAG(sender), dt_utag_size, ohp);
 #ifdef DTRACE_TAG_HARDDEBUG
 	    erts_fprintf(stderr,
 			 "Dtrace -> (%T) Spreading tag (%T) with "
@@ -961,101 +755,49 @@ erts_send_message(Process* sender,
 		    msize, tok_label, tok_lastcnt, tok_serial);
         }
 #endif
-        res = queue_message(NULL,
-			    receiver,
-			    receiver_locks,
-			    NULL,
-			    bp,
-			    message,
-			    token
-#ifdef USE_VM_PROBES
-			    , utag
-#endif
-	    );
-        BM_SWAP_TIMER(send,system);
-    } else if (sender == receiver) {
-	/* Drop message if receiver has a pending exit ... */
-#ifdef ERTS_SMP
-	ErtsProcLocks need_locks = (~(*receiver_locks)
-				    & (ERTS_PROC_LOCK_MSGQ
-				       | ERTS_PROC_LOCK_STATUS));
-	if (need_locks) {
-	    *receiver_locks |= need_locks;
-	    if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) {
-		if (need_locks == ERTS_PROC_LOCK_MSGQ) {
-		    erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
-		    need_locks = ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS;
-		}
-		erts_smp_proc_lock(receiver, need_locks);
-	    }
-	}
-	if (!ERTS_PROC_PENDING_EXIT(receiver))
-#endif
-	{
-	    ErlMessage* mp = message_alloc();
-
-            DTRACE6(message_send, sender_name, receiver_name,
-                    size_object(message), tok_label, tok_lastcnt, tok_serial);
-	    mp->data.attached = NULL;
-	    ERL_MESSAGE_TERM(mp) = message;
-	    ERL_MESSAGE_TOKEN(mp) = NIL;
-#ifdef USE_VM_PROBES
-	    ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
-	    mp->next = NULL;
-	    /*
-	     * We move 'in queue' to 'private queue' and place
-	     * message at the end of 'private queue' in order
-	     * to ensure that the 'in queue' doesn't contain
-	     * references into the heap. By ensuring this,
-	     * we don't need to include the 'in queue' in
-	     * the root set when garbage collecting.
-	     */
-	    
-	    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
-	    LINK_MESSAGE_PRIVQ(receiver, mp);
-
-	    res = receiver->msg.len;
-
-	    if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
-		trace_receive(receiver, message);
-	    }
-	}
-        BM_SWAP_TIMER(send,system);
     } else {
-	ErlOffHeap *ohp;
         Eterm *hp;
-	erts_aint32_t state;
 
-	BM_SWAP_TIMER(send,size);
-	msize = size_object(message);
-	BM_SWAP_TIMER(size,send);
-	hp = erts_alloc_message_heap_state(msize,
-					   &bp,
-					   &ohp,
-					   receiver,
-					   receiver_locks,
-					   &state);
-	BM_SWAP_TIMER(send,copy);
-	message = copy_struct(message, msize, &hp, ohp);
-	BM_MESSAGE_COPIED(msz);
-	BM_SWAP_TIMER(copy,send);
+	if (receiver == sender && !(receiver_state & ERTS_PSFLG_OFF_HEAP_MSGQ)) {
+	    mp = erts_alloc_message(0, NULL);
+	    msize = 0;
+	}
+	else {
+	    BM_SWAP_TIMER(send,size);
+	    msize = size_object(message);
+	    BM_SWAP_TIMER(size,send);
+
+	    mp = erts_alloc_message_heap_state(receiver,
+					       &receiver_state,
+					       receiver_locks,
+					       msize,
+					       &hp,
+					       &ohp);
+	    BM_SWAP_TIMER(send,copy);
+	    if (is_not_immed(message))
+		message = copy_struct(message, msize, &hp, ohp);
+	    BM_MESSAGE_COPIED(msz);
+	    BM_SWAP_TIMER(copy,send);
+	}
         DTRACE6(message_send, sender_name, receiver_name,
                 msize, tok_label, tok_lastcnt, tok_serial);
-	res = queue_message(sender,
-			    receiver,
-			    receiver_locks,
-			    &state,
-			    bp,
-			    message,
-			    token
+    }
+
+    res = queue_message(sender,
+			receiver,
+			&receiver_state,
+			receiver_locks,
+			mp,
+			message,
+			token
 #ifdef USE_VM_PROBES
-			    , NIL
+			, utag
 #endif
-	    );
-        BM_SWAP_TIMER(send,system);
-    }
-   return res;
+	);
+
+    BM_SWAP_TIMER(send,system);
+    
+    return res;
 }
 
 /*
@@ -1075,7 +817,8 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
     Uint sz_from;
     Eterm* hp;
     Eterm temptoken;
-    ErlHeapFragment* bp = NULL;
+    ErtsMessage* mp;
+    ErlOffHeap *ohp;
 
     if (token != NIL
 #ifdef USE_VM_PROBES
@@ -1087,36 +830,483 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	sz_reason = size_object(reason);
 	sz_token = size_object(token);
 	sz_from = size_object(from);
-	bp = new_message_buffer(sz_reason + sz_from + sz_token + 4);
-	hp = bp->mem;
-	mess = copy_struct(reason, sz_reason, &hp, &bp->off_heap);
-	from_copy = copy_struct(from, sz_from, &hp, &bp->off_heap);
+	mp = erts_alloc_message_heap(to, to_locksp,
+				     sz_reason + sz_from + sz_token + 4,
+				     &hp, &ohp);
+	mess = copy_struct(reason, sz_reason, &hp, ohp);
+	from_copy = copy_struct(from, sz_from, &hp, ohp);
 	save = TUPLE3(hp, am_EXIT, from_copy, mess);
 	hp += 4;
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL);
-	temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap);
-	erts_queue_message(to, to_locksp, bp, save, temptoken);
+	temptoken = copy_struct(token, sz_token, &hp, ohp);
+	erts_queue_message(to, to_locksp, mp, save, temptoken);
     } else {
-	ErlOffHeap *ohp;
 	sz_reason = size_object(reason);
 	sz_from = IS_CONST(from) ? 0 : size_object(from);
 
-	hp = erts_alloc_message_heap(sz_reason+sz_from+4,
-				     &bp,
-				     &ohp,
-				     to,
-				     to_locksp);
+	mp = erts_alloc_message_heap(to, to_locksp,
+				     sz_reason+sz_from+4, &hp, &ohp);
 
 	mess = copy_struct(reason, sz_reason, &hp, ohp);
 	from_copy = (IS_CONST(from)
 		     ? from
 		     : copy_struct(from, sz_from, &hp, ohp));
 	save = TUPLE3(hp, am_EXIT, from_copy, mess);
-	erts_queue_message(to, to_locksp, bp, save, NIL);
+	erts_queue_message(to, to_locksp, mp, save, NIL);
     }
 }
 
+void erts_save_message_in_proc(Process *p, ErtsMessage *msgp)
+{
+    ErlHeapFragment *hfp;
+
+    if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
+	hfp = &msgp->hfrag;
+    else if (msgp->data.attached) {
+	hfp = msgp->data.heap_frag;
+    }
+    else {
+	erts_free_message(msgp);
+	return; /* Nothing to save */
+    }
+
+    while (1) {
+	struct erl_off_heap_header *ohhp = hfp->off_heap.first;
+	if (ohhp) {
+	    for ( ; ohhp->next; ohhp = ohhp->next)
+		;
+	    ohhp->next = p->off_heap.first;
+	    p->off_heap.first = hfp->off_heap.first;
+	    hfp->off_heap.first = NULL;
+	}
+	p->off_heap.overhead += hfp->off_heap.overhead;
+	hfp->off_heap.overhead = 0;
+	p->mbuf_sz += hfp->used_size;
+
+	if (!hfp->next)
+	    break;
+	hfp = hfp->next;
+    }
+
+    msgp->next = p->msg_frag;
+    p->msg_frag = msgp;
+}
+
+Sint
+erts_move_messages_off_heap(Process *c_p)
+{
+    int reds = 1;
+    /*
+     * Move all messages off heap. This *only* occurs when the
+     * process had off heap message disabled and just enabled
+     * it...
+     */
+    ErtsMessage *mp;
+
+    reds += c_p->msg.len / 10;
+
+    ASSERT(erts_smp_atomic32_read_nob(&c_p->state)
+	   & ERTS_PSFLG_OFF_HEAP_MSGQ);
+    ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
+
+    for (mp = c_p->msg.first; mp; mp = mp->next) {
+	Uint msg_sz, token_sz;
+#ifdef USE_VM_PROBES
+	Uint utag_sz;
+#endif
+	Eterm *hp;
+	ErlHeapFragment *hfrag;
+
+	if (mp->data.attached)
+	    continue;
+
+	if (is_immed(ERL_MESSAGE_TERM(mp))
+#ifdef USE_VM_PROBES
+	    && is_immed(ERL_MESSAGE_DT_UTAG(mp))
+#endif
+	    && is_not_immed(ERL_MESSAGE_TOKEN(mp)))
+	    continue;
+
+	/*
+	 * The message refers into the heap. Copy the message
+	 * from the heap into a heap fragment and attach
+	 * it to the message...
+	 */
+	msg_sz = size_object(ERL_MESSAGE_TERM(mp));
+#ifdef USE_VM_PROBES
+	utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp));
+#endif
+	token_sz = size_object(ERL_MESSAGE_TOKEN(mp));
+
+	hfrag = new_message_buffer(msg_sz
+#ifdef USE_VM_PROBES
+				   + utag_sz
+#endif
+				   + token_sz);
+	hp = hfrag->mem;
+	if (is_not_immed(ERL_MESSAGE_TERM(mp)))
+	    ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp),
+					       msg_sz, &hp,
+					       &hfrag->off_heap);
+	if (is_not_immed(ERL_MESSAGE_TOKEN(mp)))
+	    ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
+						token_sz, &hp,
+						&hfrag->off_heap);
+#ifdef USE_VM_PROBES
+	if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp)))
+	    ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp),
+						  utag_sz, &hp,
+						  &hfrag->off_heap);
+#endif
+	mp->data.heap_frag = hfrag;
+	reds += 1;
+    }
+
+    return reds;
+}
+
+Sint
+erts_complete_off_heap_message_queue_change(Process *c_p)
+{
+    int reds = 1;
+
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+    ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
+    ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ);
+
+    /*
+     * This job was first initiated when the process changed
+     * "off heap message queue" state from false to true. Since
+     * then ERTS_PSFLG_OFF_HEAP_MSGQ has been set. However, the
+     * state change might have been changed again (multiple times)
+     * since then. Check users last requested state (the flag
+     * F_OFF_HEAP_MSGQ), and make the state consistent with that.
+     */
+
+    if (!(c_p->flags & F_OFF_HEAP_MSGQ))
+	erts_smp_atomic32_read_band_nob(&c_p->state,
+					~ERTS_PSFLG_OFF_HEAP_MSGQ);
+    else {
+	reds += 2;
+	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+	reds += erts_move_messages_off_heap(c_p);
+    }
+    c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG;
+    return reds;
+}
+
+typedef struct {
+    Eterm pid;
+    ErtsThrPrgrLaterOp lop;
+} ErtsChangeOffHeapMessageQueue;
+
+static void
+change_off_heap_msgq(void *vcohmq)
+{
+    ErtsChangeOffHeapMessageQueue *cohmq;
+    /*
+     * Now we've waited thread progress which ensures that all
+     * messages to the process are enqueued off heap. Schedule
+     * completion of this change as a system task on the process
+     * itself. This in order to avoid lock contention on its
+     * main lock. We will be called in
+     * erts_complete_off_heap_message_queue_change() (above) when
+     * the system task has been selected for execution.
+     */
+    cohmq = (ErtsChangeOffHeapMessageQueue *) vcohmq;
+    erts_schedule_complete_off_heap_message_queue_change(cohmq->pid);
+    erts_free(ERTS_ALC_T_MSGQ_CHNG, vcohmq);
+}
+
+Eterm
+erts_change_off_heap_message_queue_state(Process *c_p, int enable)
+{
+
+#ifdef DEBUG
+    if (c_p->flags & F_OFF_HEAP_MSGQ) {
+	ASSERT(erts_smp_atomic32_read_nob(&c_p->state)
+	       & ERTS_PSFLG_OFF_HEAP_MSGQ);
+    }
+    else {
+	if (c_p->flags & F_OFF_HEAP_MSGQ_CHNG) {
+	    ASSERT(erts_smp_atomic32_read_nob(&c_p->state)
+		   & ERTS_PSFLG_OFF_HEAP_MSGQ);
+	}
+	else {
+	    ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state)
+		     & ERTS_PSFLG_OFF_HEAP_MSGQ));
+	}
+    }
+#endif
+
+    if (c_p->flags & F_OFF_HEAP_MSGQ) {
+	/* Off heap message queue is enabled */
+
+	if (!enable) {
+	    c_p->flags &= ~F_OFF_HEAP_MSGQ;
+	    /*
+	     * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ
+	     * if a change is ongoing. It will be adjusted when the
+	     * change completes...
+	     */
+	    if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
+		/* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */
+		erts_smp_atomic32_read_band_nob(&c_p->state,
+						~ERTS_PSFLG_OFF_HEAP_MSGQ);
+	    }
+	}
+
+	return am_true; /* Old state */
+    }
+
+    /* Off heap message queue is disabled */
+
+    if (enable) {
+	c_p->flags |= F_OFF_HEAP_MSGQ;
+	/*
+	 * We do not have to schedule a change if
+	 * we have an ongoing change...
+	 */
+	if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
+	    ErtsChangeOffHeapMessageQueue *cohmq;
+	    /*
+	     * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait
+	     * thread progress before completing the change in
+	     * order to ensure that all senders observe that
+	     * messages should be passed off heap. When the
+	     * change has completed, GC does not need to inspect
+	     * the message queue at all.
+	     */
+	    erts_smp_atomic32_read_bor_nob(&c_p->state,
+					   ERTS_PSFLG_OFF_HEAP_MSGQ);
+	    c_p->flags |= F_OFF_HEAP_MSGQ_CHNG;
+	    cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG,
+			       sizeof(ErtsChangeOffHeapMessageQueue));
+	    cohmq->pid = c_p->common.id;
+	    erts_schedule_thr_prgr_later_op(change_off_heap_msgq,
+					    (void *) cohmq,
+					    &cohmq->lop);
+	}
+    }
+
+    return am_false; /* Old state */
+}
+
+int
+erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks,
+			 ErtsMessage *msgp, int force_off_heap)
+{
+    ErtsHeapFactory factory;
+    Eterm msg;
+    ErlHeapFragment *bp;
+    Sint need;
+    int decode_in_heap_frag;
+
+    decode_in_heap_frag = (force_off_heap
+			   || !(proc_locks & ERTS_PROC_LOCK_MAIN)
+			   || (proc->flags & F_OFF_HEAP_MSGQ));
+
+    if (msgp->data.dist_ext->heap_size >= 0)
+	need = msgp->data.dist_ext->heap_size;
+    else {
+	need = erts_decode_dist_ext_size(msgp->data.dist_ext);
+	if (need < 0) {
+	    /* bad msg; remove it... */
+	    if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
+		bp = erts_dist_ext_trailer(msgp->data.dist_ext);
+		erts_cleanup_offheap(&bp->off_heap);
+	    }
+	    erts_free_dist_ext_copy(msgp->data.dist_ext);
+	    msgp->data.dist_ext = NULL;
+	    return 0;
+	}
+
+	msgp->data.dist_ext->heap_size = need;
+    }
+
+    if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
+	bp = erts_dist_ext_trailer(msgp->data.dist_ext);
+	need += bp->used_size;
+    }
+
+    if (decode_in_heap_frag)
+	erts_factory_heap_frag_init(&factory, new_message_buffer(need));
+    else
+	erts_factory_proc_prealloc_init(&factory, proc, need);
+
+    ASSERT(msgp->data.dist_ext->heap_size >= 0);
+    if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) {
+	ErlHeapFragment *heap_frag;
+	heap_frag = erts_dist_ext_trailer(msgp->data.dist_ext);
+	ERL_MESSAGE_TOKEN(msgp) = copy_struct(ERL_MESSAGE_TOKEN(msgp),
+					      heap_frag->used_size,
+					      &factory.hp,
+					      factory.off_heap);
+	erts_cleanup_offheap(&heap_frag->off_heap);
+    }
+
+    msg = erts_decode_dist_ext(&factory, msgp->data.dist_ext);
+    ERL_MESSAGE_TERM(msgp) = msg;
+    erts_free_dist_ext_copy(msgp->data.dist_ext);
+    msgp->data.attached = NULL;
+
+    if (is_non_value(msg)) {
+	erts_factory_undo(&factory);
+	return 0;
+    }
+
+    erts_factory_trim_and_close(&factory, msgp->m,
+				ERL_MESSAGE_REF_ARRAY_SZ);
+
+    ASSERT(!msgp->data.heap_frag);
+
+    if (decode_in_heap_frag)
+	msgp->data.heap_frag = factory.heap_frags;
+
+    return 1;
+}
+
+/*
+ * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0 will move off heap messages
+ * into the heap of the inspected process if off_heap_message_queue
+ * is false when process_info(_, messages) is called. That is, the
+ * following GC will have more data in the rootset compared to the
+ * scenario when process_info(_, messages) had not been called.
+ *
+ * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS != 0 will keep off heap messages
+ * off heap when process_info(_, messages) is called regardless of
+ * the off_heap_message_queue setting of the process. That is, it
+ * will change the following execution of the process as little as
+ * possible.
+ */
+#define ERTS_INSPECT_MSGQ_KEEP_OH_MSGS 1
+
+Uint
+erts_prep_msgq_for_inspection(Process *c_p, Process *rp,
+			      ErtsProcLocks rp_locks, ErtsMessageInfo *mip)
+{
+    Uint tot_heap_size;
+    ErtsMessage* mp;
+    Sint i;
+    int self_on_heap;
+    
+    /*
+     * Prepare the message queue for inspection
+     * by process_info().
+     *
+     *
+     * - Decode all messages on external format
+     * - Remove all corrupt dist messages from queue
+     * - Save pointer to, and heap size need of each
+     *   message in the mip array.
+     * - Return total heap size need for all messages
+     *   that needs to be copied.
+     *
+     * If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0:
+     * - In case off heap messages is disabled and
+     *   we are inspecting our own queue, move all
+     *   off heap data into the heap.
+     */
+
+    self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ);
+
+    tot_heap_size = 0;
+    i = 0;
+    mp = rp->msg.first;
+    while (mp) {
+	Eterm msg = ERL_MESSAGE_TERM(mp);
+
+	mip[i].size = 0;
+
+	if (is_non_value(msg)) {
+	    /* Dist message on external format; decode it... */
+	    if (mp->data.attached)
+		erts_decode_dist_message(rp, rp_locks, mp,
+					 ERTS_INSPECT_MSGQ_KEEP_OH_MSGS);
+
+	    msg = ERL_MESSAGE_TERM(mp);
+
+	    if (is_non_value(msg)) {
+		ErtsMessage **mpp;
+		ErtsMessage *bad_mp = mp;
+		/*
+		 * Bad distribution message; remove
+		 * it from the queue...
+		 */
+		ASSERT(!mp->data.attached);
+
+		mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
+
+		if (rp->msg.save == &bad_mp->next)
+		    rp->msg.save = mpp;
+		if (rp->msg.last == &bad_mp->next)
+		    rp->msg.last = mpp;
+		mp = mp->next;
+		*mpp = mp;
+		rp->msg.len--;
+		bad_mp->next = NULL;
+		erts_cleanup_messages(bad_mp);
+		continue;
+	    }
+	}
+
+	ASSERT(is_value(msg));
+
+#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS
+	if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) {
+	    Uint sz = size_object(msg);
+	    mip[i].size = sz;
+	    tot_heap_size += sz;
+	}
+#else
+	if (self_on_heap) {
+	    if (mp->data.attached) {
+		ErtsMessage *tmp = NULL;
+		if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
+		    erts_link_mbuf_to_proc(rp, mp->data.heap_frag);
+		    mp->data.attached = NULL;
+		}
+		else {
+		    /*
+		     * Need to replace the message reference since
+		     * we will get references to the message data
+		     * from the heap...
+		     */
+		    ErtsMessage **mpp;
+		    tmp = erts_alloc_message(0, NULL);
+		    sys_memcpy((void *) tmp->m, (void *) mp->m,
+			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); 
+		    mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
+		    tmp->next = mp->next;
+		    if (rp->msg.save == &mp->next)
+			rp->msg.save = &tmp->next;
+		    if (rp->msg.last == &mp->next)
+			rp->msg.last = &tmp->next;
+		    *mpp = tmp;
+		    erts_save_message_in_proc(rp, mp);
+		    mp = tmp;
+		}
+	    }
+	}
+	else if (is_not_immed(msg)) {
+	    Uint sz = size_object(msg);
+	    mip[i].size = sz;
+	    tot_heap_size += sz;
+	}
+
+#endif
+
+	mip[i].msgp = mp;
+	i++;
+	mp = mp->next;
+    }
+
+    return tot_heap_size;
+}
+
 void erts_factory_proc_init(ErtsHeapFactory* factory,
 			    Process* p)
 {
@@ -1127,47 +1317,138 @@ void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory,
 				     Process* p,
 				     Sint size)
 {
+    ErlHeapFragment *bp = p->mbuf;
     factory->mode     = FACTORY_HALLOC;
     factory->p        = p;
     factory->hp_start = HAlloc(p, size);
     factory->hp       = factory->hp_start;
     factory->hp_end   = factory->hp_start + size;
     factory->off_heap = &p->off_heap;
+    factory->message  = NULL;
     factory->off_heap_saved.first    = p->off_heap.first;
     factory->off_heap_saved.overhead = p->off_heap.overhead;
-    factory->heap_frags_saved = p->mbuf;
+    factory->heap_frags_saved = bp;
+    factory->heap_frags_saved_used = bp ? bp->used_size : 0;
     factory->heap_frags = NULL; /* not used */
     factory->alloc_type = 0; /* not used */
 }
 
-void erts_factory_message_init(ErtsHeapFactory* factory,
-			       Process* rp,
-			       Eterm* hp,
-			       ErlHeapFragment* bp)
+void erts_factory_heap_frag_init(ErtsHeapFactory* factory,
+				 ErlHeapFragment* bp)
+{
+    factory->mode     = FACTORY_HEAP_FRAGS;
+    factory->p        = NULL;
+    factory->hp_start = bp->mem;
+    factory->hp       = bp->mem;
+    factory->hp_end   = bp->mem + bp->alloc_size;
+    factory->off_heap = &bp->off_heap;
+    factory->message  = NULL;
+    factory->heap_frags = bp;
+    factory->heap_frags_saved = NULL;
+    factory->heap_frags_saved_used = 0;
+    factory->alloc_type = ERTS_ALC_T_HEAP_FRAG;
+    ASSERT(!bp->next);
+    factory->off_heap_saved.first    = factory->off_heap->first;
+    factory->off_heap_saved.overhead = factory->off_heap->overhead;
+
+    ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end);
+}
+
+
+ErtsMessage *
+erts_factory_message_create(ErtsHeapFactory* factory,
+			    Process *proc,
+			    ErtsProcLocks *proc_locksp,
+			    Uint sz)
+{
+    Eterm *hp;
+    ErlOffHeap *ohp;
+    ErtsMessage *msgp;
+    int on_heap;
+    erts_aint32_t state;
+
+    state = erts_smp_atomic32_read_nob(&proc->state);
+
+    if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) {
+	msgp = erts_alloc_message(sz, &hp);
+	ohp = sz == 0 ? NULL : &msgp->hfrag.off_heap;
+	on_heap = 0;
+    }
+    else {
+	msgp = erts_try_alloc_message_on_heap(proc, &state,
+					      proc_locksp,
+					      sz, &hp, &ohp,
+					      &on_heap);
+    }
+
+    if (on_heap) {
+	ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN);
+	ASSERT(ohp == &proc->off_heap);
+	factory->mode = FACTORY_HALLOC;
+	factory->p = proc;
+	factory->heap_frags_saved = proc->mbuf;
+	factory->heap_frags_saved_used = proc->mbuf ? proc->mbuf->used_size : 0;
+    }
+    else {
+	factory->mode = FACTORY_MESSAGE;
+	factory->p = NULL;
+	factory->heap_frags_saved = NULL;
+	factory->heap_frags_saved_used = 0;
+
+	if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) {
+	    ASSERT(!msgp->hfrag.next);
+	    factory->heap_frags = NULL;
+	}
+	else {
+	    ASSERT(!msgp->data.heap_frag
+		   || !msgp->data.heap_frag->next);
+	    factory->heap_frags = msgp->data.heap_frag;
+	}
+    }
+    factory->hp_start = hp;
+    factory->hp       = hp;
+    factory->hp_end   = hp + sz;
+    factory->message  = msgp;
+    factory->off_heap = ohp;
+    factory->alloc_type = ERTS_ALC_T_HEAP_FRAG;
+    if (ohp) {
+	factory->off_heap_saved.first    = ohp->first;
+	factory->off_heap_saved.overhead = ohp->overhead;
+    }
+    else {
+	factory->off_heap_saved.first    = NULL;
+	factory->off_heap_saved.overhead = 0;
+    }
+
+    ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end);
+
+    return msgp;
+}
+
+void erts_factory_selfcontained_message_init(ErtsHeapFactory* factory,
+					     ErtsMessage *msgp,
+					     Eterm *hp)
 {
-    if (bp) {
-	factory->mode     = FACTORY_HEAP_FRAGS;
-	factory->p        = NULL;
-	factory->hp_start = bp->mem;
-	factory->hp       = hp ? hp : bp->mem;
-	factory->hp_end   = bp->mem + bp->alloc_size;
-	factory->off_heap = &bp->off_heap;
-        factory->heap_frags = bp;
-        factory->heap_frags_saved = bp;
-        factory->alloc_type = ERTS_ALC_T_HEAP_FRAG;
-        ASSERT(!bp->next);
+    ErlHeapFragment* bp;
+    if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) {
+	bp = &msgp->hfrag;
+	factory->heap_frags = NULL;
     }
     else {
-	factory->mode     = FACTORY_HALLOC;
-	factory->p        = rp;
-	factory->hp_start = hp;
-	factory->hp       = hp;
-	factory->hp_end   = HEAP_TOP(rp);
-	factory->off_heap = &rp->off_heap;
-        factory->heap_frags_saved = rp->mbuf;
-        factory->heap_frags = NULL; /* not used */
-        factory->alloc_type = 0; /* not used */
+	bp = msgp->data.heap_frag;
+	factory->heap_frags = bp;
     }
+    factory->mode     = FACTORY_MESSAGE;
+    factory->p        = NULL;
+    factory->hp_start = bp->mem;
+    factory->hp       = hp;
+    factory->hp_end   = bp->mem + bp->alloc_size;
+    factory->message  = msgp;
+    factory->off_heap = &bp->off_heap;
+    factory->heap_frags_saved = NULL;
+    factory->heap_frags_saved_used = 0;
+    factory->alloc_type = ERTS_ALC_T_HEAP_FRAG;
+    ASSERT(!bp->next);
     factory->off_heap_saved.first    = factory->off_heap->first;
     factory->off_heap_saved.overhead = factory->off_heap->overhead;
 
@@ -1230,8 +1511,16 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
 	factory->hp_end = factory->hp + need;
 	return;
 
-    case FACTORY_HEAP_FRAGS:
-	bp = factory->heap_frags;
+    case FACTORY_MESSAGE:
+	if (!factory->heap_frags) {
+	    ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG);
+	    bp = &factory->message->hfrag;
+	}
+	else {
+	    /* Fall through */
+	case FACTORY_HEAP_FRAGS:
+	    bp = factory->heap_frags;
+	}
 
         if (bp) {
 	    ASSERT(factory->hp > bp->mem);
@@ -1269,8 +1558,23 @@ void erts_factory_close(ErtsHeapFactory* factory)
 	HRelease(factory->p, factory->hp_end, factory->hp);
 	break;
 
-    case FACTORY_HEAP_FRAGS:
-	bp = factory->heap_frags;
+    case FACTORY_MESSAGE:
+	if (!factory->heap_frags) {
+	    if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG)
+		bp = &factory->message->hfrag;
+	    else
+		bp = NULL;
+	}
+	else {
+	    if (factory->message->data.attached	== ERTS_MSG_COMBINED_HFRAG)
+		factory->message->hfrag.next = factory->heap_frags;
+	    else
+		factory->message->data.heap_frag = factory->heap_frags;
+
+	    /* Fall through */
+	case FACTORY_HEAP_FRAGS:
+	    bp = factory->heap_frags;
+	}
 
 	if (bp) {
 	    ASSERT(factory->hp >= bp->mem);
@@ -1291,17 +1595,47 @@ void erts_factory_close(ErtsHeapFactory* factory)
 void erts_factory_trim_and_close(ErtsHeapFactory* factory,
 				 Eterm *brefs, Uint brefs_size)
 {
-    if (factory->mode == FACTORY_HEAP_FRAGS) {
-        ErlHeapFragment* bp = factory->heap_frags;
+    ErlHeapFragment *bp;
+
+    switch (factory->mode) {
+    case FACTORY_MESSAGE: {
+	ErtsMessage *mp = factory->message;
+	if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) {
+	    if (!mp->hfrag.next) {
+		Uint sz = factory->hp - factory->hp_start;
+		mp = erts_shrink_message(mp, sz, brefs, brefs_size);
+		factory->message = mp;
+		factory->mode = FACTORY_CLOSED;
+		return;
+	    }
+	    /*else we don't trim multi fragmented messages for now (off_heap...) */
+	    break;
+	}
+	/* Fall through... */
+    }
+    case FACTORY_HEAP_FRAGS:
+	bp = factory->heap_frags;
+	if (!bp)
+	    break;
         if (bp->next == NULL) {
             Uint used_sz = factory->hp - bp->mem;
             ASSERT(used_sz <= bp->alloc_size);
-            factory->heap_frags = erts_resize_message_buffer(bp, used_sz,
-                                                             brefs, brefs_size);
+	    if (used_sz > 0)
+		bp = erts_resize_message_buffer(bp, used_sz,
+						brefs, brefs_size);
+	    else {
+		free_message_buffer(bp);
+		bp = NULL;
+	    }
+	    factory->heap_frags = bp;
+	    if (factory->mode == FACTORY_MESSAGE)
+		factory->message->data.heap_frag = bp;
             factory->mode = FACTORY_CLOSED;
             return;
         }
-        /*else we don't trim multi fragmented messages for now */
+        /*else we don't trim multi fragmented messages for now (off_heap...) */
+    default:
+	break;
     }
     erts_factory_close(factory);
 }
@@ -1349,38 +1683,35 @@ void erts_factory_undo(ErtsHeapFactory* factory)
 
             /* Rollback heap top
 	     */
-            if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */
-                ASSERT(factory->hp_start >= HEAP_START(factory->p));
-                ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p));
 
-                HEAP_TOP(factory->p) = factory->hp_start;
-            }
-            else {
+	    if (HEAP_START(factory->p) <= factory->hp_start
+		&& factory->hp_start <= HEAP_LIMIT(factory->p)) {
+		HEAP_TOP(factory->p) = factory->hp_start;
+	    }
+
+	    /* Fix last heap frag */
+            if (factory->heap_frags_saved) {
                 ASSERT(factory->heap_frags_saved == factory->p->mbuf);
-                if (factory->hp_start == factory->heap_frags_saved->mem) {
+                if (factory->hp_start != factory->heap_frags_saved->mem)
+                    factory->heap_frags_saved->used_size = factory->heap_frags_saved_used;
+		else {
                     factory->p->mbuf = factory->p->mbuf->next;
                     ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
                                    ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
                 }
-                else if (factory->hp_start != factory->hp_end) {
-                    unsigned remains = factory->hp_start - factory->heap_frags_saved->mem;
-                    ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size);
-                    factory->heap_frags_saved->used_size = remains;
-                }
             }
         }
         break;
 
+    case FACTORY_MESSAGE:
+	if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG)
+	    factory->message->hfrag.next = factory->heap_frags;
+	else
+	    factory->message->data.heap_frag = factory->heap_frags;
+	erts_cleanup_messages(factory->message);
+	break;
     case FACTORY_HEAP_FRAGS:
-        bp = factory->heap_frags;
-        do {
-            ErlHeapFragment* next_bp = bp->next;
-
-            erts_cleanup_offheap(&bp->off_heap);
-            ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
-                           ERTS_HEAP_FRAG_SIZE(bp->size));
-            bp = next_bp;
-        }while (bp != NULL);
+	free_message_buffer(factory->heap_frags);
 	break;
 
     case FACTORY_CLOSED: break;
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index f37b430d27..740ae46a0f 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -24,6 +24,8 @@
 struct proc_bin;
 struct external_thing_;
 
+typedef struct erl_mesg ErtsMessage;
+
 /*
  * This struct represents data that must be updated by structure copy,
  * but is stored outside of any heap.
@@ -54,6 +56,7 @@ typedef struct {
     enum {
         FACTORY_CLOSED = 0,
         FACTORY_HALLOC,
+        FACTORY_MESSAGE,
         FACTORY_HEAP_FRAGS,
         FACTORY_STATIC
     } mode;
@@ -61,8 +64,10 @@ typedef struct {
     Eterm* hp_start;
     Eterm* hp;
     Eterm* hp_end;
+    ErtsMessage *message;
     struct erl_heap_fragment* heap_frags;
     struct erl_heap_fragment* heap_frags_saved;
+    Uint heap_frags_saved_used;
     ErlOffHeap* off_heap;
     ErlOffHeap off_heap_saved;
     Uint32 alloc_type;
@@ -70,7 +75,10 @@ typedef struct {
 
 void erts_factory_proc_init(ErtsHeapFactory*, Process*);
 void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size);
-void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*);
+void erts_factory_heap_frag_init(ErtsHeapFactory*, struct erl_heap_fragment*);
+ErtsMessage *erts_factory_message_create(ErtsHeapFactory *, Process *,
+					  ErtsProcLocks *, Uint sz);
+void erts_factory_selfcontained_message_init(ErtsHeapFactory*, ErtsMessage *, Eterm *);
 void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*);
 void erts_factory_dummy_init(ErtsHeapFactory*);
 
@@ -91,6 +99,8 @@ void erts_factory_undo(ErtsHeapFactory*);
 #include "external.h"
 #include "erl_process.h"
 
+#define ERTS_INVALID_HFRAG_PTR ((ErlHeapFragment *) ~((UWord) 7))
+
 /*
  * This struct represents a heap fragment, which is used when there
  * isn't sufficient room in the process heap and we can't do a GC.
@@ -105,33 +115,46 @@ struct erl_heap_fragment {
     Eterm mem[1];		/* Data */
 };
 
-typedef struct erl_mesg {
-    struct erl_mesg* next;	/* Next message */
-    union {
-	ErtsDistExternal *dist_ext;
-	ErlHeapFragment *heap_frag;
-	void *attached;
-    } data;
-#ifdef USE_VM_PROBES
-    Eterm m[3];                 /* m[0] = message, m[1] = seq trace token, m[3] = dynamic trace user tag */
-#else
-    Eterm m[2];			/* m[0] = message, m[1] = seq trace token */
-#endif
-} ErlMessage;
-
+/* m[0] = message, m[1] = seq trace token */
+#define ERL_MESSAGE_REF_ARRAY_SZ 2
 #define ERL_MESSAGE_TERM(mp) ((mp)->m[0])
 #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1])
+
 #ifdef USE_VM_PROBES
+/* m[2] = dynamic trace user tag */
+#undef ERL_MESSAGE_REF_ARRAY_SZ
+#define ERL_MESSAGE_REF_ARRAY_SZ 3
 #define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2])
+#else
 #endif
 
+#define ERL_MESSAGE_REF_FIELDS__			\
+    ErtsMessage *next;	/* Next message */		\
+    union {						\
+	ErtsDistExternal *dist_ext;			\
+	ErlHeapFragment *heap_frag;			\
+	void *attached;					\
+    } data;						\
+    Eterm m[ERL_MESSAGE_REF_ARRAY_SZ]
+
+
+typedef struct erl_msg_ref__ {
+    ERL_MESSAGE_REF_FIELDS__;
+} ErtsMessageRef;
+
+struct erl_mesg {
+    ERL_MESSAGE_REF_FIELDS__;
+
+    ErlHeapFragment hfrag;
+};
+
 /* Size of default message buffer (erl_message.c) */
 #define ERL_MESSAGE_BUF_SZ 500
 
 typedef struct {
-    ErlMessage* first;
-    ErlMessage** last;  /* point to the last next pointer */
-    ErlMessage** save;
+    ErtsMessage* first;
+    ErtsMessage** last;  /* point to the last next pointer */
+    ErtsMessage** save;
     Sint len;            /* queue length */
 
     /*
@@ -139,14 +162,14 @@ typedef struct {
      * recv_set/1 instructions.
      */
     BeamInstr* mark;		/* address to rec_loop/2 instruction */
-    ErlMessage** saved_last;	/* saved last pointer */
+    ErtsMessage** saved_last;	/* saved last pointer */
 } ErlMessageQueue;
 
 #ifdef ERTS_SMP
 
 typedef struct {
-    ErlMessage* first;
-    ErlMessage** last;  /* point to the last next pointer */
+    ErtsMessage* first;
+    ErtsMessage** last;  /* point to the last next pointer */
     Sint len;            /* queue length */
 } ErlMessageInQueue;
 
@@ -197,7 +220,7 @@ do {									\
 
 /* Unlink current message */
 #define UNLINK_MESSAGE(p,msgp) do { \
-     ErlMessage* __mp = (msgp)->next; \
+     ErtsMessage* __mp = (msgp)->next; \
      *(p)->msg.save = __mp; \
      (p)->msg.len--; \
      if (__mp == NULL) \
@@ -213,76 +236,33 @@ do {									\
 #define SAVE_MESSAGE(p) \
      (p)->msg.save = &(*(p)->msg.save)->next
 
-/*
- * ErtsMoveMsgAttachmentIntoProc() moves data attached to a message
- * onto the heap of a process. The attached data is the content of
- * the the message either on the internal format or on the external
- * format, and also possibly a seq trace token on the internal format.
- * If the message content is on the external format, the decode might
- * fail. If the decoding fails, ERL_MESSAGE_TERM(M) will contain
- * THE_NON_VALUE. That is, ERL_MESSAGE_TERM(M) *has* to be checked
- * afterwards and taken care of appropriately.
- *
- * ErtsMoveMsgAttachmentIntoProc() will shallow copy to heap if
- * possible; otherwise, move to heap via garbage collection.
- *
- * ErtsMoveMsgAttachmentIntoProc() is used when receiveing messages
- * in process_main() and in hipe_check_get_msg().
- */
-
-#define ErtsMoveMsgAttachmentIntoProc(M, P, ST, HT, FC, SWPO, SWPI)	\
-do {									\
-    if ((M)->data.attached) {						\
-	Uint need__ = erts_msg_attached_data_size((M));			\
-        { SWPO ; }							\
- 	if ((ST) - (HT) >= need__) {					\
-            ErtsHeapFactory factory__;                                  \
-            erts_factory_proc_prealloc_init(&factory__, (P), need__);   \
-	    erts_move_msg_attached_data_to_heap(&factory__, (M));       \
-            erts_factory_close(&factory__);                             \
-            if ((P)->mbuf != NULL) {                                    \
-                /* Heap was exhausted by messages. This is a rare case */   \
-                /* that can currently (OTP 18) only happen if hamts are */  \
-                /* far exceeding the estimated heap size. Do GC. */         \
-                (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
-            }								\
-	}								\
-	else {								\
-	    (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
-	}								\
-        { SWPI ; }							\
-	ASSERT(!(M)->data.attached);					\
-    }									\
-} while (0)
-
 #define ERTS_SND_FLG_NO_SEQ_TRACE		(((unsigned) 1) << 0)
 
 #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
    (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))
 
-#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS)	\
-do {							\
-    (HEAP_FRAG_P)->next = NULL;				\
-    (HEAP_FRAG_P)->alloc_size = (DATA_WORDS);		\
-    (HEAP_FRAG_P)->used_size = (DATA_WORDS);            \
-    (HEAP_FRAG_P)->off_heap.first = NULL; 	        \
-    (HEAP_FRAG_P)->off_heap.overhead = 0;		\
-} while (0)
+#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, USED_WORDS, DATA_WORDS)	\
+    do {								\
+	(HEAP_FRAG_P)->next = NULL;					\
+	(HEAP_FRAG_P)->alloc_size = (DATA_WORDS);			\
+	(HEAP_FRAG_P)->used_size = (USED_WORDS);			\
+	(HEAP_FRAG_P)->off_heap.first = NULL;				\
+	(HEAP_FRAG_P)->off_heap.overhead = 0;				\
+    } while (0)
 
 void init_message(void);
-void free_message(ErlMessage *);
 ErlHeapFragment* new_message_buffer(Uint);
 ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
 					    Eterm *, Uint);
 void free_message_buffer(ErlHeapFragment *);
 void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm);
 #ifdef USE_VM_PROBES
-void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*,
+void erts_queue_message_probe(Process*, ErtsProcLocks*, ErtsMessage*,
                               Eterm message, Eterm seq_trace_token, Eterm dt_utag);
 #define erts_queue_message(RP,RL,BP,Msg,SEQ) \
     erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL)
 #else
-void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*,
+void erts_queue_message(Process*, ErtsProcLocks*, ErtsMessage*,
                         Eterm message, Eterm seq_trace_token);
 #define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \
     erts_queue_message((RP),(RL),(BP),(Msg),(SEQ))
@@ -291,20 +271,141 @@ void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
 Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
 
-void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *);
-
-Uint erts_msg_attached_data_size_aux(ErlMessage *msg);
-void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *);
-Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **,
-			    Eterm *, ErtsDistExternal *);
+Uint erts_msg_attached_data_size_aux(ErtsMessage *msg);
 
 void erts_cleanup_offheap(ErlOffHeap *offheap);
+void erts_save_message_in_proc(Process *p, ErtsMessage *msg);
+Sint erts_move_messages_off_heap(Process *c_p);
+Sint erts_complete_off_heap_message_queue_change(Process *c_p);
+Eterm erts_change_off_heap_message_queue_state(Process *c_p, int enable);
+
+int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int);
+
+void erts_cleanup_messages(ErtsMessage *mp);
+
+typedef struct {
+    Uint size;
+    ErtsMessage *msgp;
+} ErtsMessageInfo;
+
+Uint erts_prep_msgq_for_inspection(Process *c_p,
+				   Process *rp,
+				   ErtsProcLocks rp_locks,
+				   ErtsMessageInfo *mip);
 
+void *erts_alloc_message_ref(void);
+void erts_free_message_ref(void *);
 
+#define ERTS_SMALL_FIX_MSG_SZ 10
+#define ERTS_MEDIUM_FIX_MSG_SZ 20
+#define ERTS_LARGE_FIX_MSG_SZ 30
+
+void *erts_alloc_small_message(void);
+void erts_free_small_message(void *mp);
+
+typedef struct {
+    ErtsMessage m;
+    Eterm data[ERTS_SMALL_FIX_MSG_SZ-1];
+} ErtsSmallFixSzMessage;
+
+typedef struct {
+    ErtsMessage m;
+    Eterm data[ERTS_MEDIUM_FIX_MSG_SZ-1];
+} ErtsMediumFixSzMessage;
+
+typedef struct {
+    ErtsMessage m;
+    Eterm data[ERTS_LARGE_FIX_MSG_SZ-1];
+} ErtsLargeFixSzMessage;
+
+ErtsMessage *erts_try_alloc_message_on_heap(Process *pp,
+					    erts_aint32_t *psp,
+					    ErtsProcLocks *plp,
+					    Uint sz,
+					    Eterm **hpp,
+					    ErlOffHeap **ohpp,
+					    int *on_heap_p);
+ErtsMessage *erts_realloc_shrink_message(ErtsMessage *mp, Uint sz,
+					 Eterm *brefs, Uint brefs_size);
+
+ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp);
+ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz,
+						       Eterm *brefs, Uint brefs_size);
+ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp);
 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
-ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg);
+ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg);
+
+#define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1)
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp)
+{
+    ErtsMessage *mp;
+
+    if (sz == 0) {
+	mp = erts_alloc_message_ref();
+	mp->next = NULL;
+	ERL_MESSAGE_TERM(mp) = NIL;
+	mp->data.attached = NULL;
+	if (hpp)
+	    *hpp = NULL;
+	return mp;
+    }
+
+    mp = erts_alloc(ERTS_ALC_T_MSG,
+		    sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm));
+
+    mp->next = NULL;
+    ERL_MESSAGE_TERM(mp) = NIL;
+    mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
+    ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz);
+
+    if (hpp)
+	*hpp = &mp->hfrag.mem[0];
+
+    return mp;
+}
+
+ERTS_GLB_FORCE_INLINE ErtsMessage *
+erts_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size)
+{
+    if (sz == 0) {
+	ErtsMessage *nmp;
+	if (!mp->data.attached)
+	    return mp;
+	ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG);
+	nmp = erts_alloc_message_ref();
+#ifdef DEBUG
+	if (brefs && brefs_size) {
+	    int i;
+	    for (i = 0; i < brefs_size; i++)
+		ASSERT(is_non_value(brefs[i]) || is_immed(brefs[i]));
+	}
+#endif
+	erts_free(ERTS_ALC_T_MSG, mp);
+	return nmp;
+    }
+
+    ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG);
+    ASSERT(mp->hfrag.used_size >= sz);
+
+    if (sz >= (mp->hfrag.alloc_size - mp->hfrag.alloc_size / 16)) {
+	mp->hfrag.used_size = sz;
+	return mp;
+    }
+
+    return erts_realloc_shrink_message(mp, sz, brefs, brefs_size);
+}
+
+ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp)
+{
+    if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
+	erts_free_message_ref(mp);
+    else
+	erts_free(ERTS_ALC_T_MSG, mp);
+}
+
 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp)
 {
     Uint sz = 0;
@@ -314,11 +415,17 @@ ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp)
     return sz;
 }
 
-ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
+ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg)
 {
     ASSERT(msg->data.attached);
-    if (is_value(ERL_MESSAGE_TERM(msg)))
-	return erts_used_frag_sz(msg->data.heap_frag);
+    if (is_value(ERL_MESSAGE_TERM(msg))) {
+	ErlHeapFragment *bp;
+	if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG)
+	    bp = &msg->hfrag;
+	else
+	    bp = msg->data.heap_frag;
+	return erts_used_frag_sz(bp);
+    }
     else if (msg->data.dist_ext->heap_size < 0)
 	return erts_msg_attached_data_size_aux(msg);
     else {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 01414f326d..a37cda93ef 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -314,6 +314,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     ErtsProcLocks rp_locks = 0;
     Process* rp;
     Process* c_p;
+    ErtsMessage *mp;
     ErlHeapFragment* frags;
     Eterm receiver = to_pid->pid;
     int flush_me = 0;
@@ -347,7 +348,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     ASSERT(frags == MBUF(&menv->phony_proc));
     if (frags != NULL) {
 	/* Move all offheap's from phony proc to the first fragment.
-	   Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */
+	   Quick and dirty... */
 	ASSERT(!is_offheap(&frags->off_heap));
 	frags->off_heap = MSO(&menv->phony_proc);
 	clear_offheap(&MSO(&menv->phony_proc));
@@ -359,7 +360,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     if (flush_me) {	
 	flush_env(env); /* Needed for ERTS_HOLE_CHECK */ 
     }
-    erts_queue_message(rp, &rp_locks, frags, msg, am_undefined);
+    mp = erts_alloc_message(0, NULL);
+    mp->data.heap_frag = frags;
+    erts_queue_message(rp, &rp_locks, mp, msg, am_undefined);
     if (c_p == rp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
     if (rp_locks)
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 62a44f7129..a4da288e79 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1401,56 +1401,50 @@ setup_reference_table(void)
     for (i = 0; i < max; i++) {
 	Process *proc = erts_pix2proc(i);
 	if (proc) {
-	    ErlMessage *msg;
+	    int mli;
+	    ErtsMessage *msg_list[] = {
+		proc->msg.first,
+#ifdef ERTS_SMP
+		proc->msg_inq.first,
+#endif
+		proc->msg_frag};
 
 	    /* Insert Heap */
 	    insert_offheap(&(proc->off_heap),
 			   HEAP_REF,
 			   proc->common.id);
-	    /* Insert message buffers */
+	    /* Insert heap fragments buffers */
 	    for(hfp = proc->mbuf; hfp; hfp = hfp->next)
 		insert_offheap(&(hfp->off_heap),
 			       HEAP_REF,
 			       proc->common.id);
-	    /* Insert msg msg buffers */
-	    for (msg = proc->msg.first; msg; msg = msg->next) {
-		ErlHeapFragment *heap_frag = NULL;
-		if (msg->data.attached) {
-		    if (is_value(ERL_MESSAGE_TERM(msg)))
-			heap_frag = msg->data.heap_frag;
-		    else {
-			if (msg->data.dist_ext->dep)
-			    insert_dist_entry(msg->data.dist_ext->dep,
-					      HEAP_REF, proc->common.id, 0);
-			if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
-			    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
+
+	    /* Insert msg buffers */
+	    for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) {
+		ErtsMessage *msg;
+		for (msg = msg_list[mli]; msg; msg = msg->next) {
+		    ErlHeapFragment *heap_frag = NULL;
+		    if (msg->data.attached) {
+			if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG)
+			    heap_frag = &msg->hfrag;
+			else if (is_value(ERL_MESSAGE_TERM(msg)))
+			    heap_frag = msg->data.heap_frag;
+			else {
+			    if (msg->data.dist_ext->dep)
+				insert_dist_entry(msg->data.dist_ext->dep,
+						  HEAP_REF, proc->common.id, 0);
+			    if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
+				heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
+			}
 		    }
-		}
-		if (heap_frag)
-		    insert_offheap(&(heap_frag->off_heap),
-				   HEAP_REF,
-				   proc->common.id);
-	    }
-#ifdef ERTS_SMP
-	    for (msg = proc->msg_inq.first; msg; msg = msg->next) {
-		ErlHeapFragment *heap_frag = NULL;
-		if (msg->data.attached) {
-		    if (is_value(ERL_MESSAGE_TERM(msg)))
-			heap_frag = msg->data.heap_frag;
-		    else {
-			if (msg->data.dist_ext->dep)
-			    insert_dist_entry(msg->data.dist_ext->dep,
-					      HEAP_REF, proc->common.id, 0);
-			if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
-			    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
+		    while (heap_frag) {
+			insert_offheap(&(heap_frag->off_heap),
+				       HEAP_REF,
+				       proc->common.id);
+			heap_frag = heap_frag->next;
 		    }
 		}
-		if (heap_frag)
-		    insert_offheap(&(heap_frag->off_heap),
-				   HEAP_REF,
-				   proc->common.id);
 	    }
-#endif
 	    /* Insert links */
 	    if (ERTS_P_LINKS(proc))
 		insert_links(ERTS_P_LINKS(proc), proc->common.id);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index bad9da90ea..e490ffea5a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -148,6 +148,7 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
+int erts_default_spo_flags = 0;
 int erts_eager_check_io = 1;
 int erts_sched_compact_load;
 int erts_sched_balance_util = 0;
@@ -351,7 +352,8 @@ struct erts_system_profile_flags_t erts_system_profile_flags;
 
 typedef enum {
     ERTS_PSTT_GC,	/* Garbage Collect */
-    ERTS_PSTT_CPC	/* Check Process Code */
+    ERTS_PSTT_CPC,	/* Check Process Code */
+    ERTS_PSTT_COHMQ     /* Change off heap message queue */
 } ErtsProcSysTaskType;
 
 #define ERTS_MAX_PROC_SYS_TASK_ARGS 2
@@ -982,7 +984,7 @@ reply_sched_wall_time(void *vswtrp)
     Eterm **hpp;
     Uint sz, *szp;
     ErlOffHeap *ohp = NULL;
-    ErlHeapFragment *bp = NULL;
+    ErtsMessage *mp = NULL;
 
     ASSERT(esdp);
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -1038,12 +1040,12 @@ reply_sched_wall_time(void *vswtrp)
 	if (hpp)
 	    break;
 
-	hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+	mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
 	szp = NULL;
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+    erts_queue_message(rp, &rp_locks, mp, msg, NIL);
 
     if (swtrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -6294,22 +6296,99 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks)
     schedule_process(p, state, locks);
 }
 
-static void
-schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
+static int
+schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st)
 {
-    /*
-     * Expects status lock to be locked when called, and
-     * returns with status lock unlocked...
-     */
-    erts_aint32_t a = state, n, enq_prio = -1;
+    int res;
+    int locked;
+    ErtsProcSysTaskQs *stqs, *free_stqs;
+    erts_aint32_t state, a, n, enq_prio;
     int enqueue; /* < 0 -> use proxy */
-    unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs;
+    unsigned int prof_runnable_procs;
+
+    res = 1; /* prepare for success */
+    st->next = st->prev = st; /* Prep for empty prio queue */
+    state = erts_smp_atomic32_read_nob(&p->state);
+    prof_runnable_procs = erts_system_profile_flags.runnable_procs;
+    locked = 0;
+    free_stqs = NULL;
+    if (state & ERTS_PSFLG_ACTIVE_SYS)
+	stqs = NULL;
+    else {
+    alloc_qs:
+	stqs = proc_sys_task_queues_alloc();
+	stqs->qmask = 1 << prio;
+	stqs->ncount = 0;
+	stqs->q[PRIORITY_MAX] = NULL;
+	stqs->q[PRIORITY_HIGH] = NULL;
+	stqs->q[PRIORITY_NORMAL] = NULL;
+	stqs->q[PRIORITY_LOW] = NULL;
+	stqs->q[prio] = st;
+    }
+
+    if (!locked) {
+	locked = 1;
+	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+
+	state = erts_smp_atomic32_read_nob(&p->state);
+	if (state & ERTS_PSFLG_EXITING) {
+	    free_stqs = stqs;
+	    res = 0;
+	    goto cleanup;
+	}
+    }
+
+    if (!p->sys_task_qs) {
+	if (stqs)
+	    p->sys_task_qs = stqs;
+	else
+	    goto alloc_qs;
+    }
+    else {
+	free_stqs = stqs;
+	stqs = p->sys_task_qs;
+	if (!stqs->q[prio]) {
+	    stqs->q[prio] = st;
+	    stqs->qmask |= 1 << prio;
+	}
+	else {
+	    st->next = stqs->q[prio];
+	    st->prev = stqs->q[prio]->prev;
+	    st->next->prev = st;
+	    st->prev->next = st;
+	    ASSERT(stqs->qmask & (1 << prio));
+	}
+    }
+
+    if (ERTS_PSFLGS_GET_ACT_PRIO(state) > prio) {
+	erts_aint32_t n, a, e;
+	/* Need to elevate actual prio */
+
+	a = state;
+	do {
+	    if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) {
+		n = a;
+		break;
+	    }
+	    n = e = a;
+	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
+	    n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
+	    a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e);    
+	} while (a != e);
+	state = n;
+    }
+
+
+    a = state;
+    enq_prio = -1;
 
     /* Status lock prevents out of order "runnable proc" trace msgs */
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    if (!prof_runnable_procs)
+    if (!prof_runnable_procs) {
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	locked = 0;
+    }
 
     ASSERT(!(state & ERTS_PSFLG_PROXY));
 
@@ -6317,8 +6396,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 	erts_aint32_t e;
 	n = e = a;
 
-	if (a & ERTS_PSFLG_FREE)
+	if (a & ERTS_PSFLG_FREE) {
+	    res = 0;
 	    goto cleanup; /* We don't want to schedule free processes... */
+	}
 
 	enqueue = ERTS_ENQUEUE_NOT;
 	n |= ERTS_PSFLG_ACTIVE_SYS;
@@ -6342,29 +6423,24 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 	}
 
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	prof_runnable_procs = 0;
+	locked = 0;
     }
 
-    if (enqueue != ERTS_ENQUEUE_NOT) {
-	Process *sched_p;
-	if (enqueue > 0)
-	    sched_p = p;
-	else {
-	    sched_p = make_proxy_proc(proxy, p, enq_prio);
-	    proxy = NULL;
-	}
-	add2runq(sched_p, n, enq_prio);
-    }
+    if (enqueue != ERTS_ENQUEUE_NOT)
+	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
+		 n, enq_prio);
 
 cleanup:
 
-    if (prof_runnable_procs)
+    if (locked)
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
-    if (proxy)
-	free_proxy_proc(proxy);
+    if (free_stqs)
+	proc_sys_task_queues_free(free_stqs);
 
     ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
+
+    return res;
 }
 
 static ERTS_INLINE int
@@ -9696,13 +9772,13 @@ Process *schedule(Process *p, int calls)
 	    }
 	}
 
-	if (!(state & ERTS_PSFLG_EXITING)
-	    && ((FLAGS(p) & F_FORCE_GC)
-		|| (MSO(p).overhead > BIN_VHEAP_SZ(p)))) {
-	    reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity);
-	    if (reds <= 0) {
-		p->fcalls = reds;
-		goto sched_out_proc;
+	if (ERTS_IS_GC_DESIRED(p)) {
+	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) {
+		reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity);
+		if (reds <= 0) {
+		    p->fcalls = reds;
+		    goto sched_out_proc;
+		}
 	    }
 	}
 
@@ -9742,7 +9818,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
     if (rp) {
 	ErtsProcLocks rp_locks;
 	ErlOffHeap *ohp;
-	ErlHeapFragment* bp;
+	ErtsMessage *mp;
 	Eterm *hp, msg, req_id, result;
 	Uint st_result_sz, hsz;
 #ifdef DEBUG
@@ -9754,11 +9830,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 	st_result_sz = is_immed(st_result) ? 0 : size_object(st_result);
 	hsz = st->req_id_sz + st_result_sz + 4 /* 3-tuple */;
 
-	hp = erts_alloc_message_heap(hsz,
-				     &bp,
-				     &ohp,
-				     rp,
-				     &rp_locks);
+	mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp);
 
 #ifdef DEBUG
 	hp_start = hp;
@@ -9783,7 +9855,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 	ASSERT(hp_start + hsz == hp);
 #endif
 
-	erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+	erts_queue_message(rp, &rp_locks, mp, msg, NIL);
 
 	if (c_p == rp)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -10005,6 +10077,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 		st = NULL;
 	    }
 	    break;
+	case ERTS_PSTT_COHMQ:
+	    reds += erts_complete_off_heap_message_queue_change(c_p);
+	    st_res = am_true;
+	    break;
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
 	    st_res = am_false;
@@ -10047,6 +10123,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	case ERTS_PSTT_CPC:
 	    st_res = am_false;
 	    break;
+	case ERTS_PSTT_COHMQ:
+	    st_res = am_false;
+	    break;
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
 	    st_res = am_false;
@@ -10065,10 +10144,8 @@ BIF_RETTYPE
 erts_internal_request_system_task_3(BIF_ALIST_3)
 {
     Process *rp = erts_proc_lookup(BIF_ARG_1);
-    ErtsProcSysTaskQs *stqs, *free_stqs = NULL;
     ErtsProcSysTask *st = NULL;
-    erts_aint32_t prio, rp_state;
-    int rp_locked;
+    erts_aint32_t prio;
     Eterm noproc_res, req_type;
 
     if (!rp && !is_internal_pid(BIF_ARG_1)) {
@@ -10125,7 +10202,6 @@ erts_internal_request_system_task_3(BIF_ALIST_3)
 	}
 	st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK,
 			ERTS_PROC_SYS_TASK_SIZE(tot_sz));
-	st->next = st->prev = st; /* Prep for empty prio queue */
 	ERTS_INIT_OFF_HEAP(&st->off_heap);
 	hp = &st->heap[0];
 
@@ -10169,95 +10245,11 @@ erts_internal_request_system_task_3(BIF_ALIST_3)
 	goto badarg;
     }
 
-    rp_state = erts_smp_atomic32_read_nob(&rp->state);
-
-    rp_locked = 0;
-
-    free_stqs = NULL;
-    if (rp_state & ERTS_PSFLG_ACTIVE_SYS)
-	stqs = NULL;
-    else {
-    alloc_qs:
-	stqs = proc_sys_task_queues_alloc();
-	stqs->qmask = 1 << prio;
-	stqs->ncount = 0;
-	stqs->q[PRIORITY_MAX] = NULL;
-	stqs->q[PRIORITY_HIGH] = NULL;
-	stqs->q[PRIORITY_NORMAL] = NULL;
-	stqs->q[PRIORITY_LOW] = NULL;
-	stqs->q[prio] = st;
-    }
-
-    if (!rp_locked) {
-	rp_locked = 1;
-	erts_smp_proc_lock(rp, ERTS_PROC_LOCK_STATUS);
-
-	rp_state = erts_smp_atomic32_read_nob(&rp->state);
-	if (rp_state & ERTS_PSFLG_EXITING) {
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-	    rp = NULL;
-	    free_stqs = stqs;
-	    goto noproc;
-	}
+    if (!schedule_process_sys_task(rp, prio, st)) {
+    noproc:
+	notify_sys_task_executed(BIF_P, st, noproc_res);
     }
 
-    if (!rp->sys_task_qs) {
-	if (stqs)
-	    rp->sys_task_qs = stqs;
-	else
-	    goto alloc_qs;
-    }
-    else {
-	if (stqs)
-	    free_stqs = stqs;
-	stqs = rp->sys_task_qs;
-	if (!stqs->q[prio]) {
-	    stqs->q[prio] = st;
-	    stqs->qmask |= 1 << prio;
-	}
-	else {
-	    st->next = stqs->q[prio];
-	    st->prev = stqs->q[prio]->prev;
-	    st->next->prev = st;
-	    st->prev->next = st;
-	    ASSERT(stqs->qmask & (1 << prio));
-	}
-    }
-
-    if (ERTS_PSFLGS_GET_ACT_PRIO(rp_state) > prio) {
-	erts_aint32_t n, a, e;
-	/* Need to elevate actual prio */
-
-	a = rp_state;
-	do {
-	    if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) {
-		n = a;
-		break;
-	    }
-	    n = e = a;
-	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
-	    n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
-	    a = erts_smp_atomic32_cmpxchg_nob(&rp->state, n, e);    
-	} while (a != e);
-	rp_state = n;
-    }
-
-    /*
-     * schedule_process_sys_task() unlocks status
-     * lock on process.
-     */
-    schedule_process_sys_task(rp, rp_state, NULL);
-
-    if (free_stqs)
-	proc_sys_task_queues_free(free_stqs);
-
-    BIF_RET(am_ok);
-
-noproc:
-
-    notify_sys_task_executed(BIF_P, st, noproc_res);
-    if (free_stqs)
-	proc_sys_task_queues_free(free_stqs);
     BIF_RET(am_ok);
 
 badarg:
@@ -10266,11 +10258,35 @@ badarg:
 	erts_cleanup_offheap(&st->off_heap);
 	erts_free(ERTS_ALC_T_PROC_SYS_TSK, st);
     }
-    if (free_stqs)
-	proc_sys_task_queues_free(free_stqs);
     BIF_ERROR(BIF_P, BADARG);
 }
 
+void
+erts_schedule_complete_off_heap_message_queue_change(Eterm pid)
+{
+    Process *rp = erts_proc_lookup(pid);
+    if (rp) {
+	ErtsProcSysTask *st;
+	erts_aint32_t state;
+	int i;
+
+	st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK,
+			ERTS_PROC_SYS_TASK_SIZE(0));
+	st->type = ERTS_PSTT_COHMQ;
+	st->requester = NIL;
+	st->reply_tag = NIL;
+	st->req_id = NIL;
+	st->req_id_sz = 0;
+	for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++)
+	    st->arg[i] = NIL;
+	ERTS_INIT_OFF_HEAP(&st->off_heap);
+	state = erts_smp_atomic32_read_nob(&rp->state);
+
+	if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), st))
+	    erts_free(ERTS_ALC_T_PROC_SYS_TSK, st);
+    }
+}
+
 static void
 save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
 {
@@ -10716,6 +10732,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 		   Eterm args,	/* Arguments for function (must be well-formed list). */
 		   ErlSpawnOpts* so) /* Options for spawn. */
 {
+    Uint flags = erts_default_process_flags;
     ErtsRunQueue *rq = NULL;
     Process *p;
     Sint arity;			/* Number of arguments. */
@@ -10753,6 +10770,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET)
 	      | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET));
 
+    if (so->flags & SPO_OFF_HEAP_MSGQ) {
+	state |= ERTS_PSFLG_OFF_HEAP_MSGQ;
+	flags |= F_OFF_HEAP_MSGQ;
+    }
+
     if (!rq)
 	rq = erts_get_runq_proc(parent);
 
@@ -10775,7 +10797,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     BM_SWAP_TIMER(size,system);
     heap_need = arg_size;
 
-    p->flags = erts_default_process_flags;
+    p->flags = flags;
 
     p->static_flags = 0;
     if (so->flags & SPO_SYSTEM_PROC)
@@ -10824,6 +10846,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->stop = p->hend = p->heap + sz;
     p->htop = p->heap;
     p->heap_sz = sz;
+    p->abandoned_heap = NULL;
+    p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
     p->catches = 0;
 
     p->bin_vheap_sz     = p->min_vheap_size;
@@ -10894,6 +10918,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->accessor_bif_timers = NULL;
 #endif
     p->mbuf = NULL;
+    p->msg_frag = NULL;
     p->mbuf_sz = 0;
     p->psd = NULL;
     p->dictionary = NULL;
@@ -11029,6 +11054,8 @@ void erts_init_empty_process(Process *p)
     p->stop = NULL;
     p->hend = NULL;
     p->heap = NULL;
+    p->abandoned_heap = NULL;
+    p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
     p->gen_gcs = 0;
     p->max_gen_gcs = 0;
     p->min_heap_size = 0;
@@ -11061,6 +11088,7 @@ void erts_init_empty_process(Process *p)
     p->old_htop = NULL;
     p->old_heap = NULL;
     p->mbuf = NULL;
+    p->msg_frag = NULL;
     p->mbuf_sz = 0;
     p->psd = NULL;
     ERTS_P_MONITORS(p) = NULL;
@@ -11149,6 +11177,8 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->htop == NULL);
     ASSERT(p->stop == NULL);
     ASSERT(p->hend == NULL);
+    ASSERT(p->abandoned_heap == NULL);
+    ASSERT(p->live_hf_end == ERTS_INVALID_HFRAG_PTR);
     ASSERT(p->heap == NULL);
     ASSERT(p->common.id == ERTS_INVALID_PID);
     ASSERT(ERTS_TRACER_PROC(p) == NIL);
@@ -11226,8 +11256,6 @@ erts_cleanup_empty_process(Process* p)
 static void
 delete_process(Process* p)
 {
-    ErlMessage* mp;
-
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
 
     /* Cleanup psd */
@@ -11283,24 +11311,8 @@ delete_process(Process* p)
     erts_erase_dicts(p);
 
     /* free all pending messages */
-    mp = p->msg.first;
-    while(mp != NULL) {
-	ErlMessage* next_mp = mp->next;
-	if (mp->data.attached) {
-	    if (is_value(mp->m[0]))
-		free_message_buffer(mp->data.heap_frag);
-	    else {
-		if (is_not_nil(mp->m[1])) {
-		    ErlHeapFragment *heap_frag;
-		    heap_frag = (ErlHeapFragment *) mp->data.dist_ext->ext_endp;
-		    erts_cleanup_offheap(&heap_frag->off_heap);
-		}
-		erts_free_dist_ext_copy(mp->data.dist_ext);
-	    }
-	}
-	free_message(mp);
-	mp = next_mp;
-    }
+    erts_cleanup_messages(p->msg.first);
+    p->msg.first = NULL;
 
     ASSERT(!p->nodes_monitors);
     ASSERT(!p->suspend_monitors);
@@ -11488,6 +11500,9 @@ static ERTS_INLINE void
 send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 		  Eterm exit_term, Uint term_size, Eterm token)
 {
+    ErtsMessage *mp;
+    ErlOffHeap *ohp;
+
     if (token == NIL 
 #ifdef USE_VM_PROBES
 	|| token == am_have_dt_utag
@@ -11495,14 +11510,12 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 	) {
 	Eterm* hp;
 	Eterm mess;
-	ErlHeapFragment* bp;
-	ErlOffHeap *ohp;
 
-	hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp);
+	mp = erts_alloc_message_heap(to, to_locksp,
+				     term_size, &hp, &ohp);
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
-	erts_queue_message(to, to_locksp, bp, mess, NIL);
+	erts_queue_message(to, to_locksp, mp, mess, NIL);
     } else {
-	ErlHeapFragment* bp;
 	Eterm* hp;
 	Eterm mess;
 	Eterm temp_token;
@@ -11510,13 +11523,14 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 
 	ASSERT(is_tuple(token));
 	sz_token = size_object(token);
-	bp = new_message_buffer(term_size+sz_token);
-	hp = bp->mem;
-	mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap);
+
+	mp = erts_alloc_message_heap(to, to_locksp,
+				     term_size+sz_token, &hp, &ohp);
+	mess = copy_struct(exit_term, term_size, &hp, ohp);
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL);
-	temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap);
-	erts_queue_message(to, to_locksp, bp, mess, temp_token);
+	temp_token = copy_struct(token, sz_token, &hp, ohp);
+	erts_queue_message(to, to_locksp, mp, mess, temp_token);
     }
 }
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 65422b8c15..c6376c0166 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -916,6 +916,7 @@ struct process {
     Eterm* stop;		/* Stack top */
     Eterm* heap;		/* Heap start */
     Eterm* hend;		/* Heap end */
+    Eterm* abandoned_heap;
     Uint heap_sz;		/* Size of heap in words */
     Uint min_heap_size;         /* Minimum size of heap (in words). */
     Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */
@@ -1012,8 +1013,10 @@ struct process {
     Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
     Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
     ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
-    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
-    Uint mbuf_sz;		/* Size of all message buffers */
+    ErlHeapFragment* mbuf;	/* Pointer to heap fragment list */
+    ErlHeapFragment* live_hf_end;
+    ErtsMessage *msg_frag;	/* Pointer to message fragment list */
+    Uint mbuf_sz;		/* Total size of heap fragments and message fragments */
     ErtsPSD *psd;		/* Rarely used process specific data */
 
     Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
@@ -1041,6 +1044,7 @@ struct process {
 #ifdef CHECK_FOR_HOLES
     Eterm* last_htop;		/* No need to scan the heap below this point. */
     ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
+    ErlHeapFragment* heap_hfrag; /* Heap abandoned, htop now lives in this frag */
 #endif
 
 #ifdef DEBUG
@@ -1064,6 +1068,7 @@ extern const Process erts_invalid_process;
 do {						\
   (p)->last_htop = 0;				\
   (p)->last_mbuf = 0;				\
+  (p)->heap_hfrag = NULL;			\
 } while (0)
 
 # define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
@@ -1141,14 +1146,15 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
+#define ERTS_PSFLG_OFF_HEAP_MSGQ	ERTS_PSFLG_BIT(18)
 #ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
-#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
-#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
-#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
+#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(19)
+#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(22)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 23)
 #else
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 19)
 #endif
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
@@ -1196,12 +1202,15 @@ void erts_check_for_holes(Process* p);
 #define SPO_USE_ARGS 2
 #define SPO_MONITOR 4
 #define SPO_SYSTEM_PROC 8
+#define SPO_OFF_HEAP_MSGQ 16
+
+extern int erts_default_spo_flags;
 
 /*
  * The following struct contains options for a process to be spawned.
  */
 typedef struct {
-    Uint flags;
+    int flags;
     int error_code;		/* Error code returned from create_process(). */
     Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */
 
@@ -1283,6 +1292,9 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC */
+#define F_OFF_HEAP_MSGQ      (1 << 12) /* Off heap msg queue */
+#define F_OFF_HEAP_MSGQ_CHNG (1 << 13) /* Off heap msg queue changing */
+#define F_ABANDONED_HEAP_USE (1 << 14) /* Have usage of abandoned heap */
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
@@ -1616,6 +1628,7 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
 					     void *,
 					     ErtsThrPrgrLaterOp *,
 					     UWord);
+void erts_schedule_complete_off_heap_message_queue_change(Eterm pid);
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_dbg_check_halloc_lock(Process *p);
@@ -1743,7 +1756,7 @@ Uint erts_debug_nbalance(void);
 
 int erts_debug_wait_completed(Process *c_p, int flags);
 
-Uint erts_process_memory(Process *c_p);
+Uint erts_process_memory(Process *c_p, int incl_msg_inq);
 
 #ifdef ERTS_SMP
 #  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
@@ -2058,6 +2071,22 @@ ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
 ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
 ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
 
+ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap_state(Process *pp,
+							   erts_aint32_t *psp,
+							   ErtsProcLocks *plp,
+							   Uint sz,
+							   Eterm **hpp,
+							   ErlOffHeap **ohpp);
+ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap(Process *pp,
+						     ErtsProcLocks *plp,
+						     Uint sz,
+						     Eterm **hpp,
+						     ErlOffHeap **ohpp);
+
+ERTS_GLB_INLINE void erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp,
+					      Eterm *start_hp, Eterm *used_hp, Eterm *end_hp,
+					      Eterm *brefs, Uint brefs_size);
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE
@@ -2206,6 +2235,63 @@ erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
 #endif
 }
 
+ERTS_GLB_INLINE ErtsMessage *
+erts_alloc_message_heap_state(Process *pp,
+			      erts_aint32_t *psp,
+			      ErtsProcLocks *plp,
+			      Uint sz,
+			      Eterm **hpp,
+			      ErlOffHeap **ohpp)
+{
+    int on_heap;
+
+    if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) {
+	ErtsMessage *mp = erts_alloc_message(sz, hpp);
+	*ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
+	return mp;
+    }
+
+    return erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap);
+}
+
+ERTS_GLB_INLINE ErtsMessage *
+erts_alloc_message_heap(Process *pp,
+			ErtsProcLocks *plp,
+			Uint sz,
+			Eterm **hpp,
+			ErlOffHeap **ohpp)
+{
+    erts_aint32_t state = erts_smp_atomic32_read_nob(&pp->state);
+    return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp);
+}
+
+ERTS_GLB_INLINE void
+erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp,
+			 Eterm *start_hp, Eterm *used_hp, Eterm *end_hp,
+			 Eterm *brefs, Uint brefs_size)
+{
+    ASSERT(start_hp <= used_hp && used_hp <= end_hp);
+    if ((*msgpp)->data.attached == ERTS_MSG_COMBINED_HFRAG)
+	*msgpp = erts_shrink_message(*msgpp, used_hp - start_hp,
+				     brefs, brefs_size);
+    else if (!(*msgpp)->data.attached) {
+	ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN
+			   & erts_proc_lc_my_proc_locks(pp));
+	HRelease(pp, end_hp, used_hp);
+    }
+    else {
+	ErlHeapFragment *hfrag = (*msgpp)->data.heap_frag;
+	if (start_hp != used_hp)
+	    hfrag = erts_resize_message_buffer(hfrag, used_hp - start_hp,
+					       brefs, brefs_size);
+	else {
+	    free_message_buffer(hfrag);
+	    hfrag = NULL;
+	}
+	(*msgpp)->data.heap_frag = hfrag;
+    }
+}
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 3b8ae11e94..71396561a3 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -78,13 +78,14 @@ erts_deep_process_dump(int to, void *to_arg)
     dump_binaries(to, to_arg, all_binaries);
 }
 
-Uint erts_process_memory(Process *p) {
-  ErlMessage *mp;
+Uint erts_process_memory(Process *p, int incl_msg_inq) {
+  ErtsMessage *mp;
   Uint size = 0;
   struct saved_calls *scb;
   size += sizeof(Process);
 
-  ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
+  if (incl_msg_inq)
+      ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
 
   erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size);
   erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size);
@@ -92,7 +93,7 @@ Uint erts_process_memory(Process *p) {
   if (p->old_hend && p->old_heap)
     size += (p->old_hend - p->old_heap) * sizeof(Eterm);
 
-  size += p->msg.len * sizeof(ErlMessage);
+  size += p->msg.len * sizeof(ErtsMessage);
 
   for (mp = p->msg.first; mp; mp = mp->next)
     if (mp->data.attached)
@@ -119,7 +120,7 @@ static void
 dump_process_info(int to, void *to_arg, Process *p)
 {
     Eterm* sp;
-    ErlMessage* mp;
+    ErtsMessage* mp;
     int yreg = -1;
 
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
@@ -657,6 +658,8 @@ erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) {
                 erts_print(to, to_arg, "PROXY"); break;
             case ERTS_PSFLG_DELAYED_SYS:
                 erts_print(to, to_arg, "DELAYED_SYS"); break;
+            case ERTS_PSFLG_OFF_HEAP_MSGQ:
+                erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break;
 #ifdef ERTS_DIRTY_SCHEDULERS
             case ERTS_PSFLG_DIRTY_CPU_PROC:
                 erts_print(to, to_arg, "DIRTY_CPU_PROC"); break;
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 788348e613..a64c993e8f 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -854,9 +854,6 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
 #endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
-	ASSERT(ERTS_AINT_NULL
-	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
-					    internal_pid_index(p->common.id)));
 	erts_free_proc(p);
     }
 }
@@ -872,9 +869,6 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 #endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
-	ASSERT(ERTS_AINT_NULL
-	       == erts_ptab_pix2intptr_ddrb(&erts_proc,
-					    internal_pid_index(p->common.id)));
 	erts_free_proc(p);
     }
 }
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 7327e0b48c..7ec64506e8 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1919,15 +1919,16 @@ send_time_offset_changed_notifications(void *new_offsetp)
 		ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
 		erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
 		if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) {
-		    ErlHeapFragment *bp;
+		    ErtsMessage *mp;
 		    ErlOffHeap *ohp;
 		    Eterm message;
 
-		    hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks);
+		    mp = erts_alloc_message_heap(rp, &rp_locks,
+						 hsz, &hp, &ohp);
 		    *patch_refp = ref;
 		    ASSERT(hsz == size_object(message_template));
 		    message = copy_struct(message_template, hsz, &hp, ohp);
-		    erts_queue_message(rp, &rp_locks, bp, message, NIL);
+		    erts_queue_message(rp, &rp_locks, mp, message, NIL);
 		}
 		erts_smp_proc_unlock(rp, rp_locks);
 	    }
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index e9dd96efc4..d02f1f7213 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -114,15 +114,10 @@ void erts_init_trace(void) {
 
 static Eterm system_seq_tracer;
 
-#ifdef ERTS_SMP
 #define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, UNUSED) \
   (*(BPP) = new_message_buffer((SZ)), \
    *(OHPP) = &(*(BPP))->off_heap, \
    (*(BPP))->mem)
-#else
-#define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, RPP) \
-  erts_alloc_message_heap((SZ), (BPP), (OHPP), (RPP), 0)
-#endif
 
 #ifdef ERTS_SMP
 #define ERTS_ENQ_TRACE_MSG(FPID, TPID, MSG, BP) \
@@ -131,8 +126,12 @@ do { \
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \
 } while(0)
 #else
-#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \
-    erts_queue_message((TPROC), NULL, (BP), (MSG), NIL)
+#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP)		\
+    do {							\
+	ErtsMessage *mp__ = erts_alloc_message(0, NULL);		\
+	mp__->data.heap_frag = (BP);				\
+	erts_queue_message((TPROC), NULL, mp__, (MSG), NIL);	\
+    } while (0)
 #endif
 
 /*
@@ -591,11 +590,9 @@ send_to_port(Process *c_p, Eterm message,
 static void 
 profile_send(Eterm from, Eterm message) {
     Uint sz = 0;
-    ErlHeapFragment *bp = NULL;
     Uint *hp = NULL;
     Eterm msg = NIL;
     Process *profile_p = NULL;
-    ErlOffHeap *off_heap = NULL;
 
     Eterm profiler = erts_get_system_profile();
 
@@ -621,6 +618,7 @@ profile_send(Eterm from, Eterm message) {
 	}
     	
     } else {
+	ErtsMessage *mp;
 	ASSERT(is_internal_pid(profiler));
         
 	profile_p = erts_proc_lookup(profiler);
@@ -629,10 +627,13 @@ profile_send(Eterm from, Eterm message) {
 	    return;
 
 	sz = size_object(message);
-	hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0);
-	msg = copy_struct(message, sz, &hp, &bp->off_heap);
-	
-        erts_queue_message(profile_p, NULL, bp, msg, NIL);
+	mp = erts_alloc_message(sz, &hp);
+	if (sz == 0)
+	    msg = message;
+	else
+	    msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap);
+
+        erts_queue_message(profile_p, NULL, mp, msg, NIL);
     }
 }
 
@@ -1233,7 +1234,11 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	erts_smp_mtx_unlock(&smq_mtx);
 #else
         /* trace_token must be NIL here */
-	erts_queue_message(tracer, NULL, bp, mess, NIL);
+	{
+	    ErtsMessage *mp = erts_alloc_message(0, NULL);
+	    mp->data.heap_frag = bp;
+	    erts_queue_message(tracer, NULL, mp, mess, NIL);
+	}
 #endif
     }
 }
@@ -2308,7 +2313,11 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    {
+	ErtsMessage *mp = erts_alloc_message(0, NULL);
+	mp->data.heap_frag = bp;
+	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+    }
 #endif
 }
 void 
@@ -2369,7 +2378,11 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time)
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    {
+	ErtsMessage *mp = erts_alloc_message(0, NULL);
+	mp->data.heap_frag = bp;
+	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+    }
 #endif
 }
 
@@ -2440,7 +2453,11 @@ monitor_long_gc(Process *p, Uint time) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    {
+	ErtsMessage *mp = erts_alloc_message(0, NULL);
+	mp->data.heap_frag = bp;
+	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+    }
 #endif
 }
 
@@ -2511,7 +2528,11 @@ monitor_large_heap(Process *p) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    {
+	ErtsMessage *mp = erts_alloc_message(0, NULL);
+	mp->data.heap_frag = bp;
+	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+    }
 #endif
 }
 
@@ -2539,7 +2560,11 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    {
+	ErtsMessage *mp = erts_alloc_message(0, NULL);
+	mp->data.heap_frag = bp;
+	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+    }
 #endif
 
 }
@@ -3331,8 +3356,11 @@ sys_msg_dispatcher_func(void *unused)
 		    goto failure;
 		}
 		else {
+		    ErtsMessage *mp;
 		queue_proc_msg:
-		    erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL);
+		    mp = erts_alloc_message(0, NULL);
+		    mp->data.heap_frag = smqp->bp;
+		    erts_queue_message(proc,&proc_locks,mp,smqp->msg,NIL);
 #ifdef DEBUG_PRINTOUTS
 		    erts_fprintf(stderr, "delivered\n");
 #endif
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 052994b972..594c0ccf94 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1274,11 +1274,11 @@ int erts_print_system_version(int to, void *arg, Process *c_p);
 
 int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
 
-ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr);
+ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr)
+ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr)
 {
     ASSERT(is_boxed(tptr) || is_list(tptr));
     ASSERT(ptr == ptr_val(tptr));
@@ -1347,124 +1347,6 @@ extern erts_driver_t fd_driver;
 
 int erts_beam_jump_table(void);
 
-/* Should maybe be placed in erl_message.h, but then we get an include mess. */
-ERTS_GLB_INLINE Eterm *
-erts_alloc_message_heap_state(Uint size,
-			      ErlHeapFragment **bpp,
-			      ErlOffHeap **ohpp,
-			      Process *receiver,
-			      ErtsProcLocks *receiver_locks,
-			      erts_aint32_t *statep);
-
-ERTS_GLB_INLINE Eterm *
-erts_alloc_message_heap(Uint size,
-			ErlHeapFragment **bpp,
-			ErlOffHeap **ohpp,
-			Process *receiver,
-			ErtsProcLocks *receiver_locks);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-/*
- * NOTE: erts_alloc_message_heap() releases msg q and status
- *       lock on receiver without ensuring that other locks are
- *       held. User is responsible to ensure that the receiver
- *       pointer cannot become invalid until after message has
- *       been passed. This is normal done either by increasing
- *       reference count on process (preferred) or by holding
- *       main or link lock over the whole message passing
- *       operation.
- */
-
-ERTS_GLB_INLINE Eterm *
-erts_alloc_message_heap_state(Uint size,
-			      ErlHeapFragment **bpp,
-			      ErlOffHeap **ohpp,
-			      Process *receiver,
-			      ErtsProcLocks *receiver_locks,
-			      erts_aint32_t *statep)
-{
-    Eterm *hp;
-    erts_aint32_t state;
-#ifdef ERTS_SMP
-    int locked_main = 0;
-    state = erts_smp_atomic32_read_acqb(&receiver->state);
-    if (statep)
-	*statep = state;
-    if (state & (ERTS_PSFLG_EXITING
-		 | ERTS_PSFLG_PENDING_EXIT))
-	goto allocate_in_mbuf;
-#endif
-
-    if (size > (Uint) INT_MAX)
-	erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);
-
-    if (
-#if defined(ERTS_SMP)
-	*receiver_locks & ERTS_PROC_LOCK_MAIN
-#else
-	1
-#endif
-	) {
-#ifdef ERTS_SMP
-    try_allocate_on_heap:
-#endif
-	state = erts_smp_atomic32_read_nob(&receiver->state);
-	if (statep)
-	    *statep = state;
-	if ((state & (ERTS_PSFLG_EXITING
-		      | ERTS_PSFLG_PENDING_EXIT))
-	    || (receiver->flags & F_DISABLE_GC)
-	    || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
-	    /*
-	     * The heap is either potentially in an inconsistent
-	     * state, or not large enough.
-	     */
-#ifdef ERTS_SMP
-	    if (locked_main) {
-		*receiver_locks &= ~ERTS_PROC_LOCK_MAIN;
-		erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN);
-	    }
-#endif
-	    goto allocate_in_mbuf;
-	}
-	hp = HEAP_TOP(receiver);
-	HEAP_TOP(receiver) = hp + size;
-	*bpp = NULL;
-	*ohpp = &MSO(receiver);
-    }
-#ifdef ERTS_SMP
-    else if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MAIN) == 0) {
-	locked_main = 1;
-	*receiver_locks |= ERTS_PROC_LOCK_MAIN;
-	goto try_allocate_on_heap;
-    }
-#endif
-    else {
-	ErlHeapFragment *bp;
-    allocate_in_mbuf:
-	bp = new_message_buffer(size);
-	hp = bp->mem;
-	*bpp = bp;
-	*ohpp = &bp->off_heap;
-    }
-
-    return hp;
-}
-
-ERTS_GLB_INLINE Eterm *
-erts_alloc_message_heap(Uint size,
-			ErlHeapFragment **bpp,
-			ErlOffHeap **ohpp,
-			Process *receiver,
-			ErtsProcLocks *receiver_locks)
-{
-    return erts_alloc_message_heap_state(size, bpp, ohpp, receiver,
-					 receiver_locks, NULL);
-}
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-
 #define DeclareTmpHeap(VariableName,Size,Process) \
      Eterm VariableName[Size]
 #define DeclareTypedTmpHeap(Type,VariableName,Process)	\
@@ -1522,6 +1404,7 @@ dtrace_fun_decode(Process *process,
     erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d",
                   module, function, arity);
 }
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* !__GLOBAL_H__ */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index fdd26fcc4b..1b0c617632 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1410,7 +1410,7 @@ queue_port_sched_op_reply(Process *rp,
 
     erts_factory_trim_and_close(factory, &msg, 1);
 
-    erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL);
+    erts_queue_message(rp, rp_locksp, factory->message, msg, NIL);
 }
 
 static void
@@ -1418,12 +1418,9 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
 {
     Process *rp = erts_proc_lookup_raw(to);
     if (rp) {
-	ErlOffHeap *ohp;
-	ErlHeapFragment* bp;
         ErtsHeapFactory factory;
 	Eterm msg_copy;
 	Uint hsz, msg_sz;
-	Eterm *hp;
 	ErtsProcLocks rp_locks = 0;
 
 	hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
@@ -1434,18 +1431,13 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
 	    hsz += msg_sz;
 	}
 
-	hp = erts_alloc_message_heap(hsz,
-						&bp,
-						&ohp,
-						rp,
-						&rp_locks);
-        erts_factory_message_init(&factory, rp, hp, bp);
-	if (is_immed(msg))
-	    msg_copy = msg;
-	else {
-	    msg_copy = copy_struct(msg, msg_sz, &hp, ohp);
-            factory.hp = hp;
-        }
+	(void) erts_factory_message_create(&factory, rp,
+					   &rp_locks, hsz);
+	msg_copy = (is_immed(msg)
+		    ? msg
+		    : copy_struct(msg, msg_sz,
+				  &factory.hp,
+				  factory.off_heap));
 
 	queue_port_sched_op_reply(rp,
 				  &rp_locks,
@@ -3050,16 +3042,17 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
 
     if (rp) {
 	Eterm tuple;
-	ErlHeapFragment *bp;
+	ErtsMessage *mp;
 	ErlOffHeap *ohp;
 	Eterm* hp;
 	Uint sz_res;
 
 	sz_res = size_object(res);
-	hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks);
+	mp = erts_alloc_message_heap(rp, &rp_locks,
+				     sz_res + 3, &hp, &ohp);
 	res = copy_struct(res, sz_res, &hp, ohp);
 	tuple = TUPLE2(hp, sender, res);
-	erts_queue_message(rp, &rp_locks, bp, tuple, NIL);
+	erts_queue_message(rp, &rp_locks, mp, tuple, NIL);
 
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
@@ -3087,7 +3080,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     Eterm tuple;
     Process* rp;
     Eterm* hp;
-    ErlHeapFragment *bp;
+    ErtsMessage *mp;
     ErlOffHeap *ohp;
     ErtsProcLocks rp_locks = 0;
     int scheduler = erts_get_scheduler_id() != 0;
@@ -3113,7 +3106,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     if (!rp)
 	return;
 
-    hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
+    mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp);
 
     listp = NIL;
     if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) {
@@ -3155,7 +3148,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
+    erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined);
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
@@ -3229,7 +3222,7 @@ deliver_vec_message(Port* prt,			/* Port */
     Eterm tuple;
     Process* rp;
     Eterm* hp;
-    ErlHeapFragment *bp;
+    ErtsMessage *mp;
     ErlOffHeap *ohp;
     ErtsProcLocks rp_locks = 0;
     int scheduler = erts_get_scheduler_id() != 0;
@@ -3261,7 +3254,7 @@ deliver_vec_message(Port* prt,			/* Port */
 	need += (hlen+csize)*2;
     }
 
-    hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
+    mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp);
 
     listp = NIL;
     iov += vsize;
@@ -3322,7 +3315,7 @@ deliver_vec_message(Port* prt,			/* Port */
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
+    erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined);
     erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
 	erts_proc_dec_refc(rp);
@@ -3813,7 +3806,6 @@ write_port_control_result(int control_flags,
 			  ErlDrvSizeT resp_size,
 			  char *pre_alloc_buf,
 			  Eterm **hpp,
-			  ErlHeapFragment *bp,
 			  ErlOffHeap *ohp)
 {
     Eterm res;
@@ -3887,9 +3879,6 @@ port_sig_control(Port *prt,
 
 	if (res == ERTS_PORT_OP_DONE) {
 	    Eterm msg;
-	    Eterm *hp;
-	    ErlHeapFragment *bp;
-	    ErlOffHeap *ohp;
             ErtsHeapFactory factory;
 	    Process *rp;
 	    ErtsProcLocks rp_locks = 0;
@@ -3909,22 +3898,15 @@ port_sig_control(Port *prt,
 	    hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
 
 
-	    hp = erts_alloc_message_heap(hsz,
-						    &bp,
-						    &ohp,
-						    rp,
-						    &rp_locks);
-            erts_factory_message_init(&factory, rp, hp, bp);
+	    (void) erts_factory_message_create(&factory, rp,
+					       &rp_locks, hsz);
 
 	    msg = write_port_control_result(control_flags,
 					    resp_bufp,
 					    resp_size,
 					    &resp_buf[0],
-					    &hp,
-					    bp,
-					    ohp);
-            factory.hp = hp;
-
+					    &factory.hp,
+					    factory.off_heap);
 	    queue_port_sched_op_reply(rp,
 				      &rp_locks,
                                       &factory,
@@ -4065,7 +4047,6 @@ erts_port_control(Process* c_p,
 						 resp_size,
 						 &resp_buf[0],
 						 &hp,
-						 NULL,
 						 &c_p->off_heap);
 	    BUMP_REDS(c_p, ERTS_PORT_REDS_CONTROL);
 	    return ERTS_PORT_OP_DONE;
@@ -4224,21 +4205,14 @@ port_sig_call(Port *prt,
 
 	    hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size);
 	    if (hsz >= 0) {
-                ErlHeapFragment* bp;
-                ErlOffHeap* ohp;
                 ErtsHeapFactory factory;
 		byte *endp;
 
 		hsz += 3; /* ok tuple */
 		hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
 
-                hp = erts_alloc_message_heap(hsz,
-                                             &bp,
-                                             &ohp,
-                                             rp,
-                                             &rp_locks);
+                (void) erts_factory_message_create(&factory, rp, &rp_locks, hsz);
 		endp = (byte *) resp_bufp;
-                erts_factory_message_init(&factory, rp, hp, bp);
 		msg = erts_decode_ext(&factory, &endp);
 		if (is_value(msg)) {
                     hp = erts_produce_heap(&factory,
@@ -4499,7 +4473,9 @@ port_sig_info(Port *prt,
 				    sigdp->u.info.item);
 	if (is_value(value)) {
             ErtsHeapFactory factory;
-            erts_factory_message_init(&factory, NULL, hp, bp);
+	    ErtsMessage *mp = erts_alloc_message(0, NULL);
+	    mp->data.heap_frag = bp;
+            erts_factory_selfcontained_message_init(&factory, mp, hp);
 	    queue_port_sched_op_reply(rp,
 				      &rp_locks,
                                       &factory,
@@ -4587,8 +4563,8 @@ reply_io_bytes(void *vreq)
 
     rp = erts_proc_lookup(req->pid);
     if (rp) {
-	ErlOffHeap *ohp = NULL;
-	ErlHeapFragment *bp = NULL;
+	ErlOffHeap *ohp;
+	ErtsMessage *mp;
 	ErtsProcLocks rp_locks;
 	Eterm ref, msg, ein, eout, *hp;
 	Uint64 in, out;
@@ -4610,7 +4586,7 @@ reply_io_bytes(void *vreq)
 	erts_bld_uint64(NULL, &hsz, in);
 	erts_bld_uint64(NULL, &hsz, out);
 
-	hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks);
+	mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp);
 
 	ref = make_internal_ref(hp);
 	write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]);
@@ -4620,7 +4596,7 @@ reply_io_bytes(void *vreq)
 	eout = erts_bld_uint64(&hp, NULL, out);
 
 	msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout);
-	erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+	erts_queue_message(rp, &rp_locks, mp, msg, NIL);
 
 	if (req->sched_id == sched_id)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -5065,11 +5041,11 @@ ErlDrvTermData driver_mk_term_nil(void)
 void driver_report_exit(ErlDrvPort ix, int status)
 {
    Eterm* hp;
+   ErlOffHeap *ohp;
    Eterm tuple;
    Process *rp;
    Eterm pid;
-   ErlHeapFragment *bp = NULL;
-   ErlOffHeap *ohp;
+   ErtsMessage *mp;
    ErtsProcLocks rp_locks = 0;
    int scheduler = erts_get_scheduler_id() != 0;
    Port* prt = erts_drvport2port(ix);
@@ -5089,13 +5065,13 @@ void driver_report_exit(ErlDrvPort ix, int status)
    if (!rp)
        return;
 
-   hp = erts_alloc_message_heap(3+3, &bp, &ohp, rp, &rp_locks);
+   mp = erts_alloc_message_heap(rp, &rp_locks, 3+3, &hp, &ohp);
 
    tuple = TUPLE2(hp, am_exit_status, make_small(status));
    hp += 3;
    tuple = TUPLE2(hp, prt->common.id, tuple);
 
-   erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
+   erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined);
 
    erts_smp_proc_unlock(rp, rp_locks);
    if (!scheduler)
@@ -5205,7 +5181,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
     ErtsProcLocks rp_locks = 0;
     struct b2t_states__ b2t;
     int scheduler;
-    int is_heap_need_limited = 1;
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_UNDEF(mess,NIL);
@@ -5374,9 +5349,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    need += hsz;
 	    ptr += 2;
 	    depth++;
-            if (size > MAP_SMALL_MAP_LIMIT*3) { /* may contain big map */
-                is_heap_need_limited = 0;
-            }
 	    break;
 	}
 	case ERL_DRV_MAP: { /* int */
@@ -5384,7 +5356,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	    if ((int) ptr[0] < 0) ERTS_DDT_FAIL;
             if (ptr[0] > MAP_SMALL_MAP_LIMIT) {
                 need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]);
-                is_heap_need_limited = 0;
             } else {
                 need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0];
             }
@@ -5423,17 +5394,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	goto done;
     }
 
-    /* Try copy directly to destination heap if we know there are no big maps */
-    if (is_heap_need_limited) {
-        ErlOffHeap *ohp;
-        ErlHeapFragment* bp;
-        Eterm* hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
-        erts_factory_message_init(&factory, rp, hp, bp);
-    }
-    else {
-        erts_factory_message_init(&factory, NULL, NULL,
-                                  new_message_buffer(need));
-    }
+    (void) erts_factory_message_create(&factory, rp, &rp_locks, need);
 
     /*
      * Interpret the instructions and build the term.
@@ -5702,9 +5663,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
     if (res > 0) {
 	mess = ESTACK_POP(stack);  /* get resulting value */
-	erts_factory_close(&factory);
+	erts_factory_trim_and_close(&factory, &mess, 1);
 	/* send message */
-	erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined);
+	erts_queue_message(rp, &rp_locks, factory.message, mess, am_undefined);
     }
     else {
 	if (b2t.ix > b2t.used)
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 5db7ee6d7c..90e16ca14f 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -70,8 +70,10 @@
 #endif
 
 #if ERTS_CAN_INLINE
+#define ERTS_GLB_FORCE_INLINE static ERTS_FORCE_INLINE
 #define ERTS_GLB_INLINE static ERTS_INLINE
 #else
+#define ERTS_GLB_FORCE_INLINE
 #define ERTS_GLB_INLINE
 #endif
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index a741e2e2e6..e03113b8cc 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -110,7 +110,6 @@ Eterm*
 erts_heap_alloc(Process* p, Uint need, Uint xtra)
 {
     ErlHeapFragment* bp;
-    Eterm* htop;
     Uint n;
 #if defined(DEBUG) || defined(CHECK_FOR_HOLES)
     Uint i;
@@ -156,16 +155,6 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra)
     n--;
 #endif
 
-    /*
-     * When we have created a heap fragment, we are no longer allowed
-     * to store anything more on the heap. 
-     */
-    htop = HEAP_TOP(p);
-    if (htop < HEAP_LIMIT(p)) {
-	*htop = make_pos_bignum_header(HEAP_LIMIT(p)-htop-1);
-	HEAP_TOP(p) = HEAP_LIMIT(p);
-    }
-
     bp->next = MBUF(p);
     MBUF(p) = bp;
     bp->alloc_size = n;
@@ -2285,7 +2274,11 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *
 	erts_queue_error_logger_message(from, message, bp);
     }
 #else
-    erts_queue_message(p, NULL /* only used for smp build */, bp, message, NIL);
+    {
+	ErtsMessage *mp = erts_alloc_message(0, NULL);
+	mp->data.heap_frag = bp;
+	erts_queue_message(p, NULL /* only used for smp build */, mp, message, NIL);
+    }
 #endif
 }
 
-- 
cgit v1.2.3


From b56f5a163555181dceb79cbfd0d69d3cb5015e9c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 Nov 2015 12:06:30 +0100
Subject: Use the same conditions when triggering GC after BIF

---
 erts/emulator/beam/beam_emu.c | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1dd56ff989..9521997987 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3533,14 +3533,7 @@ do {						\
 	apply_bif_or_nif_epilogue:
 	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_HOLE_CHECK(c_p);
-	    /*
-	     * We want to test with ERTS_IS_GC_DESIRED(c_p) in order
-	     * to trigger gc due to binaries based on same conditions
-	     * regardless of how the bif is called. This change will
-	     * however be introduced in a separate commit in order to
-	     * easier identify why the characteristics changed.
-	     */
-	    if (c_p->stop - c_p->htop < c_p->mbuf_sz) {
+	    if (ERTS_IS_GC_DESIRED(c_p)) {
 		nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end,
 							    nif_bif_result,
 							    reg, bif_nif_arity);
-- 
cgit v1.2.3


From 9c6f45b901ee701553afe34c0b33b7d931d73fd9 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 11 Nov 2015 11:39:32 +0100
Subject: Bump reductions on GC

---
 erts/emulator/beam/beam_bif_load.c    |   2 +-
 erts/emulator/beam/beam_emu.c         | 108 ++++++++++++++++-------------
 erts/emulator/beam/bif.c              |   6 +-
 erts/emulator/beam/erl_gc.c           | 126 ++++++++++++++++++++++------------
 erts/emulator/beam/erl_gc.h           |   3 +-
 erts/emulator/beam/erl_process.c      |  10 +--
 erts/emulator/beam/erl_process_dict.c |   6 +-
 7 files changed, 156 insertions(+), 105 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 22b4e26c77..c3ebf71a01 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -935,7 +935,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 
 	if (need_gc & ERTS_ORDINARY_GC__) {
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
-	    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+	    *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity);
 	    done_gc |= ERTS_ORDINARY_GC__;
 	}
 	if (need_gc & ERTS_LITERAL_GC__) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9521997987..f74f182863 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -237,6 +237,14 @@ void** beam_ops;
     HEAP_TOP(c_p) = HTOP;  \
     c_p->stop = E
 
+#define HEAVY_SWAPIN       \
+    SWAPIN;		   \
+    FCALLS = c_p->fcalls
+
+#define HEAVY_SWAPOUT      \
+    SWAPOUT;		   \
+    c_p->fcalls = FCALLS
+
 /*
  * Use LIGHT_SWAPOUT when the called function
  * will call HeapOnlyAlloc() (and never HAlloc()).
@@ -297,7 +305,7 @@ void** beam_ops;
      if (E - HTOP < (needed + (HeapNeed))) { \
            SWAPOUT; \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
-           FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
+           FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \
            ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            SWAPIN; \
@@ -349,7 +357,7 @@ void** beam_ops;
     if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
        SWAPOUT;                                                 		\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
-       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  		\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));  		\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        SWAPIN;                                                  		\
@@ -370,7 +378,7 @@ void** beam_ops;
     if (E - HTOP < need) {                                      \
        SWAPOUT;                                                 \
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
-       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        SWAPIN;                                                  \
@@ -391,7 +399,7 @@ void** beam_ops;
        SWAPOUT;								\
        reg[Live] = Extra;						\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
-       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1);	\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        Extra = reg[Live];						\
@@ -415,9 +423,9 @@ void** beam_ops;
 
 #define MakeFun(FunP, NumFree)					\
   do {								\
-     SWAPOUT;							\
+     HEAVY_SWAPOUT;						\
      r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree);	\
-     SWAPIN;							\
+     HEAVY_SWAPIN;						\
   } while (0)
 
 #define PutTuple(Dst, Arity)			\
@@ -1402,11 +1410,11 @@ void process_main(void)
 	    }
 
 	    live = Arg(2);
-	    SWAPOUT;
+	    HEAVY_SWAPOUT;
 	    reg[live] = increment_reg_val;
 	    reg[live+1] = make_small(increment_val);
 	    result = erts_gc_mixed_plus(c_p, reg, live);
-	    SWAPIN;
+	    HEAVY_SWAPIN;
 	    ERTS_HOLE_CHECK(c_p);
 	    if (is_value(result)) {
 		StoreBifResult(3, result);
@@ -1420,11 +1428,11 @@ void process_main(void)
      Eterm result;				\
      Uint live = Arg(1);			\
 						\
-     SWAPOUT;					\
+     HEAVY_SWAPOUT;				\
      reg[live] = Op1;				\
      reg[live+1] = Op2;				\
      result = erts_gc_##name(c_p, reg, live);	\
-     SWAPIN;					\
+     HEAVY_SWAPIN;				\
      ERTS_HOLE_CHECK(c_p);			\
      if (is_value(result)) {			\
 	 StoreBifResult(4, result);		\
@@ -1744,7 +1752,7 @@ void process_main(void)
 	     if (E - HTOP < 3) {
 		 SWAPOUT;
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
-		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
+		 FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1);
 		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 SWAPIN;
@@ -2382,9 +2390,9 @@ void process_main(void)
  OpCase(new_map_dII): {
      Eterm res;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      res = new_map(c_p, reg, I-1);
-     SWAPIN;
+     HEAVY_SWAPIN;
      StoreResult(res, Arg(0));
      Next(3+Arg(2));
  }
@@ -2472,9 +2480,9 @@ do {						\
      Eterm map;
 
      GetArg1(1, map);
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      res = update_map_assoc(c_p, reg, map, I);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (is_value(res)) {
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
@@ -2494,9 +2502,9 @@ do {						\
      Eterm map;
 
      GetArg1(1, map);
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      res = update_map_exact(c_p, reg, map, I);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (is_value(res)) {
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
@@ -3072,10 +3080,10 @@ do {						\
 	 bnot_val = make_small(~signed_val(bnot_val));
      } else {
 	 Uint live = Arg(2);
-	 SWAPOUT;
+	 HEAVY_SWAPOUT;
 	 reg[live] = bnot_val;
 	 bnot_val = erts_gc_bnot(c_p, reg, live);
-	 SWAPIN;
+	 HEAVY_SWAPIN;
 	 ERTS_HOLE_CHECK(c_p);
 	 if (is_nil(bnot_val)) {
 	     goto lb_Cl_error;
@@ -3090,9 +3098,9 @@ do {						\
 
  OpCase(i_apply): {
      BeamInstr *next;
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+1);
 	 SET_I(next);
@@ -3104,9 +3112,9 @@ do {						\
 
  OpCase(i_apply_last_P): {
      BeamInstr *next;
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
@@ -3119,9 +3127,9 @@ do {						\
 
  OpCase(i_apply_only): {
      BeamInstr *next;
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_I(next);
 	 Dispatch();
@@ -3133,9 +3141,9 @@ do {						\
  OpCase(apply_I): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = fixed_apply(c_p, reg, Arg(0));
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+2);
 	 SET_I(next);
@@ -3148,9 +3156,9 @@ do {						\
  OpCase(apply_last_IP): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = fixed_apply(c_p, reg, Arg(0));
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(1));
@@ -3164,9 +3172,9 @@ do {						\
  OpCase(i_apply_fun): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+1);
 	 SET_I(next);
@@ -3178,9 +3186,9 @@ do {						\
  OpCase(i_apply_fun_last_P): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
@@ -3193,9 +3201,9 @@ do {						\
  OpCase(i_apply_fun_only): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_I(next);
 	 Dispatchfun();
@@ -3206,9 +3214,9 @@ do {						\
  OpCase(i_call_fun_I): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+2);
 	 SET_I(next);
@@ -3220,9 +3228,9 @@ do {						\
  OpCase(i_call_fun_last_IP): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	SET_CP(c_p, (BeamInstr *) E[0]);
 	E = ADD_BYTE_OFFSET(E, Arg(1));
@@ -3421,9 +3429,9 @@ do {						\
      * code[3]: &&call_error_handler
      * code[4]: Not used
      */
-    SWAPOUT;
+    HEAVY_SWAPOUT;
     I = call_error_handler(c_p, I-3, reg, am_undefined_function);
-    SWAPIN;
+    HEAVY_SWAPIN;
     if (I) {
 	Goto(*I);
     }
@@ -3977,10 +3985,10 @@ do {						\
      Eterm Size;
 
      GetArg1(4, Size);
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      reg[live] = x(SCRATCH_X_REG);
      res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3));
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (is_non_value(res)) {
 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
 	 goto lb_Cl_error;
@@ -4005,9 +4013,9 @@ do {						\
  }
 
  OpCase(bs_init_writable): {
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      r(0) = erts_bs_init_writable(c_p, r(0));
-     SWAPIN;
+     HEAVY_SWAPIN;
      Next(0);
  }
 
@@ -4835,7 +4843,7 @@ do {						\
 	     BeamInstr *next;
 
 	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
-	     SWAPIN;
+	     HEAVY_SWAPIN;
 	     if (next != NULL) {
 		 SET_I(next);
 		 Dispatchfun();
@@ -4884,20 +4892,22 @@ do {						\
  }
 
  OpCase(i_hibernate): {
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+	 FCALLS = c_p->fcalls;
 	 c_p->flags &= ~F_HIBERNATE_SCHED;
 	 goto do_schedule;
      } else {
+	 HEAVY_SWAPIN;
 	 I = handle_error(c_p, I, reg, hibernate_3);
 	 goto post_error_handling;
      }
  }
 
  OpCase(i_debug_breakpoint): {
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      I = call_error_handler(c_p, I-3, reg, am_breakpoint);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (I) {
 	 Goto(*I);
      }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index e4283ac945..57dd045193 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3839,11 +3839,9 @@ BIF_RETTYPE now_0(BIF_ALIST_0)
 
 BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
 {
-    int reds;
-
     FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
-    reds = erts_garbage_collect(BIF_P, 0, NULL, 0);
-    BIF_RET2(am_true, reds);
+    erts_garbage_collect(BIF_P, 0, NULL, 0);
+    BIF_RET(am_true);
 }
 
 /**********************************************************************/
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6a52e1a890..7163b4839d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -135,7 +135,7 @@ static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm
 					 char* src, Uint src_size);
 static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
 				      Eterm* heap, Eterm* htop, Eterm* objv, int nobj);
-static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
+static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
 static void sweep_off_heap(Process *p, int fullsweep);
@@ -172,6 +172,20 @@ typedef struct {
     erts_smp_atomic32_t refc;
 } ErtsGCInfoReq;
 
+static ERTS_INLINE int
+gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
+{
+    Sint reds;
+
+    reds = gc_moved_live_words/10;
+    reds += resize_moved_words/100;
+    if (reds < 1)
+	return 1;
+    if (reds > INT_MAX)
+	return INT_MAX;
+    return (int) reds;
+}
+
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
                                  ErtsGCInfoReq,
                                  5,
@@ -413,6 +427,7 @@ static ERTS_INLINE void reset_active_writer(Process *p)
 }
 
 #define ERTS_DELAY_GC_EXTRA_FREE 40
+#define ERTS_ABANDON_HEAP_COST 10
 
 static int
 delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
@@ -421,6 +436,7 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
     Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
     Eterm *stop, *hend;
     Uint hsz, ssz;
+    int reds_left;
 
     ERTS_HOLE_CHECK(p);
 
@@ -482,7 +498,12 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
 
     /* Make sure that we do a proper GC as soon as possible... */
     p->flags |= F_FORCE_GC;
-    return CONTEXT_REDS;
+    reds_left = ERTS_BIF_REDS_LEFT(p);
+    if (reds_left > ERTS_ABANDON_HEAP_COST) {
+	int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
+	ERTS_VBUMP_REDS(p, vreds);
+    }
+    return ERTS_ABANDON_HEAP_COST;
 }
 
 static ERTS_FORCE_INLINE Uint
@@ -536,7 +557,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 		int need, Eterm* objv, int nobj)
 {
     Uint reclaimed_now = 0;
-    int done = 0;
+    int reds;
     ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
     ErtsSchedulerData *esdp;
 #ifdef USE_VM_PROBES
@@ -562,9 +583,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     ERTS_CHK_OFFHEAP(p);
 
     ErtsGcQuickSanityCheck(p);
-    if (GEN_GCS(p) >= MAX_GEN_GCS(p)) {
-        FLAGS(p) |= F_NEED_FULLSWEEP;
-    }
+
 #ifdef USE_VM_PROBES
     *pidbuf = '\0';
     if (DTRACE_ENABLED(gc_major_start)
@@ -577,17 +596,21 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     /*
      * Test which type of GC to do.
      */
-    while (!done) {
-	if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) {
-	    DTRACE2(gc_major_start, pidbuf, need);
-	    done = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
-	    DTRACE2(gc_major_end, pidbuf, reclaimed_now);
-	} else {
-	    DTRACE2(gc_minor_start, pidbuf, need);
-	    done = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
-	    DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
-	}
+
+    if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) {
+	DTRACE2(gc_minor_start, pidbuf, need);
+	reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+	DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
+	if (reds < 0)
+	    goto do_major_collection;
+    }
+    else {
+    do_major_collection:
+	DTRACE2(gc_major_start, pidbuf, need);
+	reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+	DTRACE2(gc_major_end, pidbuf, reclaimed_now);
     }
+
     reset_active_writer(p);
 
     /*
@@ -648,23 +671,22 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     p->last_old_htop = p->old_htop;
 #endif
 
-    /* FIXME: This function should really return an Sint, i.e., a possibly
-       64 bit wide signed integer, but that requires updating all the code
-       that calls it. For now, we just return INT_MAX if the result is too
-       large for an int. */
-    {
-      Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10;
-      if (result >= INT_MAX) return INT_MAX;
-      else return (int) result;
-    }
+    return reds;
 }
 
 int
-erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj)
 {
     return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
 }
 
+void
+erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+{
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+    BUMP_REDS(p, reds);
+}
+
 /*
  * Place all living data on a the new heap; deallocate any old heap.
  * Meant to be used by hibernate/3.
@@ -679,6 +701,7 @@ erts_garbage_collect_hibernate(Process* p)
     char* area;
     Uint area_size;
     Sint offs;
+    int reds;
 
     if (p->flags & F_DISABLE_GC)
 	ERTS_INTERNAL_ERROR("GC disabled");
@@ -774,6 +797,9 @@ erts_garbage_collect_hibernate(Process* p)
     ErtsGcQuickSanityCheck(p);
 
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
+    reds = gc_cost(actual_size, actual_size);
+    BUMP_REDS(p, reds);
 }
 
 
@@ -989,16 +1015,21 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
 	((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) &&
 	 ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
 	 ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
-	Uint stack_size, size_after, need_after, new_sz;
+	Eterm *prev_old_htop;
+	Uint stack_size, size_after, adjust_size, need_after, new_sz, new_mature;
 
 	stack_size = p->hend - p->stop;
 	new_sz = stack_size + size_before;
         new_sz = next_heap_size(p, new_sz, 0);
 
+	prev_old_htop = p->old_htop;
         do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm),
 		 new_sz, objv, nobj);
 
-        size_after = HEAP_TOP(p) - HEAP_START(p);
+	new_mature = p->old_htop - prev_old_htop;
+
+	size_after = new_mature;
+        size_after += HEAP_TOP(p) - HEAP_START(p);
         *recl += (size_before - size_after);
 
 	ErtsGcQuickSanityCheck(p);
@@ -1017,6 +1048,8 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
          * the heap size is substantial, we don't want to shrink.
          */
 
+	adjust_size = 0;
+
         if ((HEAP_SIZE(p) > 3000) && (4 * need_after < HEAP_SIZE(p)) &&
             ((HEAP_SIZE(p) > 8000) ||
              (HEAP_SIZE(p) > (OLD_HEND(p) - OLD_HEAP(p))))) {
@@ -1038,31 +1071,33 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
 					       : next_heap_size(p, wanted, 0);
             if (wanted < HEAP_SIZE(p)) {
                 shrink_new_heap(p, wanted, objv, nobj);
+		adjust_size = p->htop - p->heap;
             }
 
-            ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
-            ASSERT(MBUF(p) == NULL);
-	    return 1;		/* We are done. */
+	    goto done;
         }
 
         if (HEAP_SIZE(p) >= need_after) {
 	    /*
 	     * The heap size turned out to be just right. We are done.
 	     */
-            ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
-            ASSERT(MBUF(p) == NULL);
-            return 1;
+	    goto done;
 	}
 
         grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
-	return 1;
+	adjust_size = p->htop - p->heap;
+
+    done:
+	ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
+	ASSERT(MBUF(p) == NULL);
+
+	return gc_cost(size_after, adjust_size);
     }
 
     /*
      * Not enough room for a minor collection. Must force a major collection.
      */
-    FLAGS(p) |= F_NEED_FULLSWEEP;
-    return 0;
+    return -1;
 }
 
 /*
@@ -1341,12 +1376,13 @@ static int
 major_collection(Process* p, ErlHeapFragment *live_hf_end,
 		 int need, Eterm* objv, int nobj, Uint *recl)
 {
-    Uint size_before, stack_size;
+    Uint size_before, size_after, stack_size;
     Eterm* n_heap;
     Eterm* n_htop;
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
     Uint new_sz, stk_sz;
+    int adjusted;
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
@@ -1413,9 +1449,10 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     ErtsGcQuickSanityCheck(p);
 
-    *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+    size_after = HEAP_TOP(p) - HEAP_START(p);
+    *recl += size_before - size_after;
 
-    adjust_after_fullsweep(p, need, objv, nobj);
+    adjusted = adjust_after_fullsweep(p, need, objv, nobj);
 
 #ifdef HARDDEBUG
     disallow_heap_frag_ref_in_heap(p);
@@ -1423,7 +1460,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     remove_message_buffers(p);
 
     ErtsGcQuickSanityCheck(p);
-    return 1;			/* We are done. */
+
+    return gc_cost(size_after, adjusted ? size_after : 0);
 }
 
 static Eterm *
@@ -1524,9 +1562,10 @@ full_sweep_heaps(Process *p,
     return n_htop;
 }
 
-static void
+static int
 adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
 {
+    int adjusted = 0;
     Uint wanted, sz, need_after;
     Uint stack_size = STACK_SZ_ON_HEAP(p);
     
@@ -1539,6 +1578,7 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
         /* Too small - grow to match requested need */
         sz = next_heap_size(p, need_after, 0);
         grow_new_heap(p, sz, objv, nobj);
+	adjusted = 1;
     } else if (3 * HEAP_SIZE(p) < 4 * need_after){
         /* Need more than 75% of current, postpone to next GC.*/
         FLAGS(p) |= F_HEAP_GROW;
@@ -1555,8 +1595,10 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
 
         if (sz < HEAP_SIZE(p)) {
             shrink_new_heap(p, sz, objv, nobj);
+	    adjusted = 1;
         }
     }
+    return adjusted;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index a496c5f008..d603866cbf 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -134,7 +134,8 @@ typedef struct {
 
 void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
-int erts_garbage_collect(struct process*, int, Eterm*, int);
+int erts_garbage_collect_nobump(struct process*, int, Eterm*, int);
+void erts_garbage_collect(struct process*, int, Eterm*, int);
 void erts_garbage_collect_hibernate(struct process* p);
 Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
 				 Eterm result, Eterm* regs, Uint arity);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e490ffea5a..5e7ed0d151 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9774,7 +9774,7 @@ Process *schedule(Process *p, int calls)
 
 	if (ERTS_IS_GC_DESIRED(p)) {
 	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) {
-		reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity);
+		reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity);
 		if (reds <= 0) {
 		    p->fcalls = reds;
 		    goto sched_out_proc;
@@ -10057,10 +10057,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    else {
 		if (!garbage_collected) {
 		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
-		    reds += erts_garbage_collect(c_p,
-						 0,
-						 c_p->arg_reg,
-						 c_p->arity);
+		    reds += erts_garbage_collect_nobump(c_p,
+							0,
+							c_p->arg_reg,
+							c_p->arity);
 		    garbage_collected = 1;
 		}
 		st_res = am_true;
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 8606371bdf..f82cad745a 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -583,7 +583,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	root[0] = id;
 	root[1] = value;
 	root[2] = old;
-	BUMP_REDS(p, erts_garbage_collect(p, needed, root, 3));
+	erts_garbage_collect(p, needed, root, 3);
 	id = root[0];
 	value = root[1];
 	old = root[2];
@@ -715,7 +715,7 @@ static void shrink(Process *p, Eterm* ret)
 		    needed = 2*erts_list_length(hi);
 		}
 		if (HeapWordsLeft(p) < needed) {
-		    BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1));
+		    erts_garbage_collect(p, needed, ret, 1);
 		    hi = pd->data[(pd->splitPosition + pd->homeSize)];
 		    lo = pd->data[pd->splitPosition];
 		}
@@ -811,7 +811,7 @@ static void grow(Process *p)
 	}
     }
     if (HeapWordsLeft(p) < needed) {
-	BUMP_REDS(p, erts_garbage_collect(p, needed, 0, 0));
+	erts_garbage_collect(p, needed, 0, 0);
     }
 #ifdef DEBUG
     hp_limit = p->htop + needed;
-- 
cgit v1.2.3


From 3e490ee1b43140ffcd8cb3fc3ccfe4339c08584e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 11 Nov 2015 10:20:18 +0100
Subject: Improve yield strategy for list_to_binary()/binary_to_list()

Avoid yield in the BIF when input is small. This either yielding
before beginning to work in the BIF, or by allowing some more
reductions before yielding.
---
 erts/emulator/beam/binary.c | 81 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 74 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index e670fbf31c..d3e481c7f9 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -34,6 +34,9 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 
+#define L2B_B2L_MIN_EXEC_REDS (CONTEXT_REDS/4)
+#define L2B_B2L_RESCHED_REDS (CONTEXT_REDS/40)
+
 static Export binary_to_list_continue_export;
 static Export list_to_binary_continue_export;
 
@@ -415,10 +418,10 @@ binary_to_list_chunk(Process *c_p,
 }
 
 static ERTS_INLINE BIF_RETTYPE
-binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes, Uint size, Uint bitoffs)
+binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes,
+	       Uint size, Uint bitoffs, int reds_left, int one_chunk)
 {
-    int reds_left = ERTS_BIF_REDS_LEFT(c_p);
-    if (size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION) {
+    if (one_chunk) {
 	Eterm res;
 	BIF_RETTYPE ret;
 	int bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1;
@@ -472,11 +475,29 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
     Uint size;
     Uint bitsize;
     Uint bitoffs;
+    int reds_left;
+    int one_chunk;
 
     if (is_not_binary(BIF_ARG_1)) {
 	goto error;
     }
+
     size = binary_size(BIF_ARG_1);
+    reds_left = ERTS_BIF_REDS_LEFT(BIF_P);
+    one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION;
+    if (!one_chunk) {
+	if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) {
+	    if (reds_left <= L2B_B2L_RESCHED_REDS) {
+		/* Yield and do it with full context reds... */
+		ERTS_BIF_YIELD1(bif_export[BIF_binary_to_list_1],
+				BIF_P, BIF_ARG_1);
+	    }
+	    /* Allow a bit more reductions... */
+	    one_chunk = 1;
+	    reds_left = L2B_B2L_MIN_EXEC_REDS;
+	}
+    }
+
     ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
     if (bitsize != 0) {
 	goto error;
@@ -486,7 +507,8 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
     } else {
 	Eterm* hp = HAlloc(BIF_P, 2 * size);
 	byte* bytes = binary_bytes(real_bin)+offset;
-	return binary_to_list(BIF_P, hp, NIL, bytes, size, bitoffs);
+	return binary_to_list(BIF_P, hp, NIL, bytes, size,
+			      bitoffs, reds_left, one_chunk);
     }
 
     error:
@@ -505,6 +527,8 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
     Uint start;
     Uint stop;
     Eterm* hp;
+    int reds_left;
+    int one_chunk;
 
     if (is_not_binary(BIF_ARG_1)) {
 	goto error;
@@ -513,6 +537,21 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
 	goto error;
     }
     size = binary_size(BIF_ARG_1);
+    reds_left = ERTS_BIF_REDS_LEFT(BIF_P);
+    one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION;
+    if (!one_chunk) {
+	if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) {
+	    if (reds_left <= L2B_B2L_RESCHED_REDS) {
+		/* Yield and do it with full context reds... */
+		ERTS_BIF_YIELD3(bif_export[BIF_binary_to_list_3],
+				BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+	    }
+	    /* Allow a bit more reductions... */
+	    one_chunk = 1;
+	    reds_left = L2B_B2L_MIN_EXEC_REDS;
+	}
+    }
+
     ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize);
     if (start < 1 || start > size || stop < 1 ||
 	stop > size || stop < start ) {
@@ -520,7 +559,8 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
     }
     i = stop-start+1;
     hp = HAlloc(BIF_P, 2*i);
-    return binary_to_list(BIF_P, hp, NIL, bytes+start-1, i, bitoffs);
+    return binary_to_list(BIF_P, hp, NIL, bytes+start-1, i,
+			  bitoffs, reds_left, one_chunk);
     error:
 	BIF_ERROR(BIF_P, BADARG);
 }
@@ -537,11 +577,27 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
     byte* bytes;
     Eterm previous = NIL;
     Eterm* hp;
+    int reds_left;
+    int one_chunk;
 
     if (is_not_binary(BIF_ARG_1)) {
 	BIF_ERROR(BIF_P, BADARG);
     }
     size = binary_size(BIF_ARG_1);
+    reds_left = ERTS_BIF_REDS_LEFT(BIF_P);
+    one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION;
+    if (!one_chunk) {
+	if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) {
+	    if (reds_left <= L2B_B2L_RESCHED_REDS) {
+		/* Yield and do it with full context reds... */
+		ERTS_BIF_YIELD1(bif_export[BIF_bitstring_to_list_1],
+				BIF_P, BIF_ARG_1);
+	    }
+	    /* Allow a bit more reductions... */
+	    one_chunk = 1;
+	    reds_left = L2B_B2L_MIN_EXEC_REDS;
+	}
+    }
     ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
     bytes = binary_bytes(real_bin)+offset;
     if (bitsize == 0) {
@@ -566,7 +622,8 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 	hp += 2;
     }
 
-    return binary_to_list(BIF_P, hp, previous, bytes, size, bitoffs);
+    return binary_to_list(BIF_P, hp, previous, bytes, size,
+			  bitoffs, reds_left, one_chunk);
 }
 
 
@@ -795,8 +852,19 @@ static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1)
 
 BIF_RETTYPE erts_list_to_binary_bif(Process *c_p, Eterm arg, Export *bif)
 {
+    int orig_reds_left = ERTS_BIF_REDS_LEFT(c_p);
     BIF_RETTYPE ret;
 
+    if (orig_reds_left < L2B_B2L_MIN_EXEC_REDS) {
+	if (orig_reds_left <= L2B_B2L_RESCHED_REDS) {
+	    /* Yield and do it with full context reds... */
+	    ERTS_BIF_PREP_YIELD1(ret, bif, c_p, arg);
+	    return ret;
+	}
+	/* Allow a bit more reductions... */
+	orig_reds_left = L2B_B2L_MIN_EXEC_REDS;
+    }
+
     if (is_nil(arg))
 	ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) "", 0));
     else if (is_not_list(arg))
@@ -818,7 +886,6 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *c_p, Eterm arg, Export *bif)
 						       bif,
 						       erts_iolist_size_yielding,
 						       erts_iolist_to_buf_yielding);
-	    int orig_reds_left = ERTS_BIF_REDS_LEFT(c_p);
 
 	    /*
 	     * First try to do it all at once without having to use
-- 
cgit v1.2.3


From fd1e6b2b2623395512ea0450c3b4e656cb354f42 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Nov 2015 19:13:03 +0100
Subject: stdlib: Fix bug in binary:split for empty binary

Bug introduced om master branch at b93e9b611056828a
and reported in ERL-43.
---
 erts/emulator/beam/erl_bif_binary.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index b9640e211d..aec72bd61a 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1614,6 +1614,10 @@ static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindStat
     Eterm *hp;
     Eterm ret;
 
+    if (bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)
+        && binary_size(subject) == 0) {
+        return NIL;
+    }
     hp = HAlloc(p, 2);
     ret = CONS(hp, subject, NIL);
 
-- 
cgit v1.2.3


From dad527f55b51d60e75a0d19aa0f4f42c1065777f Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Fri, 8 Jun 2012 22:21:02 +0300
Subject: An implementation of lightweight unbounded queues

---
 erts/emulator/beam/global.h | 84 ++++++++++++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/utils.c  | 25 ++++++++++++++
 2 files changed, 108 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 594c0ccf94..20f100b427 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -380,7 +380,7 @@ extern int bif_reductions;      /* reductions + fcalls (when doing call_bif) */
 extern int stackdump_on_exit;
 
 /*
- * Here is an implementation of a lightweiht stack.
+ * Here is an implementation of a lightweight stack.
  *
  * Use it like this:
  *
@@ -845,6 +845,88 @@ do {\
 } while(0)
 
 
+/*
+ *  An implementation of lightweight unbounded queues,
+ *  using a circular dynamic array.
+ *  It does not include support for change_allocator.
+ *
+ *  Use it like this:
+ *
+ *  DECLARE_EQUEUE(Queue)	(At the start of a block)
+ *  ...
+ *  EQUEUE_PUT(Queue, Term)
+ *  ...
+ *  if (EQUEUE_ISEMPTY(Queue)) {
+ *     Queue is empty
+ *  } else {
+ *     Term = EQUEUE_GET(Stack);
+ *     Process popped Term here
+ *  }
+ *  ...
+ *  DESTROY_EQUEUE(Queue)
+ */
+
+typedef struct {
+    Eterm* start;
+    Eterm* front;
+    Eterm* back;
+    int possibly_empty;
+    Eterm* end;
+    ErtsAlcType_t alloc_type;
+}ErtsEQueue;
+
+#define DEF_EQUEUE_SIZE (16)
+
+void erl_grow_equeue(ErtsEQueue*, Eterm* def_queue);
+#define EQUE_CONCAT(a,b) a##b
+#define EQUE_DEF_QUEUE(q) EQUE_CONCAT(q,_default_equeue)
+
+#define DECLARE_EQUEUE(q)				\
+    UWord EQUE_DEF_QUEUE(q)[DEF_EQUEUE_SIZE];     	\
+    ErtsEQueue q = {					\
+        EQUE_DEF_QUEUE(q), /* start */			\
+        EQUE_DEF_QUEUE(q), /* front */			\
+        EQUE_DEF_QUEUE(q), /* back */			\
+        1,                 /* possibly_empty */		\
+        EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */	\
+        ERTS_ALC_T_ESTACK  /* alloc_type */		\
+    }
+
+#define DESTROY_EQUEUE(q)				\
+do {							\
+    if (q.start != EQUE_DEF_QUEUE(q)) {			\
+      erts_free(q.alloc_type, q.start);			\
+    }							\
+} while(0)
+
+#define EQUEUE_PUT_UNCHECKED(q, x)			\
+do {							\
+    q.possibly_empty = 0;				\
+    *(q.back) = (x);                    		\
+    if (++(q.back) == q.end) {				\
+	q.back = q.start;				\
+    }							\
+} while(0)
+
+#define EQUEUE_PUT(q, x)				\
+do {							\
+    if (q.back == q.front && !q.possibly_empty) {	\
+        erl_grow_equeue(&q, EQUE_DEF_QUEUE(q));		\
+    }							\
+    EQUEUE_PUT_UNCHECKED(q, x);				\
+} while(0)
+
+#define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty)
+
+#define EQUEUE_GET(q) ({				\
+    q.possibly_empty = 1;				\
+    UWord x = *(q.front);				\
+    if (++(q.front) == q.end) {				\
+        q.front = q.start;				\
+    }							\
+    x;							\
+})
+
 /* binary.c */
 
 void erts_emasculate_writable_binary(ProcBin* pb);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index c3735683bb..184477c36b 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -258,6 +258,31 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes)
     s->psp = s->pstart + sp_offs;
 }
 
+/*
+ * Helper function for the EQUEUE macros defined in global.h.
+ */
+
+void
+erl_grow_equeue(ErtsEQueue* q, Eterm* default_equeue)
+{
+    Uint old_size = (q->end - q->start);
+    Uint new_size = old_size * 2;
+    Uint first_part = (q->end - q->front);
+    Uint second_part = (q->back - q->start);
+    Eterm* new_ptr = erts_alloc(q->alloc_type, new_size*sizeof(Eterm));
+    ASSERT(q->back == q->front);   // of course the queue is full now!
+    if (first_part > 0)
+      sys_memcpy(new_ptr, q->front, first_part*sizeof(Eterm));
+    if (second_part > 0)
+      sys_memcpy(new_ptr+first_part, q->start, second_part*sizeof(Eterm));
+    if (q->start != default_equeue)
+      erts_free(q->alloc_type, q->start);
+    q->start = new_ptr;
+    q->end = q->start + new_size;
+    q->front = q->start;
+    q->back = q->start + old_size;
+}
+
 /* CTYPE macros */
 
 #define LATIN1
-- 
cgit v1.2.3


From 9d0a5bf2c1cc564fd38268cbb5313cd8813ea138 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Mon, 16 Nov 2015 14:57:12 +0100
Subject: erts: Add garbage_collection_info to process_info/2

This info request returns greater details about the current
gc state. This info is not included in the default process_info/1
as it would clutter the default printout with too much information.
---
 erts/emulator/beam/atom.names     |  1 +
 erts/emulator/beam/erl_bif_info.c | 30 ++++++++++++++++++++-
 erts/emulator/beam/erl_gc.c       | 51 +++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_gc.h       |  5 ++++
 erts/emulator/beam/erl_trace.c    | 57 +++++----------------------------------
 5 files changed, 92 insertions(+), 52 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index ea04495574..4ad989cac1 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -256,6 +256,7 @@ atom functions
 atom function_clause
 atom garbage_collecting
 atom garbage_collection
+atom garbage_collection_info
 atom gc_end
 atom gc_start
 atom Ge='>='
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 1eb106a551..855ef8742a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -589,7 +589,8 @@ static Eterm pi_args[] = {
     am_min_bin_vheap_size,
     am_current_location,
     am_current_stacktrace,
-    am_off_heap_message_queue
+    am_off_heap_message_queue,
+    am_garbage_collection_info
 };
 
 #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
@@ -638,6 +639,7 @@ pi_arg2ix(Eterm arg)
     case am_current_location:			return 29;
     case am_current_stacktrace:			return 30;
     case am_off_heap_message_queue:		return 31;
+    case am_garbage_collection_info:		return 32;
     default:					return -1;
     }
 }
@@ -1395,6 +1397,32 @@ process_info_aux(Process *BIF_P,
 	break;
     }
 
+    case am_garbage_collection_info: {
+        Uint sz = 0, actual_sz = 0;
+
+        if (rp == BIF_P) {
+            sz += ERTS_PROCESS_GC_INFO_MAX_SIZE;
+        } else {
+            erts_process_gc_info(rp, &sz, NULL);
+            sz += 3;
+        }
+
+        hp = HAlloc(BIF_P, sz);
+        res = erts_process_gc_info(rp, &actual_sz, &hp);
+
+        /* We may have some extra space, fill with 0 tuples */
+        if (actual_sz <= sz - 3) {
+            for (; actual_sz < sz - 3; hp++, actual_sz++)
+                hp[0] = make_arityval(0);
+        } else {
+            for (; actual_sz < sz; hp++, actual_sz++)
+                hp[0] = make_arityval(0);
+            hp = HAlloc(BIF_P, 3);
+        }
+
+        break;
+    }
+
     case am_group_leader: {
 	int sz = NC_HEAP_SIZE(rp->group_leader);
 	hp = HAlloc(BIF_P, 3 + sz);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6cb37752bc..3399b1a9f2 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2741,6 +2741,57 @@ erts_gc_info_request(Process *c_p)
     return ref;
 }
 
+Eterm
+erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp)
+{
+    ERTS_DECL_AM(bin_vheap_size);
+    ERTS_DECL_AM(bin_vheap_block_size);
+    ERTS_DECL_AM(bin_old_vheap_size);
+    ERTS_DECL_AM(bin_old_vheap_block_size);
+    Eterm tags[] = {
+        /* If you increase the number of elements here, make sure to update
+           any call sites as they may have stack allocations that depend
+           on the number of elements here. */
+        am_old_heap_block_size,
+        am_heap_block_size,
+        am_mbuf_size,
+        am_recent_size,
+        am_stack_size,
+        am_old_heap_size,
+        am_heap_size,
+        AM_bin_vheap_size,
+        AM_bin_vheap_block_size,
+        AM_bin_old_vheap_size,
+        AM_bin_old_vheap_block_size
+    };
+    UWord values[] = {
+        OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
+        HEAP_SIZE(p),
+        MBUF_SIZE(p),
+        HIGH_WATER(p) - HEAP_START(p),
+        STACK_START(p) - p->stop,
+        OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0,
+        HEAP_TOP(p) - HEAP_START(p),
+        MSO(p).overhead,
+        BIN_VHEAP_SZ(p),
+        BIN_OLD_VHEAP(p),
+        BIN_OLD_VHEAP_SZ(p)
+    };
+
+    Eterm res = THE_NON_VALUE;
+
+    ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags));
+    ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS);
+
+    res = erts_bld_atom_uword_2tup_list(hpp,
+                                        sizep,
+                                        sizeof(values)/sizeof(*values),
+                                        tags,
+                                        values);
+
+    return res;
+}
+
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 
 static int
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index d603866cbf..2cedd9361f 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -132,6 +132,11 @@ typedef struct {
   Uint64 garbage_cols;
 } ErtsGCInfo;
 
+#define ERTS_PROCESS_GC_INFO_MAX_TERMS (11)  /* number of elements in process_gc_info*/
+#define ERTS_PROCESS_GC_INFO_MAX_SIZE                                   \
+    (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE))
+Eterm erts_process_gc_info(struct process*, Uint *, Eterm **);
+
 void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
 int erts_garbage_collect_nobump(struct process*, int, Eterm*, int);
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index d02f1f7213..e47b1e4edc 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2144,11 +2144,6 @@ void save_calls(Process *p, Export *e)
 void
 trace_gc(Process *p, Eterm what)
 {
-    ERTS_DECL_AM(bin_vheap_size);
-    ERTS_DECL_AM(bin_vheap_block_size);
-    ERTS_DECL_AM(bin_old_vheap_size);
-    ERTS_DECL_AM(bin_old_vheap_block_size);
-
     ErlHeapFragment *bp = NULL;
     ErlOffHeap *off_heap;
     ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF;	/* Initialized
@@ -2159,55 +2154,22 @@ trace_gc(Process *p, Eterm what)
     Eterm msg = NIL;
     Uint size;
 
-    Eterm tags[] = {
-	am_old_heap_block_size,
-	am_heap_block_size,
-	am_mbuf_size,
-	am_recent_size,
-	am_stack_size,
-	am_old_heap_size,
-	am_heap_size,
-	AM_bin_vheap_size,
-	AM_bin_vheap_block_size,
-	AM_bin_old_vheap_size,
-	AM_bin_old_vheap_block_size
-    };
-
-    UWord values[] = {
-	OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
-	HEAP_SIZE(p),
-	MBUF_SIZE(p),
-	HIGH_WATER(p) - HEAP_START(p),
-	STACK_START(p) - p->stop,
-	OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0,
-	HEAP_TOP(p) - HEAP_START(p),
-	MSO(p).overhead,
-	BIN_VHEAP_SZ(p),
-	BIN_OLD_VHEAP(p),
-	BIN_OLD_VHEAP_SZ(p)
-    };
 #define LOCAL_HEAP_SIZE						\
-    (sizeof(values)/sizeof(*values)) *				\
-	(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) +	\
+    (ERTS_PROCESS_GC_INFO_MAX_SIZE) +                           \
 	5/*4-tuple */ + TS_HEAP_WORDS
     DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
 #ifdef DEBUG
     Eterm* limit;
 #endif
 
-    ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm));
-
     UseTmpHeap(LOCAL_HEAP_SIZE,p);
 
     if (is_internal_port(ERTS_TRACER_PROC(p))) {
 	hp = local_heap;
 #ifdef DEBUG
 	size = 0;
-	(void) erts_bld_atom_uword_2tup_list(NULL,
-					    &size,
-					    sizeof(values)/sizeof(*values),
-					    tags,
-					    values);
+        (void) erts_process_gc_info(p, &size, NULL);
+
 	size += 5/*4-tuple*/ + TS_SIZE(p);
 #endif
     } else {
@@ -2218,11 +2180,8 @@ trace_gc(Process *p, Eterm what)
 			    ERTS_TRACE_FLAGS(p));
 
 	size = 0;
-	(void) erts_bld_atom_uword_2tup_list(NULL,
-					    &size,
-					    sizeof(values)/sizeof(*values),
-					    tags,
-					    values);
+        (void) erts_process_gc_info(p, &size, NULL);
+
 	size += 5/*4-tuple*/ + TS_SIZE(p);
 
 	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
@@ -2233,11 +2192,7 @@ trace_gc(Process *p, Eterm what)
     ASSERT(size <= LOCAL_HEAP_SIZE);
 #endif
 
-    msg = erts_bld_atom_uword_2tup_list(&hp,
-				       NULL,
-				       sizeof(values)/sizeof(*values),
-				       tags,
-				       values);
+    msg = erts_process_gc_info(p, NULL, &hp);
 
     msg = TUPLE4(hp, am_trace, p->common.id, what, msg);
     hp += 5;
-- 
cgit v1.2.3


From d5711cb70be9ad2e5c78839e8a368900ba248a4a Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Fri, 8 Jun 2012 23:13:50 +0300
Subject: Add all the main machinery

Add functions size_shared, copy_shared_calculate and copy_shared_perform.
Add the infrastructure for making these communicate with each other.
Add debug information to other places in the VM, to watch interaction
with the sharing-preserving copy.

CAUTION: If you define the SHCOPY_DEBUG macro (after SHCOPY is actually
used in the VM) and make the whole OTP, there will be a lot of debugging
messages during make (it will also be enabled in erlc).  You have been
warned...
---
 erts/emulator/beam/copy.c   | 1131 +++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_gc.c |   10 +
 erts/emulator/beam/global.h |   56 ++-
 3 files changed, 1196 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index f27c526413..b31e043f08 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -75,8 +75,17 @@ Uint size_object(Eterm obj)
     Uint sum = 0;
     Eterm* ptr;
     int arity;
+#ifdef SHCOPY_DEBUG
+    Eterm mypid;
+#endif
 
     DECLARE_ESTACK(s);
+
+#ifdef SHCOPY_DEBUG
+    mypid = erts_get_current_pid();
+    VERBOSE_DEBUG("[pid=%T] size_object %p\n", mypid, obj);
+#endif
+
     for (;;) {
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST:
@@ -211,6 +220,7 @@ Uint size_object(Eterm obj)
 	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
+		VERBOSE_DEBUG("[pid=%T] size was: %u\n", mypid, sum);
 		return sum;
 	    }
 	    obj = ESTACK_POP(s);
@@ -221,6 +231,350 @@ Uint size_object(Eterm obj)
     }
 }
 
+/*
+ *  Machinery for sharing preserving information
+ *  Using a WSTACK but not very transparently; consider refactoring
+ */
+
+#define DECLARE_BITSTORE(s)						\
+    DECLARE_WSTACK(s);							\
+    int WSTK_CONCAT(s,_bitoffs) = 0;					\
+    int WSTK_CONCAT(s,_offset) = 0;					\
+    UWord WSTK_CONCAT(s,_buffer) = 0
+
+#define DESTROY_BITSTORE(s) DESTROY_WSTACK(s)
+#define BITSTORE_PUT(s,i)						\
+do {									\
+    WSTK_CONCAT(s,_buffer) |= i << WSTK_CONCAT(s,_bitoffs);		\
+    WSTK_CONCAT(s,_bitoffs) += 2;					\
+    if (WSTK_CONCAT(s,_bitoffs) >= 8*sizeof(UWord)) {			\
+	WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer));				\
+	WSTK_CONCAT(s,_bitoffs) = 0;					\
+	WSTK_CONCAT(s,_buffer) = 0;					\
+    }									\
+} while(0)
+#define BITSTORE_CLOSE(s)						\
+do {									\
+    if (WSTK_CONCAT(s,_bitoffs) > 0) {					\
+	WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer));				\
+	WSTK_CONCAT(s,_bitoffs) = 0;					\
+    }									\
+} while(0)
+
+#define BITSTORE_GET(s)	({						\
+    UWord result;							\
+    if (WSTK_CONCAT(s,_bitoffs) <= 0) {					\
+	WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)];	\
+	WSTK_CONCAT(s,_offset)++;					\
+	WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord);			\
+    }									\
+    WSTK_CONCAT(s,_bitoffs) -= 2;					\
+    result = WSTK_CONCAT(s,_buffer) & 3;				\
+    WSTK_CONCAT(s,_buffer) >>= 2;					\
+    result;								\
+})
+
+#define BOXED_VISITED_MASK	 ((Eterm) 3)
+#define BOXED_VISITED		 ((Eterm) 1)
+#define BOXED_SHARED_UNPROCESSED ((Eterm) 2)
+#define BOXED_SHARED_PROCESSED	 ((Eterm) 3)
+
+
+/*
+ *  Is an object in the local heap of a process?
+ */
+
+#define INHEAP_SIMPLE(p, ptr) (					\
+    (OLD_HEAP(p) && OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p)) ||	\
+    (HEAP_START(p) <= ptr && ptr < HEAP_END(p))			\
+  )
+#define INHEAP(p, ptr) (					\
+    INHEAP_SIMPLE(p, ptr) ||					\
+    (force_local ? (force_local = 0, 1) : 0)			\
+  )
+#define COUNT_OFF_HEAP 0
+
+
+/*
+ *  Return the real size of an object and find sharing information
+ *  This currently returns the same as erts_debug:size/1.
+ *  It is argued whether the size of subterms in constant pools
+ *  should be counted or not.
+ */
+#if HALFWORD_HEAP
+Uint size_shared_rel(Eterm obj, Eterm* base)
+#else
+Uint size_shared(Eterm obj)
+#endif
+{
+    Eterm saved_obj = obj;
+    Uint sum = 0;
+    Eterm* ptr;
+    Process* myself;
+
+    DECLARE_EQUEUE(s);
+    DECLARE_BITSTORE(b);
+
+    myself = erts_get_current_process();
+    if (myself == NULL)
+	return size_object(obj);
+
+    for (;;) {
+	VERBOSE_DEBUG("[size] visiting: %x ", obj);
+	switch (primary_tag(obj)) {
+	case TAG_PRIMARY_LIST: {
+	    Eterm head, tail;
+	    VERBOSE_DEBUG("L");
+	    ptr = list_val_rel(obj, base);
+	    /* we're not counting anything that's outside our heap */
+	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+		goto pop_next;
+	    }
+	    head = CAR(ptr);
+	    tail = CDR(ptr);
+	    /* if it's visited, don't count it */
+	    if (primary_tag(tail) == TAG_PRIMARY_HEADER ||
+		primary_tag(head) == TAG_PRIMARY_HEADER) {
+		VERBOSE_DEBUG("!");
+		goto pop_next;
+	    }
+	    /* else make it visited now */
+	    switch (primary_tag(tail)) {
+	    case TAG_PRIMARY_LIST:
+		VERBOSE_DEBUG("/L");
+		ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_PRIMARY_IMMED1:
+		VERBOSE_DEBUG("/I");
+		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
+		CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head);
+		break;
+	    case TAG_PRIMARY_BOXED:
+		VERBOSE_DEBUG("/B saved %d", primary_tag(head));
+		BITSTORE_PUT(b, primary_tag(head));
+		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
+		CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER;
+		break;
+	    }
+	    /* and count it */
+	    sum += 2;
+	    if (!IS_CONST(head)) {
+		EQUEUE_PUT(s, head);
+	    }
+	    obj = tail;
+	    break;
+	}
+	case TAG_PRIMARY_BOXED: {
+	    Eterm hdr;
+	    VERBOSE_DEBUG("B");
+	    ptr = boxed_val_rel(obj, base);
+	    /* we're not counting anything that's outside our heap */
+	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+		goto pop_next;
+	    }
+	    hdr = *ptr;
+	    /* if it's visited, don't count it */
+	    if (primary_tag(hdr) != TAG_PRIMARY_HEADER) {
+		VERBOSE_DEBUG("!");
+		goto pop_next;
+	    }
+	    /* else make it visited now */
+	    *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED;
+	    /* and count it */
+	    ASSERT(is_header(hdr));
+	    switch (hdr & _TAG_HEADER_MASK) {
+	    case ARITYVAL_SUBTAG: {
+		int arity = header_arity(hdr);
+		VERBOSE_DEBUG("/T");
+		sum += arity + 1;
+		if (arity == 0) { /* Empty tuple -- unusual. */
+		    VERBOSE_DEBUG("e");
+		    goto pop_next;
+		}
+		while (arity-- > 0) {
+		    obj = *++ptr;
+		    if (!IS_CONST(obj)) {
+			EQUEUE_PUT(s, obj);
+		    }
+		}
+		goto pop_next;
+	    }
+	    case FUN_SUBTAG: {
+		ErlFunThing* funp = (ErlFunThing *) ptr;
+		unsigned eterms = 1 /* creator */ + funp->num_free;
+		unsigned sz = thing_arityval(hdr);
+		VERBOSE_DEBUG("/F");
+		sum += 1 /* header */ + sz + eterms;
+		ptr += 1 /* header */ + sz;
+		while (eterms-- > 0) {
+		    obj = *ptr++;
+		    if (!IS_CONST(obj)) {
+			EQUEUE_PUT(s, obj);
+		    }
+		}
+		goto pop_next;
+	    }
+	    case SUB_BINARY_SUBTAG: {
+		ErlSubBin* sb = (ErlSubBin *) ptr;
+		Uint extra_bytes;
+		Eterm hdr;
+		ASSERT((sb->thing_word & ~BOXED_VISITED_MASK) == HEADER_SUB_BIN);
+		if (sb->bitsize + sb->bitoffs > 8) {
+		    sum += ERL_SUB_BIN_SIZE;
+		    extra_bytes = 2;
+		} else if (sb->bitsize + sb->bitoffs > 0) {
+		    sum += ERL_SUB_BIN_SIZE;
+		    extra_bytes = 1;
+		} else {
+		    extra_bytes = 0;
+		}
+		ptr = binary_val_rel(sb->orig, base);
+		hdr = (*ptr) & ~BOXED_VISITED_MASK;
+		if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) {
+		    sum += PROC_BIN_SIZE;
+		} else {
+		    ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG);
+		    sum += heap_bin_size(binary_size_rel(obj, base) + extra_bytes);
+		}
+		goto pop_next;
+	    }
+	    case BIN_MATCHSTATE_SUBTAG:
+		erl_exit(ERTS_ABORT_EXIT,
+			 "size_shared: matchstate term not allowed");
+	    default:
+		VERBOSE_DEBUG("/D");
+		sum += thing_arityval(hdr) + 1;
+		goto pop_next;
+	    }
+	    break;
+	}
+	case TAG_PRIMARY_IMMED1:
+	    VERBOSE_DEBUG("I");
+	pop_next:
+	    if (EQUEUE_ISEMPTY(s)) {
+		goto cleanup;
+	    }
+	    obj = EQUEUE_GET(s);
+	    break;
+	default:
+	    erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
+	}
+	VERBOSE_DEBUG("\n");
+    }
+
+cleanup:
+    VERBOSE_DEBUG("\n");
+    obj = saved_obj;
+    BITSTORE_CLOSE(b);
+    for (;;) {
+	VERBOSE_DEBUG("[size] revisiting: %x ", obj);
+	switch (primary_tag(obj)) {
+	case TAG_PRIMARY_LIST: {
+	    Eterm head, tail;
+	    VERBOSE_DEBUG("L");
+	    ptr = list_val_rel(obj, base);
+	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+		goto cleanup_next;
+	    }
+	    head = CAR(ptr);
+	    tail = CDR(ptr);
+	    /* if not already clean, clean it up */
+	    if (primary_tag(tail) == TAG_PRIMARY_HEADER) {
+		if (primary_tag(head) == TAG_PRIMARY_HEADER) {
+		    Eterm saved = BITSTORE_GET(b);
+		    VERBOSE_DEBUG("/B restoring %d", saved);
+		    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved;
+		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED;
+		} else {
+		    VERBOSE_DEBUG("/L");
+		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST;
+		}
+	    } else if (primary_tag(head) == TAG_PRIMARY_HEADER) {
+		VERBOSE_DEBUG("/I");
+		CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail);
+		CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1;
+	    } else {
+		VERBOSE_DEBUG("!");
+		goto cleanup_next;
+	    }
+	    /* and its children too */
+	    if (!IS_CONST(head)) {
+		EQUEUE_PUT_UNCHECKED(s, head);
+	    }
+	    obj = tail;
+	    break;
+	}
+	case TAG_PRIMARY_BOXED: {
+	    Eterm hdr;
+	    VERBOSE_DEBUG("B");
+	    ptr = boxed_val_rel(obj, base);
+	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+		goto cleanup_next;
+	    }
+	    hdr = *ptr;
+	    /* if not already clean, clean it up */
+	    if (primary_tag(hdr) == TAG_PRIMARY_HEADER) {
+		goto cleanup_next;
+	    }
+	    else {
+		ASSERT(primary_tag(hdr) == BOXED_VISITED);
+		*ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER;
+	    }
+	    /* and its children too */
+	    switch (hdr & _TAG_HEADER_MASK) {
+	    case ARITYVAL_SUBTAG: {
+		int arity = header_arity(hdr);
+		if (arity == 0) { /* Empty tuple -- unusual. */
+		    goto cleanup_next;
+		}
+		while (arity-- > 0) {
+		    obj = *++ptr;
+		    if (!IS_CONST(obj)) {
+			EQUEUE_PUT_UNCHECKED(s, obj);
+		    }
+		}
+		goto cleanup_next;
+	    }
+	    case FUN_SUBTAG: {
+		ErlFunThing* funp = (ErlFunThing *) ptr;
+		unsigned eterms = 1 /* creator */ + funp->num_free;
+		unsigned sz = thing_arityval(hdr);
+		ptr += 1 /* header */ + sz;
+		while (eterms-- > 0) {
+		    obj = *ptr++;
+		    if (!IS_CONST(obj)) {
+			EQUEUE_PUT_UNCHECKED(s, obj);
+		    }
+		}
+		goto cleanup_next;
+	    }
+	    default:
+		goto cleanup_next;
+	    }
+	    break;
+	}
+	case TAG_PRIMARY_IMMED1:
+	cleanup_next:
+	    if (EQUEUE_ISEMPTY(s)) {
+		goto all_clean;
+	    }
+	    obj = EQUEUE_GET(s);
+	    break;
+	default:
+	    erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
+	}
+	VERBOSE_DEBUG("\n");
+    }
+
+ all_clean:
+    VERBOSE_DEBUG("\n");
+    /* Return the result */
+    DESTROY_EQUEUE(s);
+    DESTROY_BITSTORE(b);
+    return sum;
+}
+
+
 /*
  *  Copy a structure to a heap.
  */
@@ -244,10 +598,18 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     Eterm org_obj = obj;
     Uint org_sz = sz;
 #endif
+#ifdef SHCOPY_DEBUG
+    Eterm mypid;
+#endif
 
     if (IS_CONST(obj))
 	return obj;
 
+#ifdef SHCOPY_DEBUG
+    mypid = erts_get_current_pid();
+    VERBOSE_DEBUG("[pid=%T] copy_struct %p\n", mypid, obj);
+#endif
+
     DTRACE1(copy_struct, (int32_t)sz);
 
     hp = htop = *hpp;
@@ -529,9 +891,778 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     }
 #endif
     *hpp = (Eterm *) (hstart+hsize);
+    VERBOSE_DEBUG("[pid=%T] result is at %p\n", mypid, res);
     return res;
 }
 
+
+/*
+ *  Machinery for the table used by the sharing preserving copier
+ *  Using an ESTACK but not very transparently; consider refactoring
+ */
+
+#define DECLARE_SHTABLE(s)					\
+    DECLARE_ESTACK(s);						\
+    Uint ESTK_CONCAT(s,_offset) = 0
+#define DESTROY_SHTABLE(s) DESTROY_ESTACK(s)
+#define SHTABLE_INCR 4
+#define SHTABLE_NEXT(s)	ESTK_CONCAT(s,_offset)
+#define SHTABLE_PUSH(s,x,y,b)					\
+do {								\
+    if (s.sp > s.end - SHTABLE_INCR) {				\
+        erl_grow_estack(&s, ESTK_DEF_STACK(s));			\
+    }								\
+    *s.sp++ = (x);						\
+    *s.sp++ = (y);						\
+    *s.sp++ = (Eterm) NULL;					\
+    *s.sp++ = (Eterm) (b);	/* bad in HALF_WORD */		\
+    ESTK_CONCAT(s,_offset) += SHTABLE_INCR;			\
+} while(0)
+#define SHTABLE_X(s,e) (s.start[e])
+#define SHTABLE_Y(s,e) (s.start[(e)+1])
+#define SHTABLE_FWD(s,e) ((Eterm *) (s.start[(e)+2]))
+#define SHTABLE_FWD_UPD(s,e,p) (s.start[(e)+2] = (Eterm) (p))
+#define SHTABLE_REV(s,e) ((Eterm *) (s.start[(e)+3]))
+
+#define LIST_SHARED_UNPROCESSED ((Eterm) 0)
+#define LIST_SHARED_PROCESSED	((Eterm) 1)
+
+#define HEAP_ELEM_TO_BE_FILLED	_unchecked_make_list(NULL)
+
+
+/*
+ *  Specialized macros for using/reusing the persistent state
+ */
+
+#define DECLARE_EQUEUE_INIT_INFO(q, info)		\
+    UWord* EQUE_DEF_QUEUE(q) = info->queue_default;	\
+    ErtsEQueue q = {					\
+        EQUE_DEF_QUEUE(q), /* start */			\
+        EQUE_DEF_QUEUE(q), /* front */			\
+        EQUE_DEF_QUEUE(q), /* back */			\
+        1,                 /* possibly_empty */		\
+        EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */	\
+        ERTS_ALC_T_ESTACK  /* alloc_type */		\
+    }
+
+#define DECLARE_EQUEUE_FROM_INFO(q, info)		\
+    /* no EQUE_DEF_QUEUE(q), read-only */		\
+    ErtsEQueue q = {					\
+        info->queue_start,      /* start */		\
+        info->queue_start,      /* front */		\
+        info->queue_start,      /* back */		\
+        1,                      /* possibly_empty */	\
+        info->queue_end,        /* end */		\
+        info->queue_alloc_type  /* alloc_type */	\
+    }
+
+#define DECLARE_BITSTORE_INIT_INFO(s, info)		\
+    UWord* WSTK_DEF_STACK(s) = info->bitstore_default;	\
+    ErtsWStack s = {					\
+        WSTK_DEF_STACK(s),  /* wstart */		\
+        WSTK_DEF_STACK(s),  /* wsp */			\
+        WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */	\
+        ERTS_ALC_T_ESTACK /* alloc_type */		\
+    };							\
+    int WSTK_CONCAT(s,_bitoffs) = 0;			\
+    /* no WSTK_CONCAT(s,_offset), write-only */		\
+    UWord WSTK_CONCAT(s,_buffer) = 0
+
+#define DECLARE_BITSTORE_FROM_INFO(s, info)		\
+    /* no WSTK_DEF_STACK(s), read-only */		\
+    ErtsWStack s = {					\
+        info->bitstore_start,  /* wstart */		\
+        NULL,                  /* wsp, read-only */	\
+        NULL,                  /* wend, read-only */	\
+        info->bitstore_alloc_type /* alloc_type */	\
+    };							\
+    int WSTK_CONCAT(s,_bitoffs) = 0;			\
+    int WSTK_CONCAT(s,_offset) = 0;			\
+    UWord WSTK_CONCAT(s,_buffer) = 0
+
+#define DECLARE_SHTABLE_INIT_INFO(s, info)		\
+    Eterm* ESTK_DEF_STACK(s) = info->shtable_default;	\
+    ErtsEStack s = {					\
+        ESTK_DEF_STACK(s),  /* start */			\
+        ESTK_DEF_STACK(s),  /* sp */			\
+        ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */	\
+        ERTS_ALC_T_ESTACK /* alloc_type */		\
+    };							\
+    Uint ESTK_CONCAT(s,_offset) = 0
+
+#define DECLARE_SHTABLE_FROM_INFO(s, info)		\
+    /* no ESTK_DEF_STACK(s), read-only */		\
+    ErtsEStack s = {					\
+        info->shtable_start,     /* start */		\
+        NULL,                    /* sp, read-only */	\
+        NULL,                    /* end, read-only */	\
+        info->shtable_alloc_type /* alloc_type */	\
+    };							\
+    /* no ESTK_CONCAT(s,_offset), read-only */
+
+/*
+ *  Copy object "obj" preserving sharing.
+ *  First half: count size and calculate sharing.
+ *  NOTE: We do not support HALF_WORD (yet?).
+ */
+Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
+{
+    Uint sum;
+    Uint e;
+    unsigned sz;
+    Eterm* ptr;
+    Process* myself;
+    int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF;
+
+    DECLARE_EQUEUE_INIT_INFO(s, info);
+    DECLARE_BITSTORE_INIT_INFO(b, info);
+    DECLARE_SHTABLE_INIT_INFO(t, info);
+
+    /* step #0:
+       -------------------------------------------------------
+       get rid of the easy cases first:
+       - copying constants
+       - if not a proper process, do flat copy
+    */
+
+    if (IS_CONST(obj))
+	return 0;
+
+    myself = erts_get_current_process();
+    if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
+	return size_object(obj);
+
+    VERBOSE_DEBUG("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj);
+    VERBOSE_DEBUG("[pid=%T] message is %T\n", myself->common.id, obj);
+
+    /* step #1:
+       -------------------------------------------------------
+       traverse the term and calculate the size;
+       when traversing, transform as you do in size_shared
+       but when you find shared objects:
+
+       a. add entry in the table, indexed by i
+       b. mark them:
+	  b1. boxed terms, set header to (i | 11)
+	      store (old header, NONV, NULL, backptr) in the entry
+	  b2. cons cells, set CDR to NONV, set CAR to i
+	      store (old CAR, old CDR, NULL, backptr) in the entry
+    */
+
+    sum = 0;
+
+    for (;;) {
+	VERBOSE_DEBUG("[copy] visiting: %x ", obj);
+	switch (primary_tag(obj)) {
+	case TAG_PRIMARY_LIST: {
+	    Eterm head, tail;
+	    VERBOSE_DEBUG("L");
+	    ptr = list_val_rel(obj, base);
+	    /* off heap list pointers are copied verbatim */
+	    if (!INHEAP(myself, ptr)) {
+		VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj);
+		if (myself->mbuf != NULL)
+		    VERBOSE_DEBUG("[pid=%T] BUT !!! there are message buffers!\n", myself->common.id);
+		VERBOSE_DEBUG("#");
+		goto pop_next;
+	    }
+	    head = CAR(ptr);
+	    tail = CDR(ptr);
+	    /* if it's visited, don't count it;
+	       if not already shared, make it shared and store it in the table */
+	    if (primary_tag(tail) == TAG_PRIMARY_HEADER ||
+		primary_tag(head) == TAG_PRIMARY_HEADER) {
+		VERBOSE_DEBUG("!");
+		if (tail != THE_NON_VALUE) {
+		    e = SHTABLE_NEXT(t);
+		    VERBOSE_DEBUG("[pid=%T] tabling L %p\n", myself->common.id, ptr);
+		    SHTABLE_PUSH(t, head, tail, ptr);
+		    CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED;
+		    CDR(ptr) = THE_NON_VALUE;
+		}
+		goto pop_next;
+	    }
+	    /* else make it visited now */
+	    switch (primary_tag(tail)) {
+	    case TAG_PRIMARY_LIST:
+		VERBOSE_DEBUG("/L");
+		VERBOSE_DEBUG("[pid=%T] mangling L/L %p\n", myself->common.id, ptr);
+		CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_PRIMARY_IMMED1:
+		VERBOSE_DEBUG("/I");
+		VERBOSE_DEBUG("[pid=%T] mangling L/I %p\n", myself->common.id, ptr);
+		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
+		CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head);
+		break;
+	    case TAG_PRIMARY_BOXED:
+		VERBOSE_DEBUG("/B saved %d", primary_tag(head));
+		BITSTORE_PUT(b, primary_tag(head));
+		VERBOSE_DEBUG("[pid=%T] mangling L/B %p\n", myself->common.id, ptr);
+		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
+		CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER;
+		break;
+	    }
+	    /* and count it */
+	    sum += 2;
+	    if (!IS_CONST(head)) {
+		EQUEUE_PUT(s, head);
+	    }
+	    obj = tail;
+	    break;
+	}
+	case TAG_PRIMARY_BOXED: {
+	    Eterm hdr;
+	    VERBOSE_DEBUG("B");
+	    ptr = boxed_val_rel(obj, base);
+	    /* off heap pointers to boxes are copied verbatim */
+	    if (!INHEAP(myself, ptr)) {
+		VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj);
+		VERBOSE_DEBUG("#");
+		goto pop_next;
+	    }
+	    hdr = *ptr;
+	    /* if it's visited, don't count it;
+	       if not already shared, make it shared and store it in the table */
+	    if (primary_tag(hdr) != TAG_PRIMARY_HEADER) {
+		VERBOSE_DEBUG("!");
+		if (primary_tag(hdr) == BOXED_VISITED) {
+		    e = SHTABLE_NEXT(t);
+		    VERBOSE_DEBUG("[pid=%T] tabling B %p\n", myself->common.id, ptr);
+		    SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr);
+		    *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED;
+		}
+		goto pop_next;
+	    }
+	    /* else make it visited now */
+	    VERBOSE_DEBUG("[pid=%T] mangling B %p\n", myself->common.id, ptr);
+	    *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED;
+	    /* and count it */
+	    ASSERT(is_header(hdr));
+	    switch (hdr & _TAG_HEADER_MASK) {
+	    case ARITYVAL_SUBTAG: {
+		int arity = header_arity(hdr);
+		VERBOSE_DEBUG("/T");
+		sum += arity + 1;
+		if (arity == 0) { /* Empty tuple -- unusual. */
+		    VERBOSE_DEBUG("e");
+		    goto pop_next;
+		}
+		while (arity-- > 0) {
+		    obj = *++ptr;
+		    if (!IS_CONST(obj)) {
+			EQUEUE_PUT(s, obj);
+		    }
+		}
+		goto pop_next;
+	    }
+	    case FUN_SUBTAG: {
+		ErlFunThing* funp = (ErlFunThing *) ptr;
+		unsigned eterms = 1 /* creator */ + funp->num_free;
+		sz = thing_arityval(hdr);
+		VERBOSE_DEBUG("/F");
+		sum += 1 /* header */ + sz + eterms;
+		ptr += 1 /* header */ + sz;
+		while (eterms-- > 0) {
+		    obj = *ptr++;
+		    if (!IS_CONST(obj)) {
+			EQUEUE_PUT(s, obj);
+		    }
+		}
+		goto pop_next;
+	    }
+	    case SUB_BINARY_SUBTAG: {
+		ErlSubBin* sb = (ErlSubBin *) ptr;
+		Eterm real_bin = sb->orig;
+		Uint bit_offset = sb->bitoffs;
+		Uint bit_size = sb->bitsize;
+		size_t size = sb->size;
+		Uint extra_bytes;
+		Eterm hdr;
+		if (bit_size + bit_offset > 8) {
+		    sum += ERL_SUB_BIN_SIZE;
+		    extra_bytes = 2;
+		} else if (bit_size + bit_offset > 0) {
+		    sum += ERL_SUB_BIN_SIZE;
+		    extra_bytes = 1;
+		} else {
+		    extra_bytes = 0;
+		}
+		ASSERT(is_boxed(rterm2wterm(real_bin, base)) &&
+		       (((*boxed_val(rterm2wterm(real_bin, base))) &
+			 (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK))
+			== _TAG_HEADER_REFC_BIN));
+		hdr = *_unchecked_binary_val(rterm2wterm(real_bin, base)) & ~BOXED_VISITED_MASK;
+		if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) {
+		    sum += heap_bin_size(size+extra_bytes);
+		} else {
+		    ASSERT(thing_subtag(hdr) == REFC_BINARY_SUBTAG);
+		    sum += PROC_BIN_SIZE;
+		}
+		goto pop_next;
+	    }
+	    case BIN_MATCHSTATE_SUBTAG:
+		erl_exit(ERTS_ABORT_EXIT,
+			 "size_shared: matchstate term not allowed");
+	    default:
+		VERBOSE_DEBUG("/D");
+		sum += thing_arityval(hdr) + 1;
+		goto pop_next;
+	    }
+	    break;
+	}
+	case TAG_PRIMARY_IMMED1:
+	    VERBOSE_DEBUG("I");
+	pop_next:
+	    if (EQUEUE_ISEMPTY(s)) {
+		VERBOSE_DEBUG("\n");
+		// add sentinel to the table
+		SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL);
+		// store persistent info
+		BITSTORE_CLOSE(b);
+		info->queue_start = s.start;
+		info->queue_end = s.end;
+                info->queue_alloc_type = s.alloc_type;
+		info->bitstore_start = b.wstart;
+                info->bitstore_alloc_type = b.alloc_type;
+		info->shtable_start = t.start;
+                info->shtable_alloc_type = t.alloc_type;
+		// single point of return: the size of the object
+		VERBOSE_DEBUG("[pid=%T] size was: %u\n", myself->common.id, sum);
+		return sum;
+	    }
+	    obj = EQUEUE_GET(s);
+	    break;
+	default:
+	    erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj);
+	}
+	VERBOSE_DEBUG("\n");
+    }
+}
+
+
+/*
+ *  Copy object "obj" preserving sharing.
+ *  Second half: copy and restore the object.
+ *  NOTE: We do not support HALF_WORD (yet?).
+ */
+Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags)
+{
+    Uint e;
+    unsigned sz;
+    Eterm* ptr;
+    Eterm* hp;
+    Eterm* hscan;
+    Eterm result;
+    Eterm* resp;
+    unsigned remaining;
+    Process* myself;
+    int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF;
+#if defined(DEBUG) || defined(SHCOPY_DEBUG)
+    Eterm saved_obj = obj;
+#endif
+
+    DECLARE_EQUEUE_FROM_INFO(s, info);
+    DECLARE_BITSTORE_FROM_INFO(b, info);
+    DECLARE_SHTABLE_FROM_INFO(t, info);
+
+    /* step #0:
+       -------------------------------------------------------
+       get rid of the easy cases first:
+       - copying constants
+       - if not a proper process, do flat copy
+    */
+
+    if (IS_CONST(obj))
+	return obj;
+
+    myself = erts_get_current_process();
+    if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
+	return copy_struct(obj, size, hpp, off_heap);
+
+    VERBOSE_DEBUG("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj);
+
+    /* step #2: was performed before this function was called
+       -------------------------------------------------------
+       allocate new space
+    */
+
+    hscan = hp = *hpp;
+
+    /* step #3:
+       -------------------------------------------------------
+       traverse the term a second time and when traversing:
+       a. if the object is marked as shared
+	  a1. if the entry contains a forwarding ptr, use that
+	  a2. otherwise, copy it to the new space and store the
+	      forwarding ptr to the entry
+      b. otherwise, reverse-transform as you do in size_shared
+	 and copy to the new space
+    */
+
+    resp = &result;
+    remaining = 0;
+    for (;;) {
+	VERBOSE_DEBUG("[copy] revisiting: %x ", obj);
+	switch (primary_tag(obj)) {
+	case TAG_PRIMARY_LIST: {
+	    Eterm head, tail;
+	    VERBOSE_DEBUG("L");
+	    ptr = list_val_rel(obj, base);
+	    /* off heap list pointers are copied verbatim */
+	    if (!INHEAP(myself, ptr)) {
+		VERBOSE_DEBUG("#");
+		*resp = obj;
+		goto cleanup_next;
+	    }
+	    head = CAR(ptr);
+	    tail = CDR(ptr);
+	    /* if it is shared */
+	    if (tail == THE_NON_VALUE) {
+		e = head >> _TAG_PRIMARY_SIZE;
+		/* if it has been processed, just use the forwarding pointer */
+		if (primary_tag(head) == LIST_SHARED_PROCESSED) {
+		    VERBOSE_DEBUG("!");
+		    *resp = make_list(SHTABLE_FWD(t, e));
+		    goto cleanup_next;
+		}
+		/* else, let's process it now,
+		   copy it and keep the forwarding pointer */
+		else {
+		    VERBOSE_DEBUG("$");
+		    CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED;
+		    head = SHTABLE_X(t, e);
+		    tail = SHTABLE_Y(t, e);
+		    ptr = &(SHTABLE_X(t, e));
+		    VERBOSE_DEBUG("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e));
+		    SHTABLE_FWD_UPD(t, e, hp);
+		}
+	    }
+	    /* if not already clean, clean it up and copy it */
+	    if (primary_tag(tail) == TAG_PRIMARY_HEADER) {
+		if (primary_tag(head) == TAG_PRIMARY_HEADER) {
+		    Eterm saved = BITSTORE_GET(b);
+		    VERBOSE_DEBUG("/B restoring %d", saved);
+		    VERBOSE_DEBUG("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr);
+		    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved;
+		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED;
+		} else {
+		    VERBOSE_DEBUG("/L");
+		    VERBOSE_DEBUG("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr);
+		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST;
+		}
+	    } else if (primary_tag(head) == TAG_PRIMARY_HEADER) {
+		VERBOSE_DEBUG("/I");
+		VERBOSE_DEBUG("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr);
+		CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail);
+		CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1;
+	    } else {
+		ASSERT(0 && "cannot come here");
+		goto cleanup_next;
+	    }
+	    /* and its children too */
+	    if (IS_CONST(head)) {
+		CAR(hp) = head;
+	    } else {
+		EQUEUE_PUT_UNCHECKED(s, head);
+		CAR(hp) = HEAP_ELEM_TO_BE_FILLED;
+	    }
+	    *resp = make_list(hp);
+	    resp = &(CDR(hp));
+	    hp += 2;
+	    obj = tail;
+	    break;
+	}
+	case TAG_PRIMARY_BOXED: {
+	    Eterm hdr;
+	    VERBOSE_DEBUG("B");
+	    ptr = boxed_val_rel(obj, base);
+	    /* off heap pointers to boxes are copied verbatim */
+	    if (!INHEAP(myself, ptr)) {
+		VERBOSE_DEBUG("#");
+		*resp = obj;
+		goto cleanup_next;
+	    }
+	    hdr = *ptr;
+	    /* clean it up, unless it's already clean or shared and processed */
+	    switch (primary_tag(hdr)) {
+	    case TAG_PRIMARY_HEADER:
+		ASSERT(0 && "cannot come here");
+	    /* if it is shared and has been processed,
+	       just use the forwarding pointer */
+	    case BOXED_SHARED_PROCESSED:
+		VERBOSE_DEBUG("!");
+		e = hdr >> _TAG_PRIMARY_SIZE;
+		*resp = make_boxed(SHTABLE_FWD(t, e));
+		goto cleanup_next;
+	    /* if it is shared but has not been processed yet, let's process
+	       it now: copy it and keep the forwarding pointer */
+	    case BOXED_SHARED_UNPROCESSED:
+		e = hdr >> _TAG_PRIMARY_SIZE;
+		VERBOSE_DEBUG("$");
+		*ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED;
+		hdr = SHTABLE_X(t, e);
+		ASSERT(primary_tag(hdr) == BOXED_VISITED);
+		VERBOSE_DEBUG("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e));
+		VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr);
+		SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER;
+		SHTABLE_FWD_UPD(t, e, hp);
+		break;
+	    case BOXED_VISITED:
+		VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr);
+		*ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER;
+		break;
+	    }
+	    /* and its children too */
+	    switch (hdr & _TAG_HEADER_MASK) {
+	    case ARITYVAL_SUBTAG: {
+		int arity = header_arity(hdr);
+		*resp = make_boxed(hp);
+		*hp++ = hdr;
+		while (arity-- > 0) {
+		    obj = *++ptr;
+		    if (IS_CONST(obj)) {
+			*hp++ = obj;
+		    } else {
+			EQUEUE_PUT_UNCHECKED(s, obj);
+			*hp++ = HEAP_ELEM_TO_BE_FILLED;
+		    }
+		}
+		goto cleanup_next;
+	    }
+	    case FUN_SUBTAG: {
+		ErlFunThing* funp = (ErlFunThing *) ptr;
+		unsigned eterms = 1 /* creator */ + funp->num_free;
+		sz = thing_arityval(hdr);
+		funp = (ErlFunThing *) hp;
+		*resp = make_fun(hp);
+		*hp++ = hdr;
+		ptr++;
+		while (sz-- > 0) {
+		    *hp++ = *ptr++;
+		}
+		while (eterms-- > 0) {
+		    obj = *ptr++;
+		    if (IS_CONST(obj)) {
+			*hp++ = obj;
+		    } else {
+			EQUEUE_PUT_UNCHECKED(s, obj);
+			*hp++ = HEAP_ELEM_TO_BE_FILLED;
+		    }
+		}
+		funp->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*) funp;
+		erts_refc_inc(&funp->fe->refc, 2);
+		goto cleanup_next;
+	    }
+	    case REFC_BINARY_SUBTAG: {
+		ProcBin* pb = (ProcBin *) ptr;
+		sz = thing_arityval(hdr);
+		if (pb->flags) {
+		    erts_emasculate_writable_binary(pb);
+		}
+		pb = (ProcBin *) hp;
+		*resp = make_binary(hp);
+		*hp++ = hdr;
+		ptr++;
+		while (sz-- > 0) {
+		    *hp++ = *ptr++;
+		}
+		erts_refc_inc(&pb->val->refc, 2);
+		pb->next = off_heap->first;
+		pb->flags = 0;
+		off_heap->first = (struct erl_off_heap_header*) pb;
+		OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
+		goto cleanup_next;
+	    }
+	    case SUB_BINARY_SUBTAG: {
+		ErlSubBin* sb = (ErlSubBin *) ptr;
+		Eterm real_bin = sb->orig;
+		Uint bit_offset = sb->bitoffs;
+		Uint bit_size = sb->bitsize;
+		Uint offset = sb->offs;
+		size_t size = sb->size;
+		Uint extra_bytes;
+		Uint real_size;
+		if ((bit_size + bit_offset) > 8) {
+		    extra_bytes = 2;
+		} else if ((bit_size + bit_offset) > 0) {
+		    extra_bytes = 1;
+		} else {
+		    extra_bytes = 0;
+		}
+		real_size = size+extra_bytes;
+		ASSERT(is_boxed(rterm2wterm(real_bin, base)) &&
+		       (((*boxed_val(rterm2wterm(real_bin, base))) &
+			 (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK))
+			== _TAG_HEADER_REFC_BIN));
+		ptr = _unchecked_binary_val(rterm2wterm(real_bin, base));
+		*resp = make_binary(hp);
+		if (extra_bytes != 0) {
+		    ErlSubBin* res = (ErlSubBin *) hp;
+		    hp += ERL_SUB_BIN_SIZE;
+		    res->thing_word = HEADER_SUB_BIN;
+		    res->size = size;
+		    res->bitsize = bit_size;
+		    res->bitoffs = bit_offset;
+		    res->offs = 0;
+		    res->is_writable = 0;
+		    res->orig = make_binary(hp);
+		}
+		if (thing_subtag(*ptr & ~BOXED_VISITED_MASK) == HEAP_BINARY_SUBTAG) {
+		    ErlHeapBin* from = (ErlHeapBin *) ptr;
+		    ErlHeapBin* to = (ErlHeapBin *) hp;
+		    hp += heap_bin_size(real_size);
+		    to->thing_word = header_heap_bin(real_size);
+		    to->size = real_size;
+		    sys_memcpy(to->data, ((byte *)from->data)+offset, real_size);
+		} else {
+		    ProcBin* from = (ProcBin *) ptr;
+		    ProcBin* to = (ProcBin *) hp;
+		    ASSERT(thing_subtag(*ptr & ~BOXED_VISITED_MASK) == REFC_BINARY_SUBTAG);
+		    if (from->flags) {
+			erts_emasculate_writable_binary(from);
+		    }
+		    hp += PROC_BIN_SIZE;
+		    to->thing_word = HEADER_PROC_BIN;
+		    to->size = real_size;
+		    to->val = from->val;
+		    erts_refc_inc(&to->val->refc, 2);
+		    to->bytes = from->bytes + offset;
+		    to->next = off_heap->first;
+		    to->flags = 0;
+		    off_heap->first = (struct erl_off_heap_header*) to;
+		    OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
+		}
+		goto cleanup_next;
+	    }
+	    case EXTERNAL_PID_SUBTAG:
+	    case EXTERNAL_PORT_SUBTAG:
+	    case EXTERNAL_REF_SUBTAG: {
+		ExternalThing *etp = (ExternalThing *) hp;
+		sz = thing_arityval(hdr);
+		*resp = make_external(hp);
+		*hp++ = hdr;
+		ptr++;
+		while (sz-- > 0) {
+		    *hp++ = *ptr++;
+		}
+		etp->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*) etp;
+		erts_refc_inc(&etp->node->refc, 2);
+		goto cleanup_next;
+	    }
+	    default:
+		sz = thing_arityval(hdr);
+		*resp = make_boxed(hp);
+		*hp++ = hdr;
+		ptr++;
+		while (sz-- > 0) {
+		    *hp++ = *ptr++;
+		}
+		goto cleanup_next;
+	    }
+	    break;
+	}
+	case TAG_PRIMARY_IMMED1:
+	    *resp = obj;
+	cleanup_next:
+	    if (EQUEUE_ISEMPTY(s)) {
+		goto all_clean;
+	    }
+	    obj = EQUEUE_GET(s);
+	    for (;;) {
+		ASSERT(hscan < hp);
+		if (remaining == 0) {
+		    if (*hscan == HEAP_ELEM_TO_BE_FILLED) {
+			resp = hscan;
+			hscan += 2;
+			break; /* scanning loop */
+		    } else if (primary_tag(*hscan) == TAG_PRIMARY_HEADER) {
+			switch (*hscan & _TAG_HEADER_MASK) {
+			case ARITYVAL_SUBTAG:
+			    remaining = header_arity(*hscan);
+			    hscan++;
+			    break;
+			case FUN_SUBTAG: {
+			    ErlFunThing* funp = (ErlFunThing *) hscan;
+			    hscan += 1 + thing_arityval(*hscan);
+			    remaining = 1 + funp->num_free;
+			    break;
+			}
+			case SUB_BINARY_SUBTAG:
+			    ASSERT(((ErlSubBin *) hscan)->bitoffs +
+				   ((ErlSubBin *) hscan)->bitsize > 0);
+			    hscan += ERL_SUB_BIN_SIZE;
+			    break;
+			default:
+			    hscan += 1 + thing_arityval(*hscan);
+			    break;
+			}
+		    } else {
+			hscan++;
+		    }
+		} else if (*hscan == HEAP_ELEM_TO_BE_FILLED) {
+		    resp = hscan++;
+		    remaining--;
+		    break; /* scanning loop */
+		} else {
+		    hscan++;
+		    remaining--;
+		}
+	    }
+	    ASSERT(resp < hp);
+	    break;
+	default:
+	    erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
+	}
+	VERBOSE_DEBUG("\n");
+    }
+
+    /* step #4:
+       -------------------------------------------------------
+       traverse the table and reverse-transform all stored entries
+    */
+
+all_clean:
+    VERBOSE_DEBUG("\n");
+    for (e = 0; ; e += SHTABLE_INCR) {
+	ptr = SHTABLE_REV(t, e);
+	if (ptr == NULL)
+	    break;
+	VERBOSE_DEBUG("[copy] restoring shared: %x\n", ptr);
+	/* entry was a list */
+	if (SHTABLE_Y(t, e) != THE_NON_VALUE) {
+	    VERBOSE_DEBUG("[pid=%T] untabling L %p\n", myself->common.id, ptr);
+	    CAR(ptr) = SHTABLE_X(t, e);
+	    CDR(ptr) = SHTABLE_Y(t, e);
+	}
+	/* entry was boxed */
+	else {
+	    VERBOSE_DEBUG("[pid=%T] untabling B %p\n", myself->common.id, ptr);
+	    *ptr = SHTABLE_X(t, e);
+	    ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER);
+	}
+    }
+
+#ifdef DEBUG
+    if (eq(saved_obj, result) == 0) {
+	erts_fprintf(stderr, "original = %T\n", saved_obj);
+	erts_fprintf(stderr, "copy = %T\n", result);
+	erl_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n");
+    }
+#endif
+
+    VERBOSE_DEBUG("[pid=%T] original was %T\n", myself->common.id, saved_obj);
+    VERBOSE_DEBUG("[pid=%T] copy is %T\n", myself->common.id, result);
+    VERBOSE_DEBUG("[pid=%T] result is at %p\n", myself->common.id, result);
+
+    ASSERT(hp == *hpp + size);
+    *hpp = hp;
+    return result;
+}
+
+
 /*
  * Copy a term that is guaranteed to be contained in a single
  * heap block. The heap block is copied word by word, and any
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6cb37752bc..ec96a7563a 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,6 +42,8 @@
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
 
+#undef SHCOPY_DEBUG
+
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10
@@ -1166,6 +1168,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
 
+#ifdef SHCOPY_DEBUG
+    VERBOSE_DEBUG("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p));
+#endif
+
     n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 					       sizeof(Eterm)*new_sz);
 
@@ -1383,6 +1389,10 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     Uint new_sz, stk_sz;
     int adjusted;
 
+#ifdef SHCOPY_DEBUG
+    VERBOSE_DEBUG("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p));
+#endif
+
     /*
      * Do a fullsweep GC. First figure out the size of the heap
      * to receive all live data.
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 20f100b427..d99f6548f9 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -919,8 +919,9 @@ do {							\
 #define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty)
 
 #define EQUEUE_GET(q) ({				\
+    UWord x;						\
     q.possibly_empty = 1;				\
-    UWord x = *(q.front);				\
+    x = *(q.front);					\
     if (++(q.front) == q.end) {				\
         q.front = q.start;				\
     }							\
@@ -1035,11 +1036,64 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...);
 __decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...);
 void erl_error(char*, va_list);
 
+/* This controls whether sharing-preserving copy is used by Erlang */
+
+#define SHCOPY_SEND
+#define SHCOPY_SPAWN
+
+#if defined(SHCOPY_SEND) \
+ || defined(SHCOPY_SPAWN)
+#define SHCOPY
+/* Use this with care, it is *very* verbose! */
+#undef SHCOPY_DEBUG
+#endif
+
+#define VERBOSE_DEBUG(...) do {		\
+    erts_fprintf(stderr, __VA_ARGS__);	\
+  } while(0)
+
+#define ERTS_SHCOPY_FLG_MASK	(((unsigned) 3) << 0)
+#define ERTS_SHCOPY_FLG_NONE	(((unsigned) 1) << 0)
+#define ERTS_SHCOPY_FLG_TMP_BUF	(((unsigned) 1) << 1)
+
+/* The persistent state while the sharing-preserving copier works */
+
+typedef struct shcopy_info {
+    Eterm  queue_default[DEF_EQUEUE_SIZE];
+    Eterm* queue_start;
+    Eterm* queue_end;
+    ErtsAlcType_t queue_alloc_type;
+    UWord  bitstore_default[DEF_WSTACK_SIZE];
+    UWord* bitstore_start;
+    ErtsAlcType_t bitstore_alloc_type;
+    Eterm  shtable_default[DEF_ESTACK_SIZE];
+    Eterm* shtable_start;
+    ErtsAlcType_t shtable_alloc_type;
+} shcopy_info;
+
+#define DESTROY_INFO(info)						\
+do {									\
+    if (info.queue_start != info.queue_default) {			\
+	erts_free(info.queue_alloc_type, info.queue_start);		\
+    }									\
+    if (info.bitstore_start != info.bitstore_default) {			\
+	erts_free(info.bitstore_alloc_type, info.bitstore_start);	\
+    }									\
+    if (info.shtable_start != info.shtable_default) {			\
+	erts_free(info.shtable_alloc_type, info.shtable_start);		\
+    }									\
+} while(0)
+
 /* copy.c */
 Eterm copy_object_x(Eterm, Process*, Uint);
 #define copy_object(Term, Proc) copy_object_x(Term,Proc,0)
 
 Uint size_object(Eterm);
+Uint copy_shared_calculate(Eterm, shcopy_info*, unsigned);
+Eterm copy_shared_perform(Eterm, Uint, shcopy_info*, Eterm**, ErlOffHeap*, unsigned);
+
+Uint size_shared(Eterm);
+
 Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
 
-- 
cgit v1.2.3


From cfe998988c942bed0fbf026c00a2531fdf5aba7c Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Sat, 9 Jun 2012 00:19:28 +0300
Subject: Add the BIF size_shared/1 and debug cleanup

---
 erts/emulator/beam/beam_debug.c | 14 ++++++++++++++
 erts/emulator/beam/bif.tab      | 12 +++++++-----
 erts/emulator/beam/erl_gc.c     |  2 --
 erts/emulator/beam/global.h     |  4 ++++
 4 files changed, 25 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index e989310789..40e3f4db4e 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -73,6 +73,20 @@ erts_debug_flat_size_1(BIF_ALIST_1)
     }
 }
 
+BIF_RETTYPE
+erts_debug_size_shared_1(BIF_ALIST_1)
+{
+    Process* p = BIF_P;
+    Eterm term = BIF_ARG_1;
+    Uint size = size_shared(term);
+
+    if (IS_USMALL(0, size)) {
+	BIF_RET(make_small(size));
+    } else {
+	Eterm* hp = HAlloc(p, BIG_UINT_HEAP_SIZE);
+	BIF_RET(uint_to_big(size, hp));
+    }
+}
 
 BIF_RETTYPE
 erts_debug_breakpoint_2(BIF_ALIST_2)
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 65f8d6f1f5..f45f886395 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -639,14 +639,16 @@ bif ets:update_counter/4
 bif erts_debug:map_info/1
 
 #
-# Obsolete
+# New in 19.0
 #
 
-bif erlang:hash/2
+bif binary:split/2
+bif binary:split/3
+bif erts_debug:size_shared/1
+bif 'erl.system.debug':size_shared/1	ebif_erts_debug_size_shared_1
 
 #
-# New in 19.0
+# Obsolete
 #
 
-bif binary:split/2
-bif binary:split/3
+bif erlang:hash/2
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index ec96a7563a..1cf6509012 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,8 +42,6 @@
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
 
-#undef SHCOPY_DEBUG
-
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d99f6548f9..c2056dafaa 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1048,9 +1048,13 @@ void erl_error(char*, va_list);
 #undef SHCOPY_DEBUG
 #endif
 
+#ifdef SHCOPY_DEBUG
 #define VERBOSE_DEBUG(...) do {		\
     erts_fprintf(stderr, __VA_ARGS__);	\
   } while(0)
+#else
+#define VERBOSE_DEBUG(...)
+#endif
 
 #define ERTS_SHCOPY_FLG_MASK	(((unsigned) 3) << 0)
 #define ERTS_SHCOPY_FLG_NONE	(((unsigned) 1) << 0)
-- 
cgit v1.2.3


From f3ae60838a5e8c83cfdd3000e25562dfcbbd438d Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Sat, 9 Jun 2012 01:12:50 +0300
Subject: Enable shcopy for sending messages

---
 erts/emulator/beam/erl_db.c      |   4 +-
 erts/emulator/beam/erl_message.c | 114 +++++++++++++++++++++++++--------------
 erts/emulator/beam/erl_message.h |   5 ++
 erts/emulator/beam/global.h      |   7 +++
 4 files changed, 87 insertions(+), 43 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 9ec14ab5ae..6119a9225f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1832,7 +1832,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 			     tb->common.id,
 			     from_pid,
 			     BIF_ARG_3), 
-		      0);
+		      ERTS_SND_FLG_SHCOPY_TMP_BUF);
     erts_smp_proc_unlock(to_proc, to_locks);
     UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
@@ -3211,7 +3211,7 @@ retry:
 			     tb->common.id,
 			     p->common.id,
 			     heir_data), 
-		      0);
+		      ERTS_SND_FLG_SHCOPY_TMP_BUF);
     erts_smp_proc_unlock(to_proc, to_locks);
     return !0;
 }
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 79739501a8..b1fca1df0c 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -68,6 +68,9 @@ new_message_buffer(Uint size)
     bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG,
 					    ERTS_HEAP_FRAG_SIZE(size));
     ERTS_INIT_HEAP_FRAG(bp, size, size);
+#ifdef SHCOPY_DEBUG
+    VERBOSE_DEBUG("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem);
+#endif
     return bp;
 }
 
@@ -248,36 +251,6 @@ erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_s
     return nmp;
 }
 
-void
-erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp)
-{
-    if (first_bp) {
-	ErlHeapFragment *bp = first_bp;
-
-	while (1) {
-	    /* Move any off_heap's into the process */
-	    if (bp->off_heap.first != NULL) {
-		struct erl_off_heap_header** next_p = &bp->off_heap.first;
-		while (*next_p != NULL) {
-		    next_p = &((*next_p)->next);
-		}
-		*next_p = MSO(proc).first;
-		MSO(proc).first = bp->off_heap.first;
-		bp->off_heap.first = NULL;
-		OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
-	    }
-	    MBUF_SIZE(proc) += bp->used_size;
-	    if (!bp->next)
-		break;
-	    bp = bp->next;
-	}
-
-	/* Link the message buffer */
-	bp->next = MBUF(proc);
-	MBUF(proc) = first_bp;
-    }
-}
-
 void
 erts_queue_dist_message(Process *rcvr,
 			ErtsProcLocks *rcvr_locks,
@@ -540,6 +513,36 @@ erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks,
 	);
 }
 
+void
+erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp)
+{
+    if (first_bp) {
+	ErlHeapFragment *bp = first_bp;
+
+	while (1) {
+	    /* Move any off_heap's into the process */
+	    if (bp->off_heap.first != NULL) {
+		struct erl_off_heap_header** next_p = &bp->off_heap.first;
+		while (*next_p != NULL) {
+		    next_p = &((*next_p)->next);
+		}
+		*next_p = MSO(proc).first;
+		MSO(proc).first = bp->off_heap.first;
+		bp->off_heap.first = NULL;
+		OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
+	    }
+	    MBUF_SIZE(proc) += bp->used_size;
+	    if (!bp->next)
+		break;
+	    bp = bp->next;
+	}
+
+	/* Link the message buffer */
+	bp->next = MBUF(proc);
+	MBUF(proc) = first_bp;
+    }
+}
+
 Uint
 erts_msg_attached_data_size_aux(ErtsMessage *msg)
 {
@@ -663,11 +666,15 @@ erts_send_message(Process* sender,
     Eterm utag = NIL;
 #endif
     erts_aint32_t receiver_state;
+#ifdef SHCOPY_SEND
+    unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT;
+    erts_shcopy_t info;
+#endif
     BM_STOP_TIMER(system);
     BM_MESSAGE(message,sender,receiver);
     BM_START_TIMER(send);
 
- #ifdef USE_VM_PROBES
+#ifdef USE_VM_PROBES
     *sender_name = *receiver_name = '\0';
     if (DTRACE_ENABLED(message_send)) {
         erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
@@ -686,15 +693,23 @@ erts_send_message(Process* sender,
 #ifdef USE_VM_PROBES
 	Uint dt_utag_size = 0;
 #endif
-
+#ifdef SHCOPY_SEND
+        unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT;
+        erts_shcopy_t info;
+        INITIALIZE_SHCOPY(info);
+#endif
 	BM_SWAP_TIMER(send,size);
-	msize = size_object(message);
+#ifdef SHCOPY_SEND
+        INITIALIZE_SHCOPY(info);
+        msize = copy_shared_calculate(message, &info, shflags);
+#else
+        msize = size_object(message);
+#endif
 	BM_SWAP_TIMER(size,send);
 
 #ifdef USE_VM_PROBES
 	if (stoken != am_have_dt_utag) {
 #endif
-
 	    seq_trace_update_send(sender);
 	    seq_trace_output(stoken, message, SEQ_TRACE_SEND, 
 			     receiver->common.id, sender);
@@ -720,14 +735,20 @@ erts_send_message(Process* sender,
 					   &ohp);
 
         BM_SWAP_TIMER(send,copy);
+
+#ifdef SHCOPY_SEND
+	if (is_not_immed(message))
+            message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags);
+        DESTROY_SHCOPY(info);
+#else
+	if (is_not_immed(message))
+            message = copy_struct(message, msize, &hp, ohp);
+#endif
 	if (is_immed(stoken))
 	    token = stoken;
 	else
 	    token = copy_struct(stoken, seq_trace_size, &hp, ohp);
 
-	if (is_not_immed(message))
-	    message = copy_struct(message, msize, &hp, ohp);
-
 #ifdef USE_VM_PROBES
 	if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
 	    if (is_immed(DT_UTAG(sender)))
@@ -764,7 +785,12 @@ erts_send_message(Process* sender,
 	}
 	else {
 	    BM_SWAP_TIMER(send,size);
-	    msize = size_object(message);
+#ifdef SHCOPY_SEND
+            INITIALIZE_SHCOPY(info);
+            msize = copy_shared_calculate(message, &info, shflags);
+#else
+            msize = size_object(message);
+#endif
 	    BM_SWAP_TIMER(size,send);
 
 	    mp = erts_alloc_message_heap_state(receiver,
@@ -774,8 +800,14 @@ erts_send_message(Process* sender,
 					       &hp,
 					       &ohp);
 	    BM_SWAP_TIMER(send,copy);
-	    if (is_not_immed(message))
-		message = copy_struct(message, msize, &hp, ohp);
+#ifdef SHCOPY_SEND
+            if (is_not_immed(message))
+                message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags);
+            DESTROY_SHCOPY(info);
+#else
+            if (is_not_immed(message))
+                message = copy_struct(message, msize, &hp, ohp);
+#endif
 	    BM_MESSAGE_COPIED(msz);
 	    BM_SWAP_TIMER(copy,send);
 	}
@@ -800,6 +832,7 @@ erts_send_message(Process* sender,
     return res;
 }
 
+
 /*
  * This function delivers an EXIT message to a process
  * which is trapping EXITs.
@@ -1725,4 +1758,3 @@ void erts_factory_undo(ErtsHeapFactory* factory)
     factory->heap_frags = NULL;
 #endif
 }
-
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 740ae46a0f..0de36c3199 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -238,6 +238,11 @@ do {									\
 
 #define ERTS_SND_FLG_NO_SEQ_TRACE		(((unsigned) 1) << 0)
 
+#define ERTS_SND_FLG_SHCOPY_SHIFT	1
+#define ERTS_SND_FLG_SHCOPY_MASK	(ERTS_SHCOPY_FLG_MASK    << ERTS_SND_FLG_SHCOPY_SHIFT)
+#define ERTS_SND_FLG_SHCOPY_NONE	(ERTS_SHCOPY_FLG_NONE    << ERTS_SND_FLG_SHCOPY_SHIFT)
+#define ERTS_SND_FLG_SHCOPY_TMP_BUF	(ERTS_SHCOPY_FLG_TMP_BUF << ERTS_SND_FLG_SHCOPY_SHIFT)
+
 #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
    (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c2056dafaa..a5e6ff6f1c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1075,6 +1075,13 @@ typedef struct shcopy_info {
     ErtsAlcType_t shtable_alloc_type;
 } shcopy_info;
 
+#define INITIALIZE_INFO(info)						\
+do {									\
+    info.queue_start = info.queue_default;				\
+    info.bitstore_start = info.bitstore_default;			\
+    info.shtable_start = info.shtable_default;				\
+} while(0)
+
 #define DESTROY_INFO(info)						\
 do {									\
     if (info.queue_start != info.queue_default) {			\
-- 
cgit v1.2.3


From 7141fa8e4534ab7e4dcc3cad6c46872efa0e6e66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 17 Nov 2015 14:44:52 +0100
Subject: Remove DTrace Harddebug clutter

---
 erts/emulator/beam/beam_emu.c | 19 -------------------
 1 file changed, 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 208a16dfd0..af97cba61c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1914,20 +1914,7 @@ void process_main(void)
 	 if (DT_UTAG(c_p) != NIL) {
 	     if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) {
 		 SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag;
-#ifdef DTRACE_TAG_HARDDEBUG
-		 if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) 
-		     erts_fprintf(stderr,
-				  "Dtrace -> (%T) stop spreading "
-				  "tag %T with message %T\r\n",
-				  c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
-#endif
 	     } else {
-#ifdef DTRACE_TAG_HARDDEBUG
-		 erts_fprintf(stderr,
-			      "Dtrace -> (%T) kill tag %T with "
-			      "message %T\r\n",
-			      c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
-#endif
 		 DT_UTAG(c_p) = NIL;
 		 SEQ_TRACE_TOKEN(c_p) = NIL;
 	     }
@@ -1947,12 +1934,6 @@ void process_main(void)
 		 DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp);
 	     }
 	     DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING;
-#ifdef DTRACE_TAG_HARDDEBUG
-	     erts_fprintf(stderr,
-			  "Dtrace -> (%T) receive tag (%T) "
-			  "with message %T\r\n",
-			  c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp));
-#endif
 	 } else {
 #endif
 	     ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
-- 
cgit v1.2.3


From 8d02ad60c88aa060ff83ff3179bc8c0ab66868ee Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Fri, 14 Mar 2014 11:32:46 +0200
Subject: Add machinery to enable SHCOPY dynamically

This commit is just for debugging purposes, will probably be reverted.
It comes with a the erts_debug:copy_shared/1 BIF.  If SHCOPY_DISABLE
is defined, SHCOPY starts disabled and is dynamically enabled the first
time that the BIF is called.
---
 erts/emulator/beam/beam_debug.c | 26 ++++++++++++++++++++++++++
 erts/emulator/beam/copy.c       | 12 ++++++++++++
 erts/emulator/beam/global.h     |  2 ++
 3 files changed, 40 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 40e3f4db4e..36f3cfabbc 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -88,6 +88,32 @@ erts_debug_size_shared_1(BIF_ALIST_1)
     }
 }
 
+BIF_RETTYPE
+erts_debug_copy_shared_1(BIF_ALIST_1)
+{
+    Process* p = BIF_P;
+    Eterm term = BIF_ARG_1;
+    Uint size;
+    Eterm* hp;
+    Eterm copy;
+    shcopy_info info;
+#ifdef SHCOPY_DISABLE
+    extern int disable_copy_shared;
+#endif
+    INITIALIZE_INFO(info);
+
+    size = copy_shared_calculate(term, &info, 0);
+    if (size > 0) {
+      hp = HAlloc(p, size);
+    }
+    copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap, 0);
+    DESTROY_INFO(info);
+#ifdef SHCOPY_DISABLE
+    disable_copy_shared = 0;
+#endif
+    BIF_RET(copy);
+}
+
 BIF_RETTYPE
 erts_debug_breakpoint_2(BIF_ALIST_2)
 {
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index b31e043f08..2566707717 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -901,6 +901,10 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
  *  Using an ESTACK but not very transparently; consider refactoring
  */
 
+#ifdef SHCOPY_DISABLE
+int disable_copy_shared = ERTS_SHCOPY_FLG_NONE;
+#endif
+
 #define DECLARE_SHTABLE(s)					\
     DECLARE_ESTACK(s);						\
     Uint ESTK_CONCAT(s,_offset) = 0
@@ -1028,6 +1032,10 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
     if (IS_CONST(obj))
 	return 0;
 
+#ifdef SHCOPY_DISABLE
+    flags |= disable_copy_shared;
+#endif
+
     myself = erts_get_current_process();
     if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
 	return size_object(obj);
@@ -1276,6 +1284,10 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
     if (IS_CONST(obj))
 	return obj;
 
+#ifdef SHCOPY_DISABLE
+    flags |= disable_copy_shared;
+#endif
+
     myself = erts_get_current_process();
     if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
 	return copy_struct(obj, size, hpp, off_heap);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index a5e6ff6f1c..3e1f3664bc 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1044,6 +1044,8 @@ void erl_error(char*, va_list);
 #if defined(SHCOPY_SEND) \
  || defined(SHCOPY_SPAWN)
 #define SHCOPY
+/* Use this if you want sharing-preserving copy to be initially disabled */
+#undef SHCOPY_DISABLE
 /* Use this with care, it is *very* verbose! */
 #undef SHCOPY_DEBUG
 #endif
-- 
cgit v1.2.3


From 244e9d5855d1b1f160d667b5cf369defee72829d Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Sat, 9 Jun 2012 01:37:29 +0300
Subject: Enable shcopy for spawning processes

---
 erts/emulator/beam/erl_process.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9acce8acb6..f2da5289d3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10747,6 +10747,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     Eterm res = THE_NON_VALUE;
     erts_aint32_t state = 0;
     erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL;
+#ifdef SHCOPY_SPAWN
+    unsigned shflags = 0;	/* could be taken from so->flags, if necessary */
+    shcopy_info info;
+    INITIALIZE_INFO(info);
+#endif
 
 #ifdef ERTS_SMP
     erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -10798,7 +10803,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     BM_COUNT(processes_spawned);
 
     BM_SWAP_TIMER(system,size);
+#ifdef SHCOPY_SPAWN
+    arg_size = copy_shared_calculate(args, &info, shflags);
+#else
     arg_size = size_object(args);
+#endif
     BM_SWAP_TIMER(size,system);
     heap_need = arg_size;
 
@@ -10876,7 +10885,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     BM_MESSAGE(args,p,parent);
     BM_START_TIMER(system);
     BM_SWAP_TIMER(system,copy);
+#ifdef SHCOPY_SPAWN
+    p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap, shflags);
+    DESTROY_INFO(info);
+#else
     p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap);
+#endif
     BM_MESSAGE_COPIED(arg_size);
     BM_SWAP_TIMER(copy,system);
     p->arity = 3;
@@ -11260,6 +11274,8 @@ static void
 delete_process(Process* p)
 {
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
+    VERBOSE_DEBUG("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
+                  HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p));
 
     /* Cleanup psd */
 
-- 
cgit v1.2.3


From 277e8e77384ed6628009243e63d62f0555d10c69 Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Mon, 11 Jun 2012 15:17:01 +0300
Subject: Add -debug +vc flag for debuging SHCOPY

This is very verbose, you have been warned.
It should work with the copy-spy.py script, which may be a bit outdated.
---
 erts/emulator/beam/copy.c        | 131 ++++++++++-----------------------------
 erts/emulator/beam/erl_debug.h   |   1 +
 erts/emulator/beam/erl_gc.c      |  10 ++-
 erts/emulator/beam/erl_init.c    |   2 +
 erts/emulator/beam/erl_message.c |   4 +-
 erts/emulator/beam/erl_process.c |   4 +-
 erts/emulator/beam/global.h      |  10 ---
 7 files changed, 43 insertions(+), 119 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 2566707717..f74c6b1c89 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -75,16 +75,13 @@ Uint size_object(Eterm obj)
     Uint sum = 0;
     Eterm* ptr;
     int arity;
-#ifdef SHCOPY_DEBUG
-    Eterm mypid;
+#ifdef DEBUG
+    Eterm mypid = erts_get_current_pid();
 #endif
 
     DECLARE_ESTACK(s);
 
-#ifdef SHCOPY_DEBUG
-    mypid = erts_get_current_pid();
-    VERBOSE_DEBUG("[pid=%T] size_object %p\n", mypid, obj);
-#endif
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj));
 
     for (;;) {
 	switch (primary_tag(obj)) {
@@ -220,7 +217,7 @@ Uint size_object(Eterm obj)
 	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
-		VERBOSE_DEBUG("[pid=%T] size was: %u\n", mypid, sum);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum));
 		return sum;
 	    }
 	    obj = ESTACK_POP(s);
@@ -320,11 +317,9 @@ Uint size_shared(Eterm obj)
 	return size_object(obj);
 
     for (;;) {
-	VERBOSE_DEBUG("[size] visiting: %x ", obj);
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    VERBOSE_DEBUG("L");
 	    ptr = list_val_rel(obj, base);
 	    /* we're not counting anything that's outside our heap */
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
@@ -335,22 +330,18 @@ Uint size_shared(Eterm obj)
 	    /* if it's visited, don't count it */
 	    if (primary_tag(tail) == TAG_PRIMARY_HEADER ||
 		primary_tag(head) == TAG_PRIMARY_HEADER) {
-		VERBOSE_DEBUG("!");
 		goto pop_next;
 	    }
 	    /* else make it visited now */
 	    switch (primary_tag(tail)) {
 	    case TAG_PRIMARY_LIST:
-		VERBOSE_DEBUG("/L");
 		ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER;
 		break;
 	    case TAG_PRIMARY_IMMED1:
-		VERBOSE_DEBUG("/I");
 		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
 		CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head);
 		break;
 	    case TAG_PRIMARY_BOXED:
-		VERBOSE_DEBUG("/B saved %d", primary_tag(head));
 		BITSTORE_PUT(b, primary_tag(head));
 		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
 		CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER;
@@ -366,7 +357,6 @@ Uint size_shared(Eterm obj)
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    VERBOSE_DEBUG("B");
 	    ptr = boxed_val_rel(obj, base);
 	    /* we're not counting anything that's outside our heap */
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
@@ -375,7 +365,6 @@ Uint size_shared(Eterm obj)
 	    hdr = *ptr;
 	    /* if it's visited, don't count it */
 	    if (primary_tag(hdr) != TAG_PRIMARY_HEADER) {
-		VERBOSE_DEBUG("!");
 		goto pop_next;
 	    }
 	    /* else make it visited now */
@@ -385,10 +374,8 @@ Uint size_shared(Eterm obj)
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG: {
 		int arity = header_arity(hdr);
-		VERBOSE_DEBUG("/T");
 		sum += arity + 1;
 		if (arity == 0) { /* Empty tuple -- unusual. */
-		    VERBOSE_DEBUG("e");
 		    goto pop_next;
 		}
 		while (arity-- > 0) {
@@ -403,7 +390,6 @@ Uint size_shared(Eterm obj)
 		ErlFunThing* funp = (ErlFunThing *) ptr;
 		unsigned eterms = 1 /* creator */ + funp->num_free;
 		unsigned sz = thing_arityval(hdr);
-		VERBOSE_DEBUG("/F");
 		sum += 1 /* header */ + sz + eterms;
 		ptr += 1 /* header */ + sz;
 		while (eterms-- > 0) {
@@ -442,14 +428,12 @@ Uint size_shared(Eterm obj)
 		erl_exit(ERTS_ABORT_EXIT,
 			 "size_shared: matchstate term not allowed");
 	    default:
-		VERBOSE_DEBUG("/D");
 		sum += thing_arityval(hdr) + 1;
 		goto pop_next;
 	    }
 	    break;
 	}
 	case TAG_PRIMARY_IMMED1:
-	    VERBOSE_DEBUG("I");
 	pop_next:
 	    if (EQUEUE_ISEMPTY(s)) {
 		goto cleanup;
@@ -459,19 +443,15 @@ Uint size_shared(Eterm obj)
 	default:
 	    erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
 	}
-	VERBOSE_DEBUG("\n");
     }
 
 cleanup:
-    VERBOSE_DEBUG("\n");
     obj = saved_obj;
     BITSTORE_CLOSE(b);
     for (;;) {
-	VERBOSE_DEBUG("[size] revisiting: %x ", obj);
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    VERBOSE_DEBUG("L");
 	    ptr = list_val_rel(obj, base);
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
 		goto cleanup_next;
@@ -482,19 +462,15 @@ cleanup:
 	    if (primary_tag(tail) == TAG_PRIMARY_HEADER) {
 		if (primary_tag(head) == TAG_PRIMARY_HEADER) {
 		    Eterm saved = BITSTORE_GET(b);
-		    VERBOSE_DEBUG("/B restoring %d", saved);
 		    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved;
 		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED;
 		} else {
-		    VERBOSE_DEBUG("/L");
 		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST;
 		}
 	    } else if (primary_tag(head) == TAG_PRIMARY_HEADER) {
-		VERBOSE_DEBUG("/I");
 		CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail);
 		CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1;
 	    } else {
-		VERBOSE_DEBUG("!");
 		goto cleanup_next;
 	    }
 	    /* and its children too */
@@ -506,7 +482,6 @@ cleanup:
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    VERBOSE_DEBUG("B");
 	    ptr = boxed_val_rel(obj, base);
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
 		goto cleanup_next;
@@ -563,11 +538,9 @@ cleanup:
 	default:
 	    erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
 	}
-	VERBOSE_DEBUG("\n");
     }
 
  all_clean:
-    VERBOSE_DEBUG("\n");
     /* Return the result */
     DESTROY_EQUEUE(s);
     DESTROY_BITSTORE(b);
@@ -597,18 +570,13 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 #ifdef DEBUG
     Eterm org_obj = obj;
     Uint org_sz = sz;
-#endif
-#ifdef SHCOPY_DEBUG
-    Eterm mypid;
+    Eterm mypid = erts_get_current_pid();
 #endif
 
     if (IS_CONST(obj))
 	return obj;
 
-#ifdef SHCOPY_DEBUG
-    mypid = erts_get_current_pid();
-    VERBOSE_DEBUG("[pid=%T] copy_struct %p\n", mypid, obj);
-#endif
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_struct %p\n", mypid, obj));
 
     DTRACE1(copy_struct, (int32_t)sz);
 
@@ -891,7 +859,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     }
 #endif
     *hpp = (Eterm *) (hstart+hsize);
-    VERBOSE_DEBUG("[pid=%T] result is at %p\n", mypid, res);
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res));
     return res;
 }
 
@@ -1040,8 +1008,8 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
     if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
 	return size_object(obj);
 
-    VERBOSE_DEBUG("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj);
-    VERBOSE_DEBUG("[pid=%T] message is %T\n", myself->common.id, obj);
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", myself->common.id, obj));
 
     /* step #1:
        -------------------------------------------------------
@@ -1060,18 +1028,13 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
     sum = 0;
 
     for (;;) {
-	VERBOSE_DEBUG("[copy] visiting: %x ", obj);
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    VERBOSE_DEBUG("L");
 	    ptr = list_val_rel(obj, base);
 	    /* off heap list pointers are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
-		VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj);
-		if (myself->mbuf != NULL)
-		    VERBOSE_DEBUG("[pid=%T] BUT !!! there are message buffers!\n", myself->common.id);
-		VERBOSE_DEBUG("#");
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
 		goto pop_next;
 	    }
 	    head = CAR(ptr);
@@ -1080,10 +1043,9 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 	       if not already shared, make it shared and store it in the table */
 	    if (primary_tag(tail) == TAG_PRIMARY_HEADER ||
 		primary_tag(head) == TAG_PRIMARY_HEADER) {
-		VERBOSE_DEBUG("!");
 		if (tail != THE_NON_VALUE) {
 		    e = SHTABLE_NEXT(t);
-		    VERBOSE_DEBUG("[pid=%T] tabling L %p\n", myself->common.id, ptr);
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", myself->common.id, ptr));
 		    SHTABLE_PUSH(t, head, tail, ptr);
 		    CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED;
 		    CDR(ptr) = THE_NON_VALUE;
@@ -1093,20 +1055,17 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 	    /* else make it visited now */
 	    switch (primary_tag(tail)) {
 	    case TAG_PRIMARY_LIST:
-		VERBOSE_DEBUG("/L");
-		VERBOSE_DEBUG("[pid=%T] mangling L/L %p\n", myself->common.id, ptr);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", myself->common.id, ptr));
 		CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER;
 		break;
 	    case TAG_PRIMARY_IMMED1:
-		VERBOSE_DEBUG("/I");
-		VERBOSE_DEBUG("[pid=%T] mangling L/I %p\n", myself->common.id, ptr);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", myself->common.id, ptr));
 		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
 		CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head);
 		break;
 	    case TAG_PRIMARY_BOXED:
-		VERBOSE_DEBUG("/B saved %d", primary_tag(head));
 		BITSTORE_PUT(b, primary_tag(head));
-		VERBOSE_DEBUG("[pid=%T] mangling L/B %p\n", myself->common.id, ptr);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", myself->common.id, ptr));
 		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
 		CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER;
 		break;
@@ -1121,39 +1080,34 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    VERBOSE_DEBUG("B");
 	    ptr = boxed_val_rel(obj, base);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
-		VERBOSE_DEBUG("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj);
-		VERBOSE_DEBUG("#");
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
 		goto pop_next;
 	    }
 	    hdr = *ptr;
 	    /* if it's visited, don't count it;
 	       if not already shared, make it shared and store it in the table */
 	    if (primary_tag(hdr) != TAG_PRIMARY_HEADER) {
-		VERBOSE_DEBUG("!");
 		if (primary_tag(hdr) == BOXED_VISITED) {
 		    e = SHTABLE_NEXT(t);
-		    VERBOSE_DEBUG("[pid=%T] tabling B %p\n", myself->common.id, ptr);
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", myself->common.id, ptr));
 		    SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr);
 		    *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED;
 		}
 		goto pop_next;
 	    }
 	    /* else make it visited now */
-	    VERBOSE_DEBUG("[pid=%T] mangling B %p\n", myself->common.id, ptr);
+	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", myself->common.id, ptr));
 	    *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED;
 	    /* and count it */
 	    ASSERT(is_header(hdr));
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG: {
 		int arity = header_arity(hdr);
-		VERBOSE_DEBUG("/T");
 		sum += arity + 1;
 		if (arity == 0) { /* Empty tuple -- unusual. */
-		    VERBOSE_DEBUG("e");
 		    goto pop_next;
 		}
 		while (arity-- > 0) {
@@ -1168,7 +1122,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 		ErlFunThing* funp = (ErlFunThing *) ptr;
 		unsigned eterms = 1 /* creator */ + funp->num_free;
 		sz = thing_arityval(hdr);
-		VERBOSE_DEBUG("/F");
 		sum += 1 /* header */ + sz + eterms;
 		ptr += 1 /* header */ + sz;
 		while (eterms-- > 0) {
@@ -1213,17 +1166,14 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 		erl_exit(ERTS_ABORT_EXIT,
 			 "size_shared: matchstate term not allowed");
 	    default:
-		VERBOSE_DEBUG("/D");
 		sum += thing_arityval(hdr) + 1;
 		goto pop_next;
 	    }
 	    break;
 	}
 	case TAG_PRIMARY_IMMED1:
-	    VERBOSE_DEBUG("I");
 	pop_next:
 	    if (EQUEUE_ISEMPTY(s)) {
-		VERBOSE_DEBUG("\n");
 		// add sentinel to the table
 		SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL);
 		// store persistent info
@@ -1236,7 +1186,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 		info->shtable_start = t.start;
                 info->shtable_alloc_type = t.alloc_type;
 		// single point of return: the size of the object
-		VERBOSE_DEBUG("[pid=%T] size was: %u\n", myself->common.id, sum);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum));
 		return sum;
 	    }
 	    obj = EQUEUE_GET(s);
@@ -1244,7 +1194,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 	default:
 	    erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj);
 	}
-	VERBOSE_DEBUG("\n");
     }
 }
 
@@ -1266,7 +1215,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
     unsigned remaining;
     Process* myself;
     int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF;
-#if defined(DEBUG) || defined(SHCOPY_DEBUG)
+#ifdef DEBUG
     Eterm saved_obj = obj;
 #endif
 
@@ -1292,7 +1241,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
     if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
 	return copy_struct(obj, size, hpp, off_heap);
 
-    VERBOSE_DEBUG("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj);
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj));
 
     /* step #2: was performed before this function was called
        -------------------------------------------------------
@@ -1315,15 +1264,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
     resp = &result;
     remaining = 0;
     for (;;) {
-	VERBOSE_DEBUG("[copy] revisiting: %x ", obj);
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    VERBOSE_DEBUG("L");
 	    ptr = list_val_rel(obj, base);
 	    /* off heap list pointers are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
-		VERBOSE_DEBUG("#");
 		*resp = obj;
 		goto cleanup_next;
 	    }
@@ -1334,19 +1280,17 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 		e = head >> _TAG_PRIMARY_SIZE;
 		/* if it has been processed, just use the forwarding pointer */
 		if (primary_tag(head) == LIST_SHARED_PROCESSED) {
-		    VERBOSE_DEBUG("!");
 		    *resp = make_list(SHTABLE_FWD(t, e));
 		    goto cleanup_next;
 		}
 		/* else, let's process it now,
 		   copy it and keep the forwarding pointer */
 		else {
-		    VERBOSE_DEBUG("$");
 		    CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED;
 		    head = SHTABLE_X(t, e);
 		    tail = SHTABLE_Y(t, e);
 		    ptr = &(SHTABLE_X(t, e));
-		    VERBOSE_DEBUG("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e));
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)));
 		    SHTABLE_FWD_UPD(t, e, hp);
 		}
 	    }
@@ -1354,18 +1298,15 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 	    if (primary_tag(tail) == TAG_PRIMARY_HEADER) {
 		if (primary_tag(head) == TAG_PRIMARY_HEADER) {
 		    Eterm saved = BITSTORE_GET(b);
-		    VERBOSE_DEBUG("/B restoring %d", saved);
-		    VERBOSE_DEBUG("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr);
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr));
 		    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved;
 		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED;
 		} else {
-		    VERBOSE_DEBUG("/L");
-		    VERBOSE_DEBUG("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr);
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr));
 		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST;
 		}
 	    } else if (primary_tag(head) == TAG_PRIMARY_HEADER) {
-		VERBOSE_DEBUG("/I");
-		VERBOSE_DEBUG("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr));
 		CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail);
 		CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1;
 	    } else {
@@ -1387,11 +1328,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    VERBOSE_DEBUG("B");
 	    ptr = boxed_val_rel(obj, base);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
-		VERBOSE_DEBUG("#");
 		*resp = obj;
 		goto cleanup_next;
 	    }
@@ -1403,7 +1342,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 	    /* if it is shared and has been processed,
 	       just use the forwarding pointer */
 	    case BOXED_SHARED_PROCESSED:
-		VERBOSE_DEBUG("!");
 		e = hdr >> _TAG_PRIMARY_SIZE;
 		*resp = make_boxed(SHTABLE_FWD(t, e));
 		goto cleanup_next;
@@ -1411,17 +1349,16 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 	       it now: copy it and keep the forwarding pointer */
 	    case BOXED_SHARED_UNPROCESSED:
 		e = hdr >> _TAG_PRIMARY_SIZE;
-		VERBOSE_DEBUG("$");
 		*ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED;
 		hdr = SHTABLE_X(t, e);
 		ASSERT(primary_tag(hdr) == BOXED_VISITED);
-		VERBOSE_DEBUG("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e));
-		VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr));
 		SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER;
 		SHTABLE_FWD_UPD(t, e, hp);
 		break;
 	    case BOXED_VISITED:
-		VERBOSE_DEBUG("[pid=%T] unmangling B %p\n", myself->common.id, ptr);
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr));
 		*ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER;
 		break;
 	    }
@@ -1628,7 +1565,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 	default:
 	    erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj);
 	}
-	VERBOSE_DEBUG("\n");
     }
 
     /* step #4:
@@ -1637,21 +1573,20 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
     */
 
 all_clean:
-    VERBOSE_DEBUG("\n");
     for (e = 0; ; e += SHTABLE_INCR) {
 	ptr = SHTABLE_REV(t, e);
 	if (ptr == NULL)
 	    break;
-	VERBOSE_DEBUG("[copy] restoring shared: %x\n", ptr);
+	VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr));
 	/* entry was a list */
 	if (SHTABLE_Y(t, e) != THE_NON_VALUE) {
-	    VERBOSE_DEBUG("[pid=%T] untabling L %p\n", myself->common.id, ptr);
+	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", myself->common.id, ptr));
 	    CAR(ptr) = SHTABLE_X(t, e);
 	    CDR(ptr) = SHTABLE_Y(t, e);
 	}
 	/* entry was boxed */
 	else {
-	    VERBOSE_DEBUG("[pid=%T] untabling B %p\n", myself->common.id, ptr);
+	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", myself->common.id, ptr));
 	    *ptr = SHTABLE_X(t, e);
 	    ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER);
 	}
@@ -1665,9 +1600,9 @@ all_clean:
     }
 #endif
 
-    VERBOSE_DEBUG("[pid=%T] original was %T\n", myself->common.id, saved_obj);
-    VERBOSE_DEBUG("[pid=%T] copy is %T\n", myself->common.id, result);
-    VERBOSE_DEBUG("[pid=%T] result is at %p\n", myself->common.id, result);
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", myself->common.id, saved_obj));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result));
 
     ASSERT(hp == *hpp + size);
     *hpp = hp;
diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h
index f4259e7dae..029320691d 100644
--- a/erts/emulator/beam/erl_debug.h
+++ b/erts/emulator/beam/erl_debug.h
@@ -48,6 +48,7 @@
 #define DEBUG_THREADS      0x0010    /* Thread-related stuff                 */
 #define DEBUG_PROCESSES    0x0020    /* Process creation and removal         */
 #define DEBUG_MEMORY       0x0040    /* Display results of memory checks     */
+#define DEBUG_SHCOPY       0x0080    /* Sharing-preserving copying of terms  */
 
 extern Uint32 verbose;
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 1cf6509012..c50756d56b 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1166,9 +1166,8 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
 
-#ifdef SHCOPY_DEBUG
-    VERBOSE_DEBUG("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p));
-#endif
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id,
+                           HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
 
     n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 					       sizeof(Eterm)*new_sz);
@@ -1387,9 +1386,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     Uint new_sz, stk_sz;
     int adjusted;
 
-#ifdef SHCOPY_DEBUG
-    VERBOSE_DEBUG("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p));
-#endif
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id,
+                           HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f396a0a156..296cfdabc3 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1401,6 +1401,7 @@ erl_start(int argc, char **argv)
 		    case 't': verbose |= DEBUG_THREADS; break;
 		    case 'p': verbose |= DEBUG_PROCESSES; break;
 		    case 'm': verbose |= DEBUG_MESSAGES; break;
+		    case 'c': verbose |= DEBUG_SHCOPY; break;
 		    default : erts_fprintf(stderr,"Unknown verbose option: %c\n",*ch);
 		    }
 		}
@@ -1413,6 +1414,7 @@ erl_start(int argc, char **argv)
 	    if (verbose & DEBUG_THREADS) erts_printf("THREADS ");
 	    if (verbose & DEBUG_PROCESSES) erts_printf("PROCESSES ");
 	    if (verbose & DEBUG_MESSAGES) erts_printf("MESSAGES ");
+	    if (verbose & DEBUG_SHCOPY) erts_printf("SHCOPY ");
             erts_printf("\n");
 #else
 	    erts_fprintf(stderr, "warning: -v (only in debug compiled code)\n");
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index b1fca1df0c..11890a756d 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -68,9 +68,7 @@ new_message_buffer(Uint size)
     bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG,
 					    ERTS_HEAP_FRAG_SIZE(size));
     ERTS_INIT_HEAP_FRAG(bp, size, size);
-#ifdef SHCOPY_DEBUG
-    VERBOSE_DEBUG("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem);
-#endif
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem));
     return bp;
 }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f2da5289d3..96d17306a5 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11274,8 +11274,8 @@ static void
 delete_process(Process* p)
 {
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
-    VERBOSE_DEBUG("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
-                  HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
+                           HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
 
     /* Cleanup psd */
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 3e1f3664bc..303b9ee51b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1046,16 +1046,6 @@ void erl_error(char*, va_list);
 #define SHCOPY
 /* Use this if you want sharing-preserving copy to be initially disabled */
 #undef SHCOPY_DISABLE
-/* Use this with care, it is *very* verbose! */
-#undef SHCOPY_DEBUG
-#endif
-
-#ifdef SHCOPY_DEBUG
-#define VERBOSE_DEBUG(...) do {		\
-    erts_fprintf(stderr, __VA_ARGS__);	\
-  } while(0)
-#else
-#define VERBOSE_DEBUG(...)
 #endif
 
 #define ERTS_SHCOPY_FLG_MASK	(((unsigned) 3) << 0)
-- 
cgit v1.2.3


From eebdde01b149ea45966c7412bc2a062136457b54 Mon Sep 17 00:00:00 2001
From: Yiannis Tsiouris <gtsiour@softlab.ntua.gr>
Date: Tue, 22 Jan 2013 18:16:33 +0200
Subject: Add --enable-sharing-preserving configure flag

---
 erts/emulator/beam/erl_bif_info.c | 3 +++
 erts/emulator/beam/global.h       | 5 +----
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 1eb106a551..82c2aa4b9e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -129,6 +129,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #endif
 #ifdef USE_SYSTEMTAP
 				     " [systemtap]"
+#endif
+#ifdef SHCOPY
+				     " [sharing-preserving]"
 #endif
 				     "\n");
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 303b9ee51b..3c59df5f41 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1038,12 +1038,9 @@ void erl_error(char*, va_list);
 
 /* This controls whether sharing-preserving copy is used by Erlang */
 
+#ifdef SHCOPY
 #define SHCOPY_SEND
 #define SHCOPY_SPAWN
-
-#if defined(SHCOPY_SEND) \
- || defined(SHCOPY_SPAWN)
-#define SHCOPY
 /* Use this if you want sharing-preserving copy to be initially disabled */
 #undef SHCOPY_DISABLE
 #endif
-- 
cgit v1.2.3


From bcdf32795a3f5a6aad95567cffba78fcecb9f40f Mon Sep 17 00:00:00 2001
From: "Nikolaos S. Papaspyrou" <nickie@softlab.ntua.gr>
Date: Thu, 24 Apr 2014 12:59:39 +0300
Subject: Add support for maps in preserved copy

---
 erts/emulator/beam/copy.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index f74c6b1c89..cb1f38429a 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -424,6 +424,19 @@ Uint size_shared(Eterm obj)
 		}
 		goto pop_next;
 	    }
+            case MAP_SUBTAG: {
+                map_t *mp = (map_t *) ptr;
+                Uint n = map_get_size(mp) + 1;
+                sum += n + 2;
+                ptr += 2; /* hdr + size words */
+                while (n-- > 0) {
+                    obj = *ptr++;
+                    if (!IS_CONST(obj)) {
+                        EQUEUE_PUT(s, obj);
+                    }
+                }
+                goto pop_next;
+            }
 	    case BIN_MATCHSTATE_SUBTAG:
 		erl_exit(ERTS_ABORT_EXIT,
 			 "size_shared: matchstate term not allowed");
@@ -523,6 +536,18 @@ cleanup:
 		}
 		goto cleanup_next;
 	    }
+            case MAP_SUBTAG: {
+                map_t *mp = (map_t *) ptr;
+                Uint n = map_get_size(mp) + 1;
+                ptr += 2; /* hdr + size words */
+                while (n-- > 0) {
+                    obj = *ptr++;
+                    if (!IS_CONST(obj)) {
+                        EQUEUE_PUT_UNCHECKED(s, obj);
+                    }
+                }
+                goto cleanup_next;
+            }
 	    default:
 		goto cleanup_next;
 	    }
@@ -1162,6 +1187,19 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 		}
 		goto pop_next;
 	    }
+            case MAP_SUBTAG: {
+                map_t *mp = (map_t *) ptr;
+                Uint n = map_get_size(mp) + 1;
+                sum += n + 2;
+                ptr += 2; /* hdr + size words */
+                while (n-- > 0) {
+                    obj = *ptr++;
+                    if (!IS_CONST(obj)) {
+                        EQUEUE_PUT(s, obj);
+                    }
+                }
+                goto pop_next;
+            }
 	    case BIN_MATCHSTATE_SUBTAG:
 		erl_exit(ERTS_ABORT_EXIT,
 			 "size_shared: matchstate term not allowed");
@@ -1404,6 +1442,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 		erts_refc_inc(&funp->fe->refc, 2);
 		goto cleanup_next;
 	    }
+	    case MAP_SUBTAG: {
+		map_t *mp = (map_t *) ptr;
+		Uint n = map_get_size(mp) + 1;
+		*resp = make_map(hp);
+		*hp++ = hdr;
+                *hp++ = *++ptr;
+		while (n-- > 0) {
+		    obj = *++ptr;
+		    if (IS_CONST(obj)) {
+			*hp++ = obj;
+		    } else {
+			EQUEUE_PUT_UNCHECKED(s, obj);
+			*hp++ = HEAP_ELEM_TO_BE_FILLED;
+		    }
+		}
+		goto cleanup_next;
+	    }
 	    case REFC_BINARY_SUBTAG: {
 		ProcBin* pb = (ProcBin *) ptr;
 		sz = thing_arityval(hdr);
@@ -1539,6 +1594,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 			    remaining = 1 + funp->num_free;
 			    break;
 			}
+			case MAP_SUBTAG: {
+			    map_t *mp = (map_t *) hscan;
+			    remaining = map_get_size(mp) + 1;
+                            hscan += 2;
+			    break;
+			}
 			case SUB_BINARY_SUBTAG:
 			    ASSERT(((ErlSubBin *) hscan)->bitoffs +
 				   ((ErlSubBin *) hscan)->bitsize > 0);
-- 
cgit v1.2.3


From 99de9eb01d8884ac293fdda20ab65c045d215d46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 15 Sep 2015 17:47:36 +0200
Subject: Fix internal stacks

The internal stacks has changed between releases.
---
 erts/emulator/beam/copy.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index cb1f38429a..added85b46 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -907,7 +907,7 @@ int disable_copy_shared = ERTS_SHCOPY_FLG_NONE;
 #define SHTABLE_PUSH(s,x,y,b)					\
 do {								\
     if (s.sp > s.end - SHTABLE_INCR) {				\
-        erl_grow_estack(&s, ESTK_DEF_STACK(s));			\
+        erl_grow_estack(&(s), SHTABLE_INCR);	       		\
     }								\
     *s.sp++ = (x);						\
     *s.sp++ = (y);						\
@@ -959,6 +959,7 @@ do {								\
         WSTK_DEF_STACK(s),  /* wstart */		\
         WSTK_DEF_STACK(s),  /* wsp */			\
         WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */	\
+        WSTK_DEF_STACK(s),  /* wdflt */ 		\
         ERTS_ALC_T_ESTACK /* alloc_type */		\
     };							\
     int WSTK_CONCAT(s,_bitoffs) = 0;			\
@@ -969,8 +970,9 @@ do {								\
     /* no WSTK_DEF_STACK(s), read-only */		\
     ErtsWStack s = {					\
         info->bitstore_start,  /* wstart */		\
-        NULL,                  /* wsp, read-only */	\
+        NULL,                  /* wsp,  read-only */	\
         NULL,                  /* wend, read-only */	\
+        NULL,                  /* wdef, read-only */	\
         info->bitstore_alloc_type /* alloc_type */	\
     };							\
     int WSTK_CONCAT(s,_bitoffs) = 0;			\
@@ -983,6 +985,7 @@ do {								\
         ESTK_DEF_STACK(s),  /* start */			\
         ESTK_DEF_STACK(s),  /* sp */			\
         ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */	\
+        ESTK_DEF_STACK(s),  /* default */		\
         ERTS_ALC_T_ESTACK /* alloc_type */		\
     };							\
     Uint ESTK_CONCAT(s,_offset) = 0
@@ -991,8 +994,9 @@ do {								\
     /* no ESTK_DEF_STACK(s), read-only */		\
     ErtsEStack s = {					\
         info->shtable_start,     /* start */		\
-        NULL,                    /* sp, read-only */	\
+        NULL,                    /* sp,  read-only */	\
         NULL,                    /* end, read-only */	\
+        NULL,                    /* def, read-only */	\
         info->shtable_alloc_type /* alloc_type */	\
     };							\
     /* no ESTK_CONCAT(s,_offset), read-only */
-- 
cgit v1.2.3


From 2ab76f7f1fe6ec4c94541ba5a2f0cd9dc1b9c79a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 15 Sep 2015 16:49:28 +0200
Subject: Fix Halfword removal

Halfword is no longer present in the runtime system.
---
 erts/emulator/beam/copy.c | 40 +++++++++++++++++-----------------------
 1 file changed, 17 insertions(+), 23 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index added85b46..42b4bee29d 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -298,11 +298,7 @@ do {									\
  *  It is argued whether the size of subterms in constant pools
  *  should be counted or not.
  */
-#if HALFWORD_HEAP
-Uint size_shared_rel(Eterm obj, Eterm* base)
-#else
 Uint size_shared(Eterm obj)
-#endif
 {
     Eterm saved_obj = obj;
     Uint sum = 0;
@@ -320,7 +316,7 @@ Uint size_shared(Eterm obj)
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    ptr = list_val_rel(obj, base);
+	    ptr = list_val(obj);
 	    /* we're not counting anything that's outside our heap */
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
 		goto pop_next;
@@ -357,7 +353,7 @@ Uint size_shared(Eterm obj)
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    ptr = boxed_val_rel(obj, base);
+	    ptr = boxed_val(obj);
 	    /* we're not counting anything that's outside our heap */
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
 		goto pop_next;
@@ -414,13 +410,13 @@ Uint size_shared(Eterm obj)
 		} else {
 		    extra_bytes = 0;
 		}
-		ptr = binary_val_rel(sb->orig, base);
+		ptr = binary_val(sb->orig);
 		hdr = (*ptr) & ~BOXED_VISITED_MASK;
 		if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) {
 		    sum += PROC_BIN_SIZE;
 		} else {
 		    ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG);
-		    sum += heap_bin_size(binary_size_rel(obj, base) + extra_bytes);
+		    sum += heap_bin_size(binary_size(obj) + extra_bytes);
 		}
 		goto pop_next;
 	    }
@@ -465,7 +461,7 @@ cleanup:
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    ptr = list_val_rel(obj, base);
+	    ptr = list_val(obj);
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
 		goto cleanup_next;
 	    }
@@ -495,7 +491,7 @@ cleanup:
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    ptr = boxed_val_rel(obj, base);
+	    ptr = boxed_val(obj);
 	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
 		goto cleanup_next;
 	    }
@@ -912,7 +908,7 @@ do {								\
     *s.sp++ = (x);						\
     *s.sp++ = (y);						\
     *s.sp++ = (Eterm) NULL;					\
-    *s.sp++ = (Eterm) (b);	/* bad in HALF_WORD */		\
+    *s.sp++ = (Eterm) (b);		                	\
     ESTK_CONCAT(s,_offset) += SHTABLE_INCR;			\
 } while(0)
 #define SHTABLE_X(s,e) (s.start[e])
@@ -1004,7 +1000,6 @@ do {								\
 /*
  *  Copy object "obj" preserving sharing.
  *  First half: count size and calculate sharing.
- *  NOTE: We do not support HALF_WORD (yet?).
  */
 Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 {
@@ -1060,7 +1055,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    ptr = list_val_rel(obj, base);
+	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
@@ -1109,7 +1104,7 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    ptr = boxed_val_rel(obj, base);
+	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
@@ -1178,11 +1173,11 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 		} else {
 		    extra_bytes = 0;
 		}
-		ASSERT(is_boxed(rterm2wterm(real_bin, base)) &&
-		       (((*boxed_val(rterm2wterm(real_bin, base))) &
+		ASSERT(is_boxed(real_bin) &&
+		       (((*boxed_val(real_bin)) &
 			 (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK))
 			== _TAG_HEADER_REFC_BIN));
-		hdr = *_unchecked_binary_val(rterm2wterm(real_bin, base)) & ~BOXED_VISITED_MASK;
+		hdr = *_unchecked_binary_val(real_bin) & ~BOXED_VISITED_MASK;
 		if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) {
 		    sum += heap_bin_size(size+extra_bytes);
 		} else {
@@ -1243,7 +1238,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 /*
  *  Copy object "obj" preserving sharing.
  *  Second half: copy and restore the object.
- *  NOTE: We do not support HALF_WORD (yet?).
  */
 Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags)
 {
@@ -1309,7 +1303,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
-	    ptr = list_val_rel(obj, base);
+	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
 		*resp = obj;
@@ -1370,7 +1364,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 	}
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
-	    ptr = boxed_val_rel(obj, base);
+	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
 		*resp = obj;
@@ -1500,11 +1494,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 		    extra_bytes = 0;
 		}
 		real_size = size+extra_bytes;
-		ASSERT(is_boxed(rterm2wterm(real_bin, base)) &&
-		       (((*boxed_val(rterm2wterm(real_bin, base))) &
+		ASSERT(is_boxed(real_bin) &&
+		       (((*boxed_val(real_bin)) &
 			 (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK))
 			== _TAG_HEADER_REFC_BIN));
-		ptr = _unchecked_binary_val(rterm2wterm(real_bin, base));
+		ptr = _unchecked_binary_val(real_bin);
 		*resp = make_binary(hp);
 		if (extra_bytes != 0) {
 		    ErlSubBin* res = (ErlSubBin *) hp;
-- 
cgit v1.2.3


From 95b25f7a3a33c43912079af713bf1f386a764071 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 15 Sep 2015 17:36:22 +0200
Subject: Fix Map preserved sharing copy implementation

The Map implementation has changed since initial preserved copy implementation.
---
 erts/emulator/beam/copy.c | 135 +++++++++++++++++++++++++++++-----------------
 1 file changed, 85 insertions(+), 50 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 42b4bee29d..c7806ed8ff 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -420,19 +420,28 @@ Uint size_shared(Eterm obj)
 		}
 		goto pop_next;
 	    }
-            case MAP_SUBTAG: {
-                map_t *mp = (map_t *) ptr;
-                Uint n = map_get_size(mp) + 1;
-                sum += n + 2;
-                ptr += 2; /* hdr + size words */
-                while (n-- > 0) {
-                    obj = *ptr++;
-                    if (!IS_CONST(obj)) {
-                        EQUEUE_PUT(s, obj);
+            case MAP_SUBTAG:
+                switch (MAP_HEADER_TYPE(hdr)) {
+                    case MAP_HEADER_TAG_FLATMAP_HEAD : {
+                        flatmap_t *mp  = (flatmap_t*)flatmap_val(obj);
+                        Uint n = flatmap_get_size(mp) + 1;
+                        ptr  = (Eterm *)mp;
+                        sum += n + 2;
+                        ptr += 2; /* hdr + size words */
+                        while (n--) {
+                            obj = *ptr++;
+                            if (!IS_CONST(obj)) {
+                                EQUEUE_PUT(s, obj);
+                            }
+                        }
+                        goto pop_next;
                     }
+                    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+                    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                    default:
+                        erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
                 }
-                goto pop_next;
-            }
 	    case BIN_MATCHSTATE_SUBTAG:
 		erl_exit(ERTS_ABORT_EXIT,
 			 "size_shared: matchstate term not allowed");
@@ -533,10 +542,10 @@ cleanup:
 		goto cleanup_next;
 	    }
             case MAP_SUBTAG: {
-                map_t *mp = (map_t *) ptr;
-                Uint n = map_get_size(mp) + 1;
+                flatmap_t *mp = (flatmap_t *) ptr;
+                Uint n = flatmap_get_size(mp) + 1;
                 ptr += 2; /* hdr + size words */
-                while (n-- > 0) {
+                while (n--) {
                     obj = *ptr++;
                     if (!IS_CONST(obj)) {
                         EQUEUE_PUT_UNCHECKED(s, obj);
@@ -1186,20 +1195,28 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 		}
 		goto pop_next;
 	    }
-            case MAP_SUBTAG: {
-                map_t *mp = (map_t *) ptr;
-                Uint n = map_get_size(mp) + 1;
-                sum += n + 2;
-                ptr += 2; /* hdr + size words */
-                while (n-- > 0) {
-                    obj = *ptr++;
-                    if (!IS_CONST(obj)) {
-                        EQUEUE_PUT(s, obj);
+            case MAP_SUBTAG:
+                switch (MAP_HEADER_TYPE(hdr)) {
+                    case MAP_HEADER_TAG_FLATMAP_HEAD : {
+                        flatmap_t *mp = (flatmap_t *) ptr;
+                        Uint n = flatmap_get_size(mp) + 1;
+                        sum += n + 2;
+                        ptr += 2; /* hdr + size words */
+                        while (n--) {
+                            obj = *ptr++;
+                            if (!IS_CONST(obj)) {
+                                EQUEUE_PUT(s, obj);
+                            }
+                        }
+                        goto pop_next;
                     }
+                    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+                    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                    default:
+                        erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
                 }
-                goto pop_next;
-            }
-	    case BIN_MATCHSTATE_SUBTAG:
+            case BIN_MATCHSTATE_SUBTAG:
 		erl_exit(ERTS_ABORT_EXIT,
 			 "size_shared: matchstate term not allowed");
 	    default:
@@ -1234,7 +1251,6 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
     }
 }
 
-
 /*
  *  Copy object "obj" preserving sharing.
  *  Second half: copy and restore the object.
@@ -1440,23 +1456,31 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 		erts_refc_inc(&funp->fe->refc, 2);
 		goto cleanup_next;
 	    }
-	    case MAP_SUBTAG: {
-		map_t *mp = (map_t *) ptr;
-		Uint n = map_get_size(mp) + 1;
-		*resp = make_map(hp);
-		*hp++ = hdr;
-                *hp++ = *++ptr;
-		while (n-- > 0) {
-		    obj = *++ptr;
-		    if (IS_CONST(obj)) {
-			*hp++ = obj;
-		    } else {
-			EQUEUE_PUT_UNCHECKED(s, obj);
-			*hp++ = HEAP_ELEM_TO_BE_FILLED;
-		    }
-		}
-		goto cleanup_next;
-	    }
+	    case MAP_SUBTAG:
+                switch (MAP_HEADER_TYPE(hdr)) {
+                    case MAP_HEADER_TAG_FLATMAP_HEAD : {
+                        flatmap_t *mp = (flatmap_t *) ptr;
+                        Uint n = flatmap_get_size(mp) + 1;
+                        *resp  = make_flatmap(hp);
+                        *hp++  = hdr;
+                        *hp++  = *++ptr;
+                        while (n--) {
+                            obj = *++ptr;
+                            if (IS_CONST(obj)) {
+                                *hp++ = obj;
+                            } else {
+                                EQUEUE_PUT_UNCHECKED(s, obj);
+                                *hp++ = HEAP_ELEM_TO_BE_FILLED;
+                            }
+                        }
+                        goto cleanup_next;
+                    }
+                    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+                    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                    default:
+                        erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+                }
 	    case REFC_BINARY_SUBTAG: {
 		ProcBin* pb = (ProcBin *) ptr;
 		sz = thing_arityval(hdr);
@@ -1592,12 +1616,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 			    remaining = 1 + funp->num_free;
 			    break;
 			}
-			case MAP_SUBTAG: {
-			    map_t *mp = (map_t *) hscan;
-			    remaining = map_get_size(mp) + 1;
-                            hscan += 2;
-			    break;
-			}
+			case MAP_SUBTAG:
+                            switch (MAP_HEADER_TYPE(*hscan)) {
+                                case MAP_HEADER_TAG_FLATMAP_HEAD : {
+                                    flatmap_t *mp = (flatmap_t *) hscan;
+                                    remaining = flatmap_get_size(mp) + 1;
+                                    hscan += 2;
+                                    break;
+                                }
+                                case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+                                case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+                                case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                                default:
+                                    erl_exit(ERTS_ABORT_EXIT,
+                                            "copy_shared_perform: bad hashmap type %d\n",
+                                            MAP_HEADER_TYPE(*hscan));
+                            }
+                            break;
 			case SUB_BINARY_SUBTAG:
 			    ASSERT(((ErlSubBin *) hscan)->bitoffs +
 				   ((ErlSubBin *) hscan)->bitsize > 0);
-- 
cgit v1.2.3


From 2e10fe29f61bbe3246902e8eaf1636dd6457979f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 16 Sep 2015 16:21:43 +0200
Subject: Add support for HAMT maps in preserved copy

---
 erts/emulator/beam/copy.c | 115 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 88 insertions(+), 27 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index c7806ed8ff..aa17713d07 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -438,7 +438,18 @@ Uint size_shared(Eterm obj)
                     }
                     case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
                     case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
-                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP : {
+                        Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                        sum += 1 + n + header_arity(hdr);
+                        ptr += 1 + header_arity(hdr);
+                        while (n--) {
+                            obj = *ptr++;
+                            if (!IS_CONST(obj)) {
+                                EQUEUE_PUT(s, obj);
+                            }
+                        }
+                        goto pop_next;
+                    }
                     default:
                         erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
                 }
@@ -541,18 +552,37 @@ cleanup:
 		}
 		goto cleanup_next;
 	    }
-            case MAP_SUBTAG: {
-                flatmap_t *mp = (flatmap_t *) ptr;
-                Uint n = flatmap_get_size(mp) + 1;
-                ptr += 2; /* hdr + size words */
-                while (n--) {
-                    obj = *ptr++;
-                    if (!IS_CONST(obj)) {
-                        EQUEUE_PUT_UNCHECKED(s, obj);
+            case MAP_SUBTAG:
+                switch (MAP_HEADER_TYPE(hdr)) {
+                    case MAP_HEADER_TAG_FLATMAP_HEAD : {
+                        flatmap_t *mp = (flatmap_t *) ptr;
+                        Uint n = flatmap_get_size(mp) + 1;
+                        ptr += 2; /* hdr + size words */
+                        while (n--) {
+                            obj = *ptr++;
+                            if (!IS_CONST(obj)) {
+                                EQUEUE_PUT_UNCHECKED(s, obj);
+                            }
+                        }
+                        goto cleanup_next;
+                    }
+                    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+                    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP : {
+                        Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                        sum += 1 + n + header_arity(hdr);
+                        ptr += 1 + header_arity(hdr);
+                        while (n--) {
+                            obj = *ptr++;
+                            if (!IS_CONST(obj)) {
+                                EQUEUE_PUT_UNCHECKED(s, obj);
+                            }
+                        }
+                        goto cleanup_next;
                     }
+                    default:
+                        erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
                 }
-                goto cleanup_next;
-            }
 	    default:
 		goto cleanup_next;
 	    }
@@ -1212,7 +1242,22 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
                     }
                     case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
                     case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
-                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP : {
+                        Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                        sum += 1 + n + header_arity(hdr);
+                        ptr += 1 + header_arity(hdr);
+
+                        if (n == 0) {
+                            goto pop_next;
+                        }
+                        while(n--) {
+                            obj = *ptr++;
+                            if (!IS_CONST(obj)) {
+                                EQUEUE_PUT(s, obj);
+                            }
+                        }
+                        goto pop_next;
+                    }
                     default:
                         erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
                 }
@@ -1228,20 +1273,20 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
 	case TAG_PRIMARY_IMMED1:
 	pop_next:
 	    if (EQUEUE_ISEMPTY(s)) {
-		// add sentinel to the table
-		SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL);
-		// store persistent info
-		BITSTORE_CLOSE(b);
-		info->queue_start = s.start;
-		info->queue_end = s.end;
+                /* add sentinel to the table */
+                SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL);
+                /* store persistent info */
+                BITSTORE_CLOSE(b);
+                info->queue_start = s.start;
+                info->queue_end = s.end;
                 info->queue_alloc_type = s.alloc_type;
-		info->bitstore_start = b.wstart;
+                info->bitstore_start = b.wstart;
                 info->bitstore_alloc_type = b.alloc_type;
-		info->shtable_start = t.start;
+                info->shtable_start = t.start;
                 info->shtable_alloc_type = t.alloc_type;
-		// single point of return: the size of the object
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum));
-		return sum;
+                /* single point of return: the size of the object */
+                VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum));
+                return sum;
 	    }
 	    obj = EQUEUE_GET(s);
 	    break;
@@ -1457,13 +1502,13 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
 		goto cleanup_next;
 	    }
 	    case MAP_SUBTAG:
+                *resp  = make_flatmap(hp);
+                *hp++  = hdr;
                 switch (MAP_HEADER_TYPE(hdr)) {
                     case MAP_HEADER_TAG_FLATMAP_HEAD : {
                         flatmap_t *mp = (flatmap_t *) ptr;
                         Uint n = flatmap_get_size(mp) + 1;
-                        *resp  = make_flatmap(hp);
-                        *hp++  = hdr;
-                        *hp++  = *++ptr;
+                        *hp++  = *++ptr; /* keys */
                         while (n--) {
                             obj = *++ptr;
                             if (IS_CONST(obj)) {
@@ -1477,7 +1522,20 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
                     }
                     case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
                     case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
-                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+			*hp++ = *++ptr; /* total map size */
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP : {
+                         Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                         while (n--)  {
+                             obj = *++ptr;
+                             if (IS_CONST(obj)) {
+                                 *hp++ = obj;
+                             } else {
+                                 EQUEUE_PUT_UNCHECKED(s, obj);
+                                 *hp++ = HEAP_ELEM_TO_BE_FILLED;
+                             }
+                         }
+                        goto cleanup_next;
+                    }
                     default:
                         erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
                 }
@@ -1627,6 +1685,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
                                 case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
                                 case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
                                 case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                                    remaining = hashmap_bitcount(MAP_HEADER_VAL(*hscan));
+                                    hscan += MAP_HEADER_ARITY(*hscan) + 1;
+                                    break;
                                 default:
                                     erl_exit(ERTS_ABORT_EXIT,
                                             "copy_shared_perform: bad hashmap type %d\n",
-- 
cgit v1.2.3


From 748c73f1687b2375d4c607487f40036ba990c4c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 17 Sep 2015 16:17:10 +0200
Subject: Refactor copy sharing

---
 erts/emulator/beam/beam_debug.c  |  6 +++---
 erts/emulator/beam/copy.c        | 10 ++++-----
 erts/emulator/beam/erl_db.c      |  4 ++--
 erts/emulator/beam/erl_message.h |  6 +++---
 erts/emulator/beam/erl_process.c |  6 +++---
 erts/emulator/beam/global.h      | 46 ++++++++++++++++++++--------------------
 6 files changed, 39 insertions(+), 39 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 36f3cfabbc..b007d000bf 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -96,18 +96,18 @@ erts_debug_copy_shared_1(BIF_ALIST_1)
     Uint size;
     Eterm* hp;
     Eterm copy;
-    shcopy_info info;
+    erts_shcopy_t info;
 #ifdef SHCOPY_DISABLE
     extern int disable_copy_shared;
 #endif
-    INITIALIZE_INFO(info);
+    INITIALIZE_SHCOPY(info);
 
     size = copy_shared_calculate(term, &info, 0);
     if (size > 0) {
       hp = HAlloc(p, size);
     }
     copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap, 0);
-    DESTROY_INFO(info);
+    DESTROY_SHCOPY(info);
 #ifdef SHCOPY_DISABLE
     disable_copy_shared = 0;
 #endif
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index aa17713d07..f2dd73d862 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1040,14 +1040,14 @@ do {								\
  *  Copy object "obj" preserving sharing.
  *  First half: count size and calculate sharing.
  */
-Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
+Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 {
     Uint sum;
     Uint e;
     unsigned sz;
     Eterm* ptr;
     Process* myself;
-    int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF;
+    int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF;
 
     DECLARE_EQUEUE_INIT_INFO(s, info);
     DECLARE_BITSTORE_INIT_INFO(b, info);
@@ -1300,8 +1300,8 @@ Uint copy_shared_calculate(Eterm obj, shcopy_info *info, unsigned flags)
  *  Copy object "obj" preserving sharing.
  *  Second half: copy and restore the object.
  */
-Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, ErlOffHeap* off_heap, unsigned flags)
-{
+Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
+                         Eterm** hpp, ErlOffHeap* off_heap, Uint32 flags) {
     Uint e;
     unsigned sz;
     Eterm* ptr;
@@ -1311,7 +1311,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, shcopy_info *info, Eterm** hpp, E
     Eterm* resp;
     unsigned remaining;
     Process* myself;
-    int force_local = flags & ERTS_SHCOPY_FLG_TMP_BUF;
+    int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF;
 #ifdef DEBUG
     Eterm saved_obj = obj;
 #endif
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 6119a9225f..59a0c0b808 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1832,7 +1832,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 			     tb->common.id,
 			     from_pid,
 			     BIF_ARG_3), 
-		      ERTS_SND_FLG_SHCOPY_TMP_BUF);
+		      ERTS_SND_FLG_SHCOPY_TMPBUF);
     erts_smp_proc_unlock(to_proc, to_locks);
     UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
@@ -3211,7 +3211,7 @@ retry:
 			     tb->common.id,
 			     p->common.id,
 			     heir_data), 
-		      ERTS_SND_FLG_SHCOPY_TMP_BUF);
+		      ERTS_SND_FLG_SHCOPY_TMPBUF);
     erts_smp_proc_unlock(to_proc, to_locks);
     return !0;
 }
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 0de36c3199..52a648fe0b 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -239,9 +239,9 @@ do {									\
 #define ERTS_SND_FLG_NO_SEQ_TRACE		(((unsigned) 1) << 0)
 
 #define ERTS_SND_FLG_SHCOPY_SHIFT	1
-#define ERTS_SND_FLG_SHCOPY_MASK	(ERTS_SHCOPY_FLG_MASK    << ERTS_SND_FLG_SHCOPY_SHIFT)
-#define ERTS_SND_FLG_SHCOPY_NONE	(ERTS_SHCOPY_FLG_NONE    << ERTS_SND_FLG_SHCOPY_SHIFT)
-#define ERTS_SND_FLG_SHCOPY_TMP_BUF	(ERTS_SHCOPY_FLG_TMP_BUF << ERTS_SND_FLG_SHCOPY_SHIFT)
+#define ERTS_SND_FLG_SHCOPY_MASK        (ERTS_SHCOPY_FLG_MASK   << ERTS_SND_FLG_SHCOPY_SHIFT)
+#define ERTS_SND_FLG_SHCOPY_NONE        (ERTS_SHCOPY_FLG_NONE   << ERTS_SND_FLG_SHCOPY_SHIFT)
+#define ERTS_SND_FLG_SHCOPY_TMPBUF      (ERTS_SHCOPY_FLG_TMPBUF << ERTS_SND_FLG_SHCOPY_SHIFT)
 
 #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
    (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 96d17306a5..a691a3c773 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10749,8 +10749,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL;
 #ifdef SHCOPY_SPAWN
     unsigned shflags = 0;	/* could be taken from so->flags, if necessary */
-    shcopy_info info;
-    INITIALIZE_INFO(info);
+    erts_shcopy_t info;
+    INITIALIZE_SHCOPY(info);
 #endif
 
 #ifdef ERTS_SMP
@@ -10887,7 +10887,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     BM_SWAP_TIMER(system,copy);
 #ifdef SHCOPY_SPAWN
     p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap, shflags);
-    DESTROY_INFO(info);
+    DESTROY_SHCOPY(info);
 #else
     p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap);
 #endif
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 3c59df5f41..bdee20969d 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1045,13 +1045,13 @@ void erl_error(char*, va_list);
 #undef SHCOPY_DISABLE
 #endif
 
-#define ERTS_SHCOPY_FLG_MASK	(((unsigned) 3) << 0)
-#define ERTS_SHCOPY_FLG_NONE	(((unsigned) 1) << 0)
-#define ERTS_SHCOPY_FLG_TMP_BUF	(((unsigned) 1) << 1)
+#define ERTS_SHCOPY_FLG_MASK    (((Uint32) 3) << 0)
+#define ERTS_SHCOPY_FLG_NONE    (((Uint32) 1) << 0)
+#define ERTS_SHCOPY_FLG_TMPBUF  (((Uint32) 1) << 1) /* forces INHEAP to true */
 
 /* The persistent state while the sharing-preserving copier works */
 
-typedef struct shcopy_info {
+typedef struct {
     Eterm  queue_default[DEF_EQUEUE_SIZE];
     Eterm* queue_start;
     Eterm* queue_end;
@@ -1062,26 +1062,26 @@ typedef struct shcopy_info {
     Eterm  shtable_default[DEF_ESTACK_SIZE];
     Eterm* shtable_start;
     ErtsAlcType_t shtable_alloc_type;
-} shcopy_info;
+} erts_shcopy_t;
 
-#define INITIALIZE_INFO(info)						\
-do {									\
-    info.queue_start = info.queue_default;				\
-    info.bitstore_start = info.bitstore_default;			\
-    info.shtable_start = info.shtable_default;				\
+#define INITIALIZE_SHCOPY(info)                         \
+do {                                                    \
+    info.queue_start = info.queue_default;              \
+    info.bitstore_start = info.bitstore_default;        \
+    info.shtable_start = info.shtable_default;          \
 } while(0)
 
-#define DESTROY_INFO(info)						\
-do {									\
-    if (info.queue_start != info.queue_default) {			\
-	erts_free(info.queue_alloc_type, info.queue_start);		\
-    }									\
-    if (info.bitstore_start != info.bitstore_default) {			\
-	erts_free(info.bitstore_alloc_type, info.bitstore_start);	\
-    }									\
-    if (info.shtable_start != info.shtable_default) {			\
-	erts_free(info.shtable_alloc_type, info.shtable_start);		\
-    }									\
+#define DESTROY_SHCOPY(info)                                            \
+do {                                                                    \
+    if (info.queue_start != info.queue_default) {                       \
+        erts_free(info.queue_alloc_type, info.queue_start);             \
+    }                                                                   \
+    if (info.bitstore_start != info.bitstore_default) {                 \
+        erts_free(info.bitstore_alloc_type, info.bitstore_start);       \
+    }                                                                   \
+    if (info.shtable_start != info.shtable_default) {                   \
+        erts_free(info.shtable_alloc_type, info.shtable_start);         \
+    }                                                                   \
 } while(0)
 
 /* copy.c */
@@ -1089,8 +1089,8 @@ Eterm copy_object_x(Eterm, Process*, Uint);
 #define copy_object(Term, Proc) copy_object_x(Term,Proc,0)
 
 Uint size_object(Eterm);
-Uint copy_shared_calculate(Eterm, shcopy_info*, unsigned);
-Eterm copy_shared_perform(Eterm, Uint, shcopy_info*, Eterm**, ErlOffHeap*, unsigned);
+Uint copy_shared_calculate(Eterm, erts_shcopy_t*, Uint32);
+Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, Uint32);
 
 Uint size_shared(Eterm);
 
-- 
cgit v1.2.3


From 5d2d888f368ba8ca0098d8bd9936ca9d67df7d3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 17 Sep 2015 19:30:08 +0200
Subject: Copy literals in copy sharing

---
 erts/emulator/beam/copy.c   | 49 +++++++++++++++++++++++++++++----------------
 erts/emulator/beam/global.h |  6 +++++-
 2 files changed, 37 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index f2dd73d862..4f4d20cb83 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -611,7 +611,7 @@ cleanup:
 /*
  *  Copy a structure to a heap.
  */
-Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz)
 {
     char* hstart;
     Uint hsize;
@@ -626,6 +626,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     Eterm* argp;
     Eterm* const_tuple;
     Eterm hdr;
+    Eterm *hend;
     int i;
 #ifdef DEBUG
     Eterm org_obj = obj;
@@ -641,7 +642,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     DTRACE1(copy_struct, (int32_t)sz);
 
     hp = htop = *hpp;
-    hbot   = htop + sz;
+    hbot = hend = htop + sz;
     hstart = (char *)htop;
     hsize = (char*) hbot - hstart;
     const_tuple = 0;
@@ -906,19 +907,24 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	}
     }
 
+    if (bsz) {
+        *hpp = htop;
+        *bsz = hend - hbot;
+    } else {
 #ifdef DEBUG
-    if (htop != hbot)
-	erl_exit(ERTS_ABORT_EXIT,
-		 "Internal error in copy_struct() when copying %T:"
-		 " htop=%p != hbot=%p (sz=%beu)\n",
-		 org_obj, htop, hbot, org_sz); 
+        if (htop != hbot)
+            erl_exit(ERTS_ABORT_EXIT,
+                    "Internal error in copy_struct() when copying %T:"
+                    " htop=%p != hbot=%p (sz=%beu)\n",
+                    org_obj, htop, hbot, org_sz);
 #else
-    if (htop > hbot) {
-	erl_exit(ERTS_ABORT_EXIT,
-		 "Internal error in copy_struct(): htop, hbot overrun\n");
-    }
+        if (htop > hbot) {
+            erl_exit(ERTS_ABORT_EXIT,
+                    "Internal error in copy_struct(): htop, hbot overrun\n");
+        }
 #endif
-    *hpp = (Eterm *) (hstart+hsize);
+        *hpp = (Eterm *) (hstart+hsize);
+    }
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res));
     return res;
 }
@@ -1098,6 +1104,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 	    /* off heap list pointers are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
+                info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
 	    head = CAR(ptr);
@@ -1147,6 +1154,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
+                info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
 	    hdr = *ptr;
@@ -1286,7 +1294,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
                 info->shtable_alloc_type = t.alloc_type;
                 /* single point of return: the size of the object */
                 VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum));
-                return sum;
+                return sum + info->literal_size;
 	    }
 	    obj = EQUEUE_GET(s);
 	    break;
@@ -1309,6 +1317,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     Eterm* hscan;
     Eterm result;
     Eterm* resp;
+    Eterm *hbot, *hend;
     unsigned remaining;
     Process* myself;
     int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF;
@@ -1346,6 +1355,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     */
 
     hscan = hp = *hpp;
+    hbot  = hend = hp + size;
 
     /* step #3:
        -------------------------------------------------------
@@ -1367,7 +1377,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
-		*resp = obj;
+                Uint bsz;
+                *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                hbot -= bsz;
 		goto cleanup_next;
 	    }
 	    head = CAR(ptr);
@@ -1428,7 +1440,9 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (!INHEAP(myself, ptr)) {
-		*resp = obj;
+                Uint bsz;
+                *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                hbot -= bsz;
 		goto cleanup_next;
 	    }
 	    hdr = *ptr;
@@ -1759,8 +1773,9 @@ all_clean:
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result));
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result));
 
-    ASSERT(hp == *hpp + size);
-    *hpp = hp;
+    ASSERT(hbot == hp);
+    ASSERT(size == ((hp - *hpp) + (hend - hbot)));
+    *hpp = hend;
     return result;
 }
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bdee20969d..f106b941ef 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1062,6 +1062,7 @@ typedef struct {
     Eterm  shtable_default[DEF_ESTACK_SIZE];
     Eterm* shtable_start;
     ErtsAlcType_t shtable_alloc_type;
+    Uint literal_size;
 } erts_shcopy_t;
 
 #define INITIALIZE_SHCOPY(info)                         \
@@ -1069,6 +1070,7 @@ do {                                                    \
     info.queue_start = info.queue_default;              \
     info.bitstore_start = info.bitstore_default;        \
     info.shtable_start = info.shtable_default;          \
+    info.literal_size = 0;                              \
 } while(0)
 
 #define DESTROY_SHCOPY(info)                                            \
@@ -1094,7 +1096,9 @@ Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, Uin
 
 Uint size_shared(Eterm);
 
-Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
+Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz);
+#define copy_struct(Obj,Sz,HPP,OH) \
+    copy_struct_x(Obj,Sz,HPP,OH,NULL)
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
 
 void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
-- 
cgit v1.2.3


From 8f8aa9c5c4e26e563c935e06f8346175fa15d876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 18 Sep 2015 19:20:19 +0200
Subject: Add BIF for setting internal copy literal range

---
 erts/emulator/beam/beam_bif_load.c | 104 ++++++++++++++++++++++++++++++++++---
 erts/emulator/beam/bif.tab         |   1 +
 2 files changed, 99 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index c3ebf71a01..e61d9ab0a6 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -154,7 +154,7 @@ static struct /* Protected by code_write_permission */
 {
     Process* stager;
     ErtsThrPrgrLaterOp lop;
-}commiter_state;
+} committer_state;
 #endif
 
 static Eterm
@@ -367,9 +367,9 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 	 * schedulers to read active code_ix in a safe way while executing
 	 * without any memory barriers at all. 
 	 */
-	ASSERT(commiter_state.stager == NULL);
-	commiter_state.stager = c_p;
-	erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &commiter_state.lop);
+	ASSERT(committer_state.stager == NULL);
+	committer_state.stager = c_p;
+	erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &committer_state.lop);
 	erts_proc_inc_refc(c_p);
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	/*
@@ -385,11 +385,11 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 #ifdef ERTS_SMP
 static void smp_code_ix_commiter(void* null)
 {
-    Process* p = commiter_state.stager;
+    Process* p = committer_state.stager;
 
     erts_commit_staging_code_ix();
 #ifdef DEBUG
-    commiter_state.stager = NULL;
+    committer_state.stager = NULL;
 #endif
     erts_release_code_write_permission();
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
@@ -1007,6 +1007,98 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
     return 0;
 }
 
+#undef in_area
+
+#ifdef ERTS_SMP
+static void copy_literals_commit(void*);
+#endif
+
+typedef struct {
+    Eterm *ptr;
+    Uint   sz;
+} copy_literals_t;
+
+copy_literals_t erts_clrange = {NULL, 0};
+
+/* copy literals
+ *
+ * copy_literals.ptr = LitPtr
+ * copy_literals.sz  = LitSz
+ * ------ THR PROG COMMIT -----
+ *
+ *  - check process code
+ *  - check process code
+ *  ...
+ * copy_literals.ptr = NULL
+ * copy_literals.sz  = 0
+ * ------ THR PROG COMMIT -----
+ *  ...
+ */
+
+
+BIF_RETTYPE copy_literals_2(BIF_ALIST_2)
+{
+    Module* modp;
+    ErtsCodeIndex code_ix;
+    Eterm res = am_true;
+
+    if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    if (!erts_try_seize_code_write_permission(BIF_P)) {
+	ERTS_BIF_YIELD2(bif_export[BIF_copy_literals_2], BIF_P, BIF_ARG_1, BIF_ARG_2);
+    }
+
+    code_ix = erts_active_code_ix();
+
+    if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL || !modp->old.code_hdr) {
+        res = am_false;
+        goto done;
+    }
+
+    if (BIF_ARG_2 == am_true) {
+        if (erts_clrange.ptr != NULL) {
+            res = am_aborted;
+            goto done;
+       }
+        erts_clrange.ptr = (Eterm*) modp->old.code_hdr->literals_start;
+        erts_clrange.sz  = (Eterm*) modp->old.code_hdr->literals_end - erts_clrange.ptr;
+    } else if (BIF_ARG_2 == am_false) {
+        erts_clrange.ptr = NULL;
+        erts_clrange.sz  = 0;
+    }
+
+#ifdef ERTS_SMP
+    ASSERT(committer_state.stager == NULL);
+    committer_state.stager = BIF_P;
+    erts_schedule_thr_prgr_later_op(copy_literals_commit, NULL, &committer_state.lop);
+    erts_proc_inc_refc(BIF_P);
+    erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
+    ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+#endif
+done:
+    erts_release_code_write_permission();
+    BIF_RET(res);
+}
+
+#ifdef ERTS_SMP
+static void copy_literals_commit(void* null) {
+    Process* p = committer_state.stager;
+#ifdef DEBUG
+    committer_state.stager = NULL;
+#endif
+    erts_release_code_write_permission();
+    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    if (!ERTS_PROC_IS_EXITING(p)) {
+	erts_resume(p, ERTS_PROC_LOCK_STATUS);
+    }
+    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_dec_refc(p);
+}
+#endif /* ERTS_SMP */
+
+
 BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 {
     ErtsCodeIndex code_ix;
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f45f886395..63a0d0b1f0 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -642,6 +642,7 @@ bif erts_debug:map_info/1
 # New in 19.0
 #
 
+bif erlang:copy_literals/2
 bif binary:split/2
 bif binary:split/3
 bif erts_debug:size_shared/1
-- 
cgit v1.2.3


From 1391715d8bbba315e1509e60e6245159a009bd9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 25 Sep 2015 17:03:02 +0200
Subject: Use copy literal range check in message passing and purging

---
 erts/emulator/beam/beam_bif_load.c |   5 --
 erts/emulator/beam/copy.c          | 136 ++++++++++++++++++-------------------
 erts/emulator/beam/global.h        |  10 +++
 3 files changed, 75 insertions(+), 76 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index e61d9ab0a6..6b6c066211 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1013,11 +1013,6 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 static void copy_literals_commit(void*);
 #endif
 
-typedef struct {
-    Eterm *ptr;
-    Uint   sz;
-} copy_literals_t;
-
 copy_literals_t erts_clrange = {NULL, 0};
 
 /* copy literals
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 4f4d20cb83..5bca3877b5 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -276,49 +276,35 @@ do {									\
 #define BOXED_SHARED_UNPROCESSED ((Eterm) 2)
 #define BOXED_SHARED_PROCESSED	 ((Eterm) 3)
 
+#define COUNT_OFF_HEAP (0)
 
-/*
- *  Is an object in the local heap of a process?
- */
-
-#define INHEAP_SIMPLE(p, ptr) (					\
-    (OLD_HEAP(p) && OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p)) ||	\
-    (HEAP_START(p) <= ptr && ptr < HEAP_END(p))			\
-  )
-#define INHEAP(p, ptr) (					\
-    INHEAP_SIMPLE(p, ptr) ||					\
-    (force_local ? (force_local = 0, 1) : 0)			\
-  )
-#define COUNT_OFF_HEAP 0
-
-
+#define IN_LITERAL_PURGE_AREA(info, ptr)                 \
+    ((info)->range_ptr && (                              \
+        (info)->range_ptr <= (ptr) &&                    \
+        (ptr) < ((info)->range_ptr + (info)->range_sz)))
 /*
  *  Return the real size of an object and find sharing information
  *  This currently returns the same as erts_debug:size/1.
  *  It is argued whether the size of subterms in constant pools
  *  should be counted or not.
  */
+
 Uint size_shared(Eterm obj)
 {
     Eterm saved_obj = obj;
     Uint sum = 0;
     Eterm* ptr;
-    Process* myself;
 
     DECLARE_EQUEUE(s);
     DECLARE_BITSTORE(b);
 
-    myself = erts_get_current_process();
-    if (myself == NULL)
-	return size_object(obj);
-
     for (;;) {
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
 	    ptr = list_val(obj);
 	    /* we're not counting anything that's outside our heap */
-	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+	    if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) {
 		goto pop_next;
 	    }
 	    head = CAR(ptr);
@@ -355,7 +341,7 @@ Uint size_shared(Eterm obj)
 	    Eterm hdr;
 	    ptr = boxed_val(obj);
 	    /* we're not counting anything that's outside our heap */
-	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+	    if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) {
 		goto pop_next;
 	    }
 	    hdr = *ptr;
@@ -482,7 +468,7 @@ cleanup:
 	case TAG_PRIMARY_LIST: {
 	    Eterm head, tail;
 	    ptr = list_val(obj);
-	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+	    if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) {
 		goto cleanup_next;
 	    }
 	    head = CAR(ptr);
@@ -512,7 +498,7 @@ cleanup:
 	case TAG_PRIMARY_BOXED: {
 	    Eterm hdr;
 	    ptr = boxed_val(obj);
-	    if (!COUNT_OFF_HEAP && !INHEAP_SIMPLE(myself, ptr)) {
+	    if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) {
 		goto cleanup_next;
 	    }
 	    hdr = *ptr;
@@ -1052,8 +1038,9 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
     Uint e;
     unsigned sz;
     Eterm* ptr;
-    Process* myself;
-    int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF;
+#ifdef DEBUG
+    Eterm mypid = erts_get_current_pid();
+#endif
 
     DECLARE_EQUEUE_INIT_INFO(s, info);
     DECLARE_BITSTORE_INIT_INFO(b, info);
@@ -1073,12 +1060,11 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
     flags |= disable_copy_shared;
 #endif
 
-    myself = erts_get_current_process();
-    if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
+    if (flags & ERTS_SHCOPY_FLG_NONE)
 	return size_object(obj);
 
-    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", myself->common.id, obj));
-    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", myself->common.id, obj));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj));
 
     /* step #1:
        -------------------------------------------------------
@@ -1102,9 +1088,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 	    Eterm head, tail;
 	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
-	    if (!INHEAP(myself, ptr)) {
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
-                info->literal_size += size_object(obj);
+	    if (erts_is_literal(obj,ptr)) {
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
+                if (IN_LITERAL_PURGE_AREA(info,ptr))
+                    info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
 	    head = CAR(ptr);
@@ -1115,7 +1102,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 		primary_tag(head) == TAG_PRIMARY_HEADER) {
 		if (tail != THE_NON_VALUE) {
 		    e = SHTABLE_NEXT(t);
-		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", myself->common.id, ptr));
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", mypid, ptr));
 		    SHTABLE_PUSH(t, head, tail, ptr);
 		    CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED;
 		    CDR(ptr) = THE_NON_VALUE;
@@ -1125,17 +1112,17 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 	    /* else make it visited now */
 	    switch (primary_tag(tail)) {
 	    case TAG_PRIMARY_LIST:
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", myself->common.id, ptr));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", mypid, ptr));
 		CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER;
 		break;
 	    case TAG_PRIMARY_IMMED1:
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", myself->common.id, ptr));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", mypid, ptr));
 		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
 		CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head);
 		break;
 	    case TAG_PRIMARY_BOXED:
 		BITSTORE_PUT(b, primary_tag(head));
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", myself->common.id, ptr));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", mypid, ptr));
 		CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER;
 		CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER;
 		break;
@@ -1152,9 +1139,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 	    Eterm hdr;
 	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
-	    if (!INHEAP(myself, ptr)) {
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", myself->common.id, ptr, obj));
-                info->literal_size += size_object(obj);
+	    if (erts_is_literal(obj,ptr)) {
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
+                if (IN_LITERAL_PURGE_AREA(info,ptr))
+                    info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
 	    hdr = *ptr;
@@ -1163,14 +1151,14 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
 	    if (primary_tag(hdr) != TAG_PRIMARY_HEADER) {
 		if (primary_tag(hdr) == BOXED_VISITED) {
 		    e = SHTABLE_NEXT(t);
-		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", myself->common.id, ptr));
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", mypid, ptr));
 		    SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr);
 		    *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED;
 		}
 		goto pop_next;
 	    }
 	    /* else make it visited now */
-	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", myself->common.id, ptr));
+	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", mypid, ptr));
 	    *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED;
 	    /* and count it */
 	    ASSERT(is_header(hdr));
@@ -1293,7 +1281,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
                 info->shtable_start = t.start;
                 info->shtable_alloc_type = t.alloc_type;
                 /* single point of return: the size of the object */
-                VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", myself->common.id, sum));
+                VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum));
                 return sum + info->literal_size;
 	    }
 	    obj = EQUEUE_GET(s);
@@ -1319,9 +1307,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     Eterm* resp;
     Eterm *hbot, *hend;
     unsigned remaining;
-    Process* myself;
-    int force_local = flags & ERTS_SHCOPY_FLG_TMPBUF;
 #ifdef DEBUG
+    Eterm mypid = erts_get_current_pid();
     Eterm saved_obj = obj;
 #endif
 
@@ -1343,11 +1330,10 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     flags |= disable_copy_shared;
 #endif
 
-    myself = erts_get_current_process();
-    if (myself == NULL || (flags & ERTS_SHCOPY_FLG_NONE))
+    if (flags & ERTS_SHCOPY_FLG_NONE)
 	return copy_struct(obj, size, hpp, off_heap);
 
-    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", myself->common.id, obj));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj));
 
     /* step #2: was performed before this function was called
        -------------------------------------------------------
@@ -1376,10 +1362,14 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    Eterm head, tail;
 	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
-	    if (!INHEAP(myself, ptr)) {
-                Uint bsz;
-                *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
-                hbot -= bsz;
+	    if (erts_is_literal(obj,ptr)) {
+                if (!IN_LITERAL_PURGE_AREA(info,ptr)) {
+                    *resp = obj;
+                } else {
+                    Uint bsz = 0;
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                    hbot -= bsz;
+                }
 		goto cleanup_next;
 	    }
 	    head = CAR(ptr);
@@ -1399,23 +1389,23 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 		    head = SHTABLE_X(t, e);
 		    tail = SHTABLE_Y(t, e);
 		    ptr = &(SHTABLE_X(t, e));
-		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)));
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", mypid, ptr, SHTABLE_REV(t, e)));
 		    SHTABLE_FWD_UPD(t, e, hp);
 		}
 	    }
 	    /* if not already clean, clean it up and copy it */
 	    if (primary_tag(tail) == TAG_PRIMARY_HEADER) {
 		if (primary_tag(head) == TAG_PRIMARY_HEADER) {
-		    Eterm saved = BITSTORE_GET(b);
-		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", myself->common.id, ptr));
-		    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved;
-		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED;
+                    Eterm saved = BITSTORE_GET(b);
+                    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr));
+                    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved;
+                    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED;
 		} else {
-		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", myself->common.id, ptr));
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr));
 		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST;
 		}
 	    } else if (primary_tag(head) == TAG_PRIMARY_HEADER) {
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", myself->common.id, ptr));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", mypid, ptr));
 		CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail);
 		CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1;
 	    } else {
@@ -1439,10 +1429,14 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    Eterm hdr;
 	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
-	    if (!INHEAP(myself, ptr)) {
-                Uint bsz;
-                *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
-                hbot -= bsz;
+	    if (erts_is_literal(obj,ptr)) {
+                if (!IN_LITERAL_PURGE_AREA(info,ptr)) {
+                    *resp = obj;
+                } else {
+                    Uint bsz = 0;
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                    hbot -= bsz;
+                }
 		goto cleanup_next;
 	    }
 	    hdr = *ptr;
@@ -1463,13 +1457,13 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 		*ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED;
 		hdr = SHTABLE_X(t, e);
 		ASSERT(primary_tag(hdr) == BOXED_VISITED);
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", myself->common.id, ptr, SHTABLE_REV(t, e)));
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", mypid, ptr, SHTABLE_REV(t, e)));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr));
 		SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER;
 		SHTABLE_FWD_UPD(t, e, hp);
 		break;
 	    case BOXED_VISITED:
-		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", myself->common.id, ptr));
+		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr));
 		*ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER;
 		break;
 	    }
@@ -1749,13 +1743,13 @@ all_clean:
 	VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr));
 	/* entry was a list */
 	if (SHTABLE_Y(t, e) != THE_NON_VALUE) {
-	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", myself->common.id, ptr));
+	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", mypid, ptr));
 	    CAR(ptr) = SHTABLE_X(t, e);
 	    CDR(ptr) = SHTABLE_Y(t, e);
 	}
 	/* entry was boxed */
 	else {
-	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", myself->common.id, ptr));
+	    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", mypid, ptr));
 	    *ptr = SHTABLE_X(t, e);
 	    ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER);
 	}
@@ -1769,9 +1763,9 @@ all_clean:
     }
 #endif
 
-    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", myself->common.id, saved_obj));
-    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", myself->common.id, result));
-    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", myself->common.id, result));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", mypid, saved_obj));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", mypid, result));
+    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, result));
 
     ASSERT(hbot == hp);
     ASSERT(size == ((hp - *hpp) + (hend - hbot)));
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index f106b941ef..71d072fa7e 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -982,6 +982,12 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 /* beam_bif_load.c */
 Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp);
 
+typedef struct {
+    Eterm *ptr;
+    Uint   sz;
+} copy_literals_t;
+
+extern copy_literals_t erts_clrange;
 
 /* beam_load.c */
 typedef struct {
@@ -1063,6 +1069,8 @@ typedef struct {
     Eterm* shtable_start;
     ErtsAlcType_t shtable_alloc_type;
     Uint literal_size;
+    Eterm *range_ptr;
+    Uint  range_sz;
 } erts_shcopy_t;
 
 #define INITIALIZE_SHCOPY(info)                         \
@@ -1071,6 +1079,8 @@ do {                                                    \
     info.bitstore_start = info.bitstore_default;        \
     info.shtable_start = info.shtable_default;          \
     info.literal_size = 0;                              \
+    info.range_ptr = erts_clrange.ptr;                  \
+    info.range_sz  = erts_clrange.sz;                   \
 } while(0)
 
 #define DESTROY_SHCOPY(info)                                            \
-- 
cgit v1.2.3


From 0d5ee7a4ad7bc41b7cca878990926fb5ba57f6a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 1 Oct 2015 12:19:50 +0200
Subject: Do not use GCC extensions in copy

---
 erts/emulator/beam/copy.c   | 37 ++++++++++++++++++++-----------------
 erts/emulator/beam/global.h | 25 +++++++++++++++----------
 2 files changed, 35 insertions(+), 27 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 5bca3877b5..83ca527334 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -258,18 +258,19 @@ do {									\
     }									\
 } while(0)
 
-#define BITSTORE_GET(s)	({						\
-    UWord result;							\
-    if (WSTK_CONCAT(s,_bitoffs) <= 0) {					\
-	WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)];	\
-	WSTK_CONCAT(s,_offset)++;					\
-	WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord);			\
-    }									\
-    WSTK_CONCAT(s,_bitoffs) -= 2;					\
-    result = WSTK_CONCAT(s,_buffer) & 3;				\
-    WSTK_CONCAT(s,_buffer) >>= 2;					\
-    result;								\
-})
+#define BITSTORE_FETCH(s,dst)                                           \
+do {                                                                    \
+    UWord result;                                                       \
+    if (WSTK_CONCAT(s,_bitoffs) <= 0) {                                 \
+        WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)];      \
+        WSTK_CONCAT(s,_offset)++;                                       \
+        WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord);                      \
+    }                                                                   \
+    WSTK_CONCAT(s,_bitoffs) -= 2;                                       \
+    result = WSTK_CONCAT(s,_buffer) & 3;                                \
+    WSTK_CONCAT(s,_buffer) >>= 2;                                       \
+    (dst) = result;                                                     \
+} while(0)
 
 #define BOXED_VISITED_MASK	 ((Eterm) 3)
 #define BOXED_VISITED		 ((Eterm) 1)
@@ -476,7 +477,8 @@ cleanup:
 	    /* if not already clean, clean it up */
 	    if (primary_tag(tail) == TAG_PRIMARY_HEADER) {
 		if (primary_tag(head) == TAG_PRIMARY_HEADER) {
-		    Eterm saved = BITSTORE_GET(b);
+		    Eterm saved;
+                    BITSTORE_FETCH(b, saved);
 		    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved;
 		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED;
 		} else {
@@ -1396,10 +1398,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    /* if not already clean, clean it up and copy it */
 	    if (primary_tag(tail) == TAG_PRIMARY_HEADER) {
 		if (primary_tag(head) == TAG_PRIMARY_HEADER) {
-                    Eterm saved = BITSTORE_GET(b);
-                    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr));
-                    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved;
-                    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED;
+		    Eterm saved;
+                    BITSTORE_FETCH(b, saved);
+		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr));
+		    CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved;
+		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED;
 		} else {
 		    VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr));
 		    CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 71d072fa7e..e9f7901a51 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -873,7 +873,7 @@ typedef struct {
     int possibly_empty;
     Eterm* end;
     ErtsAlcType_t alloc_type;
-}ErtsEQueue;
+} ErtsEQueue;
 
 #define DEF_EQUEUE_SIZE (16)
 
@@ -918,15 +918,20 @@ do {							\
 
 #define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty)
 
-#define EQUEUE_GET(q) ({				\
-    UWord x;						\
-    q.possibly_empty = 1;				\
-    x = *(q.front);					\
-    if (++(q.front) == q.end) {				\
-        q.front = q.start;				\
-    }							\
-    x;							\
-})
+ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q) {
+    Eterm x;
+    q->possibly_empty = 1;
+    x = *(q->front);
+    if (++(q->front) == q->end) {
+        q->front = q->start;
+    }
+    return x;
+}
+#endif
+#define EQUEUE_GET(q) erts_equeue_get(&(q));
 
 /* binary.c */
 
-- 
cgit v1.2.3


From 15c26af0a35fb482408885ea90b3e669d64d71f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 17 Nov 2015 16:26:35 +0100
Subject: Fix erts_debug:copy_shared/1 prototype

---
 erts/emulator/beam/bif.tab | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 63a0d0b1f0..c49a3ff313 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -646,7 +646,7 @@ bif erlang:copy_literals/2
 bif binary:split/2
 bif binary:split/3
 bif erts_debug:size_shared/1
-bif 'erl.system.debug':size_shared/1	ebif_erts_debug_size_shared_1
+bif erts_debug:copy_shared/1
 
 #
 # Obsolete
-- 
cgit v1.2.3


From 3f29cf1f72f50d20ce1864f76e1a298602429ca4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 18 Nov 2015 16:59:08 +0100
Subject: Fix rebase of SHCOPY seq_tokens

---
 erts/emulator/beam/erl_message.c | 55 ++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 27 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 11890a756d..66b337402d 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -691,22 +691,15 @@ erts_send_message(Process* sender,
 #ifdef USE_VM_PROBES
 	Uint dt_utag_size = 0;
 #endif
-#ifdef SHCOPY_SEND
-        unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT;
-        erts_shcopy_t info;
-        INITIALIZE_SHCOPY(info);
-#endif
-	BM_SWAP_TIMER(send,size);
-#ifdef SHCOPY_SEND
-        INITIALIZE_SHCOPY(info);
-        msize = copy_shared_calculate(message, &info, shflags);
-#else
-        msize = size_object(message);
-#endif
-	BM_SWAP_TIMER(size,send);
+        BM_SWAP_TIMER(send,size);
 
+        /* SHCOPY corrupts the heap between
+         * copy_shared_calculate, and
+         * copy_shared_perform. (it inserts move_markers like the gc).
+         * Make sure we don't use the heap between those instances.
+         */
 #ifdef USE_VM_PROBES
-	if (stoken != am_have_dt_utag) {
+        if (stoken != am_have_dt_utag) {
 #endif
 	    seq_trace_update_send(sender);
 	    seq_trace_output(stoken, message, SEQ_TRACE_SEND, 
@@ -714,23 +707,31 @@ erts_send_message(Process* sender,
 	    seq_trace_size = 6; /* TUPLE5 */
 #ifdef USE_VM_PROBES
 	}
-	if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
-	    dt_utag_size = size_object(DT_UTAG(sender));
-	} else if (stoken == am_have_dt_utag ) {
-	    stoken = NIL;
-	}
+        if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
+            dt_utag_size = size_object(DT_UTAG(sender));
+        } else if (stoken == am_have_dt_utag ) {
+            stoken = NIL;
+        }
+#endif
+
+#ifdef SHCOPY_SEND
+        INITIALIZE_SHCOPY(info);
+        msize = copy_shared_calculate(message, &info, shflags);
+#else
+        msize = size_object(message);
 #endif
+        BM_SWAP_TIMER(size,send);
 
-	mp = erts_alloc_message_heap_state(receiver,
-					   &receiver_state,
-					   receiver_locks,
-					   (msize
+        mp = erts_alloc_message_heap_state(receiver,
+                                           &receiver_state,
+                                           receiver_locks,
+                                           (msize
 #ifdef USE_VM_PROBES
-					    + dt_utag_size
+                                            + dt_utag_size
 #endif
-					    + seq_trace_size),
-					   &hp,
-					   &ohp);
+                                            + seq_trace_size),
+                                           &hp,
+                                           &ohp);
 
         BM_SWAP_TIMER(send,copy);
 
-- 
cgit v1.2.3


From 5d764f988ab09326d24e39a172083b09ab364c6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 18 Nov 2015 17:58:14 +0100
Subject: Refactor sharing preserved copy flags

The TMPBUF option is no longer needed due to is_literal test and NONE
was only used for initial debugging. So we remove the entire option.
---
 erts/emulator/beam/beam_debug.c  | 10 ++--------
 erts/emulator/beam/copy.c        | 22 ++--------------------
 erts/emulator/beam/erl_db.c      |  4 ++--
 erts/emulator/beam/erl_message.c |  9 ++++-----
 erts/emulator/beam/erl_message.h |  5 -----
 erts/emulator/beam/erl_process.c |  5 ++---
 erts/emulator/beam/global.h      | 10 ++--------
 7 files changed, 14 insertions(+), 51 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index b007d000bf..e37bd4d78c 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -97,20 +97,14 @@ erts_debug_copy_shared_1(BIF_ALIST_1)
     Eterm* hp;
     Eterm copy;
     erts_shcopy_t info;
-#ifdef SHCOPY_DISABLE
-    extern int disable_copy_shared;
-#endif
     INITIALIZE_SHCOPY(info);
 
-    size = copy_shared_calculate(term, &info, 0);
+    size = copy_shared_calculate(term, &info);
     if (size > 0) {
       hp = HAlloc(p, size);
     }
-    copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap, 0);
+    copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap);
     DESTROY_SHCOPY(info);
-#ifdef SHCOPY_DISABLE
-    disable_copy_shared = 0;
-#endif
     BIF_RET(copy);
 }
 
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 83ca527334..67a96f6442 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -923,10 +923,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
  *  Using an ESTACK but not very transparently; consider refactoring
  */
 
-#ifdef SHCOPY_DISABLE
-int disable_copy_shared = ERTS_SHCOPY_FLG_NONE;
-#endif
-
 #define DECLARE_SHTABLE(s)					\
     DECLARE_ESTACK(s);						\
     Uint ESTK_CONCAT(s,_offset) = 0
@@ -1034,7 +1030,7 @@ do {								\
  *  Copy object "obj" preserving sharing.
  *  First half: count size and calculate sharing.
  */
-Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
+Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
 {
     Uint sum;
     Uint e;
@@ -1058,13 +1054,6 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
     if (IS_CONST(obj))
 	return 0;
 
-#ifdef SHCOPY_DISABLE
-    flags |= disable_copy_shared;
-#endif
-
-    if (flags & ERTS_SHCOPY_FLG_NONE)
-	return size_object(obj);
-
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj));
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj));
 
@@ -1299,7 +1288,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info, Uint32 flags)
  *  Second half: copy and restore the object.
  */
 Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
-                         Eterm** hpp, ErlOffHeap* off_heap, Uint32 flags) {
+                         Eterm** hpp, ErlOffHeap* off_heap) {
     Uint e;
     unsigned sz;
     Eterm* ptr;
@@ -1328,13 +1317,6 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     if (IS_CONST(obj))
 	return obj;
 
-#ifdef SHCOPY_DISABLE
-    flags |= disable_copy_shared;
-#endif
-
-    if (flags & ERTS_SHCOPY_FLG_NONE)
-	return copy_struct(obj, size, hpp, off_heap);
-
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj));
 
     /* step #2: was performed before this function was called
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 59a0c0b808..3030c1c91a 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1832,7 +1832,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 			     tb->common.id,
 			     from_pid,
 			     BIF_ARG_3), 
-		      ERTS_SND_FLG_SHCOPY_TMPBUF);
+                      0);
     erts_smp_proc_unlock(to_proc, to_locks);
     UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
@@ -3211,7 +3211,7 @@ retry:
 			     tb->common.id,
 			     p->common.id,
 			     heir_data), 
-		      ERTS_SND_FLG_SHCOPY_TMPBUF);
+                      0);
     erts_smp_proc_unlock(to_proc, to_locks);
     return !0;
 }
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 66b337402d..a964f1968b 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -665,7 +665,6 @@ erts_send_message(Process* sender,
 #endif
     erts_aint32_t receiver_state;
 #ifdef SHCOPY_SEND
-    unsigned shflags = (flags & ERTS_SND_FLG_SHCOPY_MASK) >> ERTS_SND_FLG_SHCOPY_SHIFT;
     erts_shcopy_t info;
 #endif
     BM_STOP_TIMER(system);
@@ -716,7 +715,7 @@ erts_send_message(Process* sender,
 
 #ifdef SHCOPY_SEND
         INITIALIZE_SHCOPY(info);
-        msize = copy_shared_calculate(message, &info, shflags);
+        msize = copy_shared_calculate(message, &info);
 #else
         msize = size_object(message);
 #endif
@@ -737,7 +736,7 @@ erts_send_message(Process* sender,
 
 #ifdef SHCOPY_SEND
 	if (is_not_immed(message))
-            message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags);
+            message = copy_shared_perform(message, msize, &info, &hp, ohp);
         DESTROY_SHCOPY(info);
 #else
 	if (is_not_immed(message))
@@ -786,7 +785,7 @@ erts_send_message(Process* sender,
 	    BM_SWAP_TIMER(send,size);
 #ifdef SHCOPY_SEND
             INITIALIZE_SHCOPY(info);
-            msize = copy_shared_calculate(message, &info, shflags);
+            msize = copy_shared_calculate(message, &info);
 #else
             msize = size_object(message);
 #endif
@@ -801,7 +800,7 @@ erts_send_message(Process* sender,
 	    BM_SWAP_TIMER(send,copy);
 #ifdef SHCOPY_SEND
             if (is_not_immed(message))
-                message = copy_shared_perform(message, msize, &info, &hp, ohp, shflags);
+                message = copy_shared_perform(message, msize, &info, &hp, ohp);
             DESTROY_SHCOPY(info);
 #else
             if (is_not_immed(message))
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 52a648fe0b..740ae46a0f 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -238,11 +238,6 @@ do {									\
 
 #define ERTS_SND_FLG_NO_SEQ_TRACE		(((unsigned) 1) << 0)
 
-#define ERTS_SND_FLG_SHCOPY_SHIFT	1
-#define ERTS_SND_FLG_SHCOPY_MASK        (ERTS_SHCOPY_FLG_MASK   << ERTS_SND_FLG_SHCOPY_SHIFT)
-#define ERTS_SND_FLG_SHCOPY_NONE        (ERTS_SHCOPY_FLG_NONE   << ERTS_SND_FLG_SHCOPY_SHIFT)
-#define ERTS_SND_FLG_SHCOPY_TMPBUF      (ERTS_SHCOPY_FLG_TMPBUF << ERTS_SND_FLG_SHCOPY_SHIFT)
-
 #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
    (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a691a3c773..23616f36bf 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10748,7 +10748,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     erts_aint32_t state = 0;
     erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL;
 #ifdef SHCOPY_SPAWN
-    unsigned shflags = 0;	/* could be taken from so->flags, if necessary */
     erts_shcopy_t info;
     INITIALIZE_SHCOPY(info);
 #endif
@@ -10804,7 +10803,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
     BM_SWAP_TIMER(system,size);
 #ifdef SHCOPY_SPAWN
-    arg_size = copy_shared_calculate(args, &info, shflags);
+    arg_size = copy_shared_calculate(args, &info);
 #else
     arg_size = size_object(args);
 #endif
@@ -10886,7 +10885,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     BM_START_TIMER(system);
     BM_SWAP_TIMER(system,copy);
 #ifdef SHCOPY_SPAWN
-    p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap, shflags);
+    p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap);
     DESTROY_SHCOPY(info);
 #else
     p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index e9f7901a51..98c275a20c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1052,14 +1052,8 @@ void erl_error(char*, va_list);
 #ifdef SHCOPY
 #define SHCOPY_SEND
 #define SHCOPY_SPAWN
-/* Use this if you want sharing-preserving copy to be initially disabled */
-#undef SHCOPY_DISABLE
 #endif
 
-#define ERTS_SHCOPY_FLG_MASK    (((Uint32) 3) << 0)
-#define ERTS_SHCOPY_FLG_NONE    (((Uint32) 1) << 0)
-#define ERTS_SHCOPY_FLG_TMPBUF  (((Uint32) 1) << 1) /* forces INHEAP to true */
-
 /* The persistent state while the sharing-preserving copier works */
 
 typedef struct {
@@ -1106,8 +1100,8 @@ Eterm copy_object_x(Eterm, Process*, Uint);
 #define copy_object(Term, Proc) copy_object_x(Term,Proc,0)
 
 Uint size_object(Eterm);
-Uint copy_shared_calculate(Eterm, erts_shcopy_t*, Uint32);
-Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, Uint32);
+Uint copy_shared_calculate(Eterm, erts_shcopy_t*);
+Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*);
 
 Uint size_shared(Eterm);
 
-- 
cgit v1.2.3


From 8b93e77a6d3df65e45a3ca3e2ec9dd4c52464f63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 18 Nov 2015 18:47:55 +0100
Subject: Use sharing preserving copy in enif_make_copy

---
 erts/emulator/beam/erl_nif.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index a37cda93ef..2ff509e6d0 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -379,9 +379,19 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
 {
     Uint sz;
     Eterm* hp;
+#ifdef SHCOPY
+    erts_shcopy_t info;
+    INITIALIZE_SHCOPY(info);
+    sz = copy_shared_calculate(src_term, &info);
+    hp = alloc_heap(dst_env, sz);
+    src_term = copy_shared_perform(src_term, sz, &info, &hp, &MSO(dst_env->proc));
+    DESTROY_SHCOPY(info);
+    return src_term;
+#else
     sz = size_object(src_term);
     hp = alloc_heap(dst_env, sz);
     return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc));
+#endif
 }
 
 
-- 
cgit v1.2.3


From 02f038355d16e9b6474837727878f58e4ca669c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 18 Nov 2015 20:12:38 +0100
Subject: Use sharing preserving copy error messages and exceptions

---
 erts/emulator/beam/erl_message.c | 28 +++++++++++++++++++++++++---
 erts/emulator/beam/erl_process.c | 33 +++++++++++++++++++++++----------
 2 files changed, 48 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index a964f1968b..f64385d92b 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -850,6 +850,9 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
     Eterm temptoken;
     ErtsMessage* mp;
     ErlOffHeap *ohp;
+#ifdef SHCOPY_SEND
+    erts_shcopy_t info;
+#endif
 
     if (token != NIL
 #ifdef USE_VM_PROBES
@@ -858,13 +861,23 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	) {
 
 	ASSERT(is_tuple(token));
-	sz_reason = size_object(reason);
 	sz_token = size_object(token);
 	sz_from = size_object(from);
+#ifdef SHCOPY_SEND
+        INITIALIZE_SHCOPY(info);
+        sz_reason = copy_shared_calculate(reason, &info);
+#else
+	sz_reason = size_object(reason);
+#endif
 	mp = erts_alloc_message_heap(to, to_locksp,
 				     sz_reason + sz_from + sz_token + 4,
 				     &hp, &ohp);
+#ifdef SHCOPY_SEND
+        mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp);
+        DESTROY_SHCOPY(info);
+#else
 	mess = copy_struct(reason, sz_reason, &hp, ohp);
+#endif
 	from_copy = copy_struct(from, sz_from, &hp, ohp);
 	save = TUPLE3(hp, am_EXIT, from_copy, mess);
 	hp += 4;
@@ -873,13 +886,22 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	temptoken = copy_struct(token, sz_token, &hp, ohp);
 	erts_queue_message(to, to_locksp, mp, save, temptoken);
     } else {
-	sz_reason = size_object(reason);
 	sz_from = IS_CONST(from) ? 0 : size_object(from);
-
+#ifdef SHCOPY_SEND
+        INITIALIZE_SHCOPY(info);
+        sz_reason = copy_shared_calculate(reason, &info);
+#else
+	sz_reason = size_object(reason);
+#endif
 	mp = erts_alloc_message_heap(to, to_locksp,
 				     sz_reason+sz_from+4, &hp, &ohp);
 
+#ifdef SHCOPY_SEND
+        mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp);
+        DESTROY_SHCOPY(info);
+#else
 	mess = copy_struct(reason, sz_reason, &hp, ohp);
+#endif
 	from_copy = (IS_CONST(from)
 		     ? from
 		     : copy_struct(from, sz_from, &hp, ohp));
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 23616f36bf..f347d57a9d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11520,31 +11520,44 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 {
     ErtsMessage *mp;
     ErlOffHeap *ohp;
+    Eterm* hp;
+    Eterm mess;
+#ifdef SHCOPY_SEND
+    erts_shcopy_t info;
+#endif
 
     if (token == NIL 
 #ifdef USE_VM_PROBES
 	|| token == am_have_dt_utag
 #endif
 	) {
-	Eterm* hp;
-	Eterm mess;
-
-	mp = erts_alloc_message_heap(to, to_locksp,
-				     term_size, &hp, &ohp);
+#ifdef SHCOPY_SEND
+        INITIALIZE_SHCOPY(info);
+        term_size = copy_shared_calculate(exit_term, &info);
+	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
+        mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp);
+        DESTROY_SHCOPY(info);
+#else
+	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
+#endif
 	erts_queue_message(to, to_locksp, mp, mess, NIL);
     } else {
-	Eterm* hp;
-	Eterm mess;
 	Eterm temp_token;
 	Uint sz_token;
 
 	ASSERT(is_tuple(token));
 	sz_token = size_object(token);
-
-	mp = erts_alloc_message_heap(to, to_locksp,
-				     term_size+sz_token, &hp, &ohp);
+#ifdef SHCOPY_SEND
+        INITIALIZE_SHCOPY(info);
+        term_size = copy_shared_calculate(exit_term, &info);
+	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
+        mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp);
+        DESTROY_SHCOPY(info);
+#else
+	mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp);
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
+#endif
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL);
 	temp_token = copy_struct(token, sz_token, &hp, ohp);
-- 
cgit v1.2.3


From 6ff15f23c68db356bbad6ab5f939c191b58d453d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 19 Nov 2015 19:36:08 +0100
Subject: Refactor have seq_trace token test

---
 erts/emulator/beam/beam_emu.c      |  2 +-
 erts/emulator/beam/bif.c           |  6 +-----
 erts/emulator/beam/dist.c          | 24 ++++++------------------
 erts/emulator/beam/erl_bif_trace.c | 26 +++++---------------------
 erts/emulator/beam/erl_db_util.c   | 12 ++----------
 erts/emulator/beam/erl_message.c   | 19 ++++++-------------
 erts/emulator/beam/erl_message.h   |  7 +++++++
 erts/emulator/beam/erl_process.c   | 12 ++----------
 erts/emulator/beam/erl_trace.c     |  6 +-----
 9 files changed, 31 insertions(+), 83 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index af97cba61c..1a4133bceb 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1963,7 +1963,7 @@ void process_main(void)
 
          dtrace_proc_str(c_p, receiver_name);
          token2 = SEQ_TRACE_TOKEN(c_p);
-         if (token2 != NIL && token2 != am_have_dt_utag) {
+         if (have_seqtrace(token2)) {
              tok_label = signed_val(SEQ_TRACE_T_LABEL(token2));
              tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2));
              tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2));
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index f0340540cb..14ab113b32 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2038,11 +2038,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 	    save_calls(p, &exp_send);
 	
-	if (SEQ_TRACE_TOKEN(p) != NIL
-#ifdef USE_VM_PROBES
-	    && SEQ_TRACE_TOKEN(p) != am_have_dt_utag
-#endif
-	    ) {
+        if (have_seqtrace(SEQ_TRACE_TOKEN(p))) {
 	    seq_trace_update_send(p);
 	    seq_trace_output(SEQ_TRACE_TOKEN(p), msg, 
 			     SEQ_TRACE_SEND, portid, p);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 68745fc448..f480915256 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -881,11 +881,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
     DTRACE_CHARBUF(receiver_name, 64);
 #endif
 
-    if (SEQ_TRACE_TOKEN(sender) != NIL
-#ifdef USE_VM_PROBES
-	&& SEQ_TRACE_TOKEN(sender) != am_have_dt_utag 
-#endif
-	) {
+    if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) {
 	seq_trace_update_send(sender);
 	token = SEQ_TRACE_TOKEN(sender);
 	seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender);
@@ -900,7 +896,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
         erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
                       "%T", remote);
         msize = size_object(message);
-        if (token != NIL && token != am_have_dt_utag) {
+        if (have_seqtrace(token)) {
             tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
@@ -942,11 +938,7 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message,
     DTRACE_CHARBUF(receiver_name, 128);
 #endif
 
-    if (SEQ_TRACE_TOKEN(sender) != NIL
-#ifdef USE_VM_PROBES
-	&& SEQ_TRACE_TOKEN(sender) != am_have_dt_utag 
-#endif
-	) {
+    if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) {
 	seq_trace_update_send(sender);
 	token = SEQ_TRACE_TOKEN(sender);
 	seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender);
@@ -961,7 +953,7 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message,
         erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
                       "{%T,%s}", remote_name, node_name);
         msize = size_object(message);
-        if (token != NIL && token != am_have_dt_utag) {
+        if (have_seqtrace(token)) {
             tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
@@ -1006,11 +998,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
 #endif
 
     UseTmpHeapNoproc(6);
-    if (token != NIL 
-#ifdef USE_VM_PROBES
-	&& token != am_have_dt_utag
-#endif
-	) {	
+    if (have_seqtrace(token)) {
 	seq_trace_update_send(dsdp->proc);
 	seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local);
 	ctl = TUPLE5(&ctl_heap[0],
@@ -1029,7 +1017,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
                       "{%T,%s}", remote, node_name);
         erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)),
                       "%T", reason);
-        if (token != NIL && token != am_have_dt_utag) {
+        if (have_seqtrace(token)) {
             tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 03f51132b1..4d67e39e7e 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1879,11 +1879,7 @@ new_seq_trace_token(Process* p)
 {
     Eterm* hp;
 
-    if (SEQ_TRACE_TOKEN(p) == NIL
-#ifdef USE_VM_PROBES
-	|| SEQ_TRACE_TOKEN(p) == am_have_dt_utag
-#endif
-	) {
+    if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) {
 	hp = HAlloc(p, 6);
 	SEQ_TRACE_TOKEN(p) = TUPLE5(hp, make_small(0),		/* Flags  */ 
 				    make_small(0),		/* Label  */
@@ -1903,12 +1899,8 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item)
 	BIF_ERROR(p, BADARG);
     }
 
-    if (SEQ_TRACE_TOKEN(p) == NIL
-#ifdef USE_VM_PROBES
-	|| SEQ_TRACE_TOKEN(p) == am_have_dt_utag
-#endif
-	) {
-	if ((item == am_send) || (item == am_receive) || 
+    if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) {
+	if ((item == am_send)  || (item == am_receive) || 
 	    (item == am_print) || (item == am_timestamp)) {
 	    hp = HAlloc(p,3);
 	    res = TUPLE2(hp, item, am_false);
@@ -1964,11 +1956,7 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1)
  */
 BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1)    
 {
-    if (SEQ_TRACE_TOKEN(BIF_P) == NIL 
-#ifdef USE_VM_PROBES
-	|| SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag
-#endif
-	) {
+    if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) {
 	BIF_RET(am_false);
     }
     seq_trace_update_send(BIF_P);
@@ -1987,11 +1975,7 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1)
  */
 BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2)    
 {
-    if (SEQ_TRACE_TOKEN(BIF_P) == NIL
-#ifdef USE_VM_PROBES
-	|| SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag
-#endif
-	) {
+    if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) {
 	BIF_RET(am_false);
     }
     if (!(is_atom(BIF_ARG_1) || is_small(BIF_ARG_1))) {
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 0bf9558ac9..399be6058c 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2308,11 +2308,7 @@ restart:
 	    *esp++ = am_true;
 	    break;
 	case matchIsSeqTrace:
-	    if (SEQ_TRACE_TOKEN(c_p) != NIL
-#ifdef USE_VM_PROBES
-		&& SEQ_TRACE_TOKEN(c_p) != am_have_dt_utag
-#endif
-		)
+            if (have_seqtrace(SEQ_TRACE_TOKEN(c_p)))
 		*esp++ = am_true;
 	    else
 		*esp++ = am_false;
@@ -2336,11 +2332,7 @@ restart:
 	    --esp;
 	    break;
 	case matchGetSeqToken:
-	    if (SEQ_TRACE_TOKEN(c_p) == NIL
-#ifdef USE_VM_PROBES
-		|| SEQ_TRACE_TOKEN(c_p) == am_have_dt_utag
-#endif
-		) 
+            if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p)))
 		*esp++ = NIL;
 	    else {
 		Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index f64385d92b..1811651a58 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -314,7 +314,7 @@ erts_queue_dist_message(Process *rcvr,
 		DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
 
 		dtrace_proc_str(rcvr, receiver_name);
-		if (token != NIL && token != am_have_dt_utag) {
+                if (have_seqtrace(token)) {
 		    tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
 		    tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
 		    tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
@@ -335,7 +335,7 @@ erts_queue_dist_message(Process *rcvr,
             DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
 
             dtrace_proc_str(rcvr, receiver_name);
-            if (token != NIL && token != am_have_dt_utag) {
+            if (have_seqtrace(token)) {
                 tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
                 tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
                 tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
@@ -697,15 +697,13 @@ erts_send_message(Process* sender,
          * copy_shared_perform. (it inserts move_markers like the gc).
          * Make sure we don't use the heap between those instances.
          */
-#ifdef USE_VM_PROBES
-        if (stoken != am_have_dt_utag) {
-#endif
+        if (have_seqtrace(stoken)) {
 	    seq_trace_update_send(sender);
 	    seq_trace_output(stoken, message, SEQ_TRACE_SEND, 
 			     receiver->common.id, sender);
 	    seq_trace_size = 6; /* TUPLE5 */
-#ifdef USE_VM_PROBES
 	}
+#ifdef USE_VM_PROBES
         if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
             dt_utag_size = size_object(DT_UTAG(sender));
         } else if (stoken == am_have_dt_utag ) {
@@ -765,7 +763,7 @@ erts_send_message(Process* sender,
 
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(message_send)) {
-	    if (stoken != NIL && stoken != am_have_dt_utag) {
+            if (have_seqtrace(stoken)) {
 		tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
 		tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
 		tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
@@ -854,12 +852,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
     erts_shcopy_t info;
 #endif
 
-    if (token != NIL
-#ifdef USE_VM_PROBES
-	&& token != am_have_dt_utag
-#endif
-	) {
-
+    if (have_seqtrace(token)) {
 	ASSERT(is_tuple(token));
 	sz_token = size_object(token);
 	sz_from = size_object(from);
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 740ae46a0f..76387bc34c 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -128,6 +128,13 @@ struct erl_heap_fragment {
 #else
 #endif
 
+#ifdef USE_VM_PROBES
+#define have_no_seqtrace(T) ((T) == NIL || (T) == am_have_dt_utag)
+#else
+#define have_no_seqtrace(T) ((T) == NIL)
+#endif
+#define have_seqtrace(T)    (!have_no_seqtrace(T))
+
 #define ERL_MESSAGE_REF_FIELDS__			\
     ErtsMessage *next;	/* Next message */		\
     union {						\
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f347d57a9d..5907dd4567 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11526,11 +11526,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
     erts_shcopy_t info;
 #endif
 
-    if (token == NIL 
-#ifdef USE_VM_PROBES
-	|| token == am_have_dt_utag
-#endif
-	) {
+    if (!have_seqtrace(token)) {
 #ifdef SHCOPY_SEND
         INITIALIZE_SHCOPY(info);
         term_size = copy_shared_calculate(exit_term, &info);
@@ -11670,11 +11666,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 
     if ((state & ERTS_PSFLG_TRAP_EXIT)
 	&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
-	if (is_not_nil(token) 
-#ifdef USE_VM_PROBES
-	    && token != am_have_dt_utag
-#endif
-	    && token_update)
+        if (have_seqtrace(token))
 	    seq_trace_update_send(token_update);
 	if (is_value(exit_tuple))
 	    send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token);
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index d02f1f7213..b774ba97af 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1046,11 +1046,7 @@ seq_trace_update_send(Process *p)
 {
     Eterm seq_tracer = erts_get_system_seq_tracer();
     ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p))));
-    if ( (p->common.id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL)
-#ifdef USE_VM_PROBES
-	 || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag)
-#endif
-	 ) {
+    if ((p->common.id == seq_tracer) || have_no_seqtrace(SEQ_TRACE_TOKEN(p))) {
 	return 0;
     }
     SEQ_TRACE_TOKEN_SENDER(p) = p->common.id;
-- 
cgit v1.2.3


From 7dd3fcb287c6d307ce28800d79d1e4c86a533952 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 20 Nov 2015 16:07:41 +0100
Subject: erts: Fix maps decode in erlang:binary_to_term/1

Decoding a term with a large (HAMT) map in an small (FLAT) map could cause
a critical error if the external format was not produced by beam.
---
 erts/emulator/beam/external.c | 76 +++++++++++++++++++++----------------------
 1 file changed, 37 insertions(+), 39 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c6d7e3fcc5..a85aa15403 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1179,7 +1179,7 @@ typedef struct {
     ErtsHeapFactory factory;
     int remaining_n;
     char* remaining_bytes;
-    Eterm* maps_list;
+    ErtsWStack flat_maps;
     ErtsPStack hamt_array;
 } B2TDecodeContext;
 
@@ -1519,7 +1519,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
             ctx->u.dc.res = (Eterm) (UWord) NULL;
             ctx->u.dc.next = &ctx->u.dc.res;
 	    erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size);
-	    ctx->u.dc.maps_list = NULL;
+	    ctx->u.dc.flat_maps.wstart = NULL;
 	    ctx->u.dc.hamt_array.pstart = NULL;
             ctx->state = B2TDecode;
             /*fall through*/
@@ -2938,7 +2938,7 @@ dec_term(ErtsDistExternal *edep,
     int n;
     ErtsAtomEncoding char_enc;
     register Eterm* hp;        /* Please don't take the address of hp */
-    Eterm *maps_list;   /* for preprocessing of small maps */
+    DECLARE_WSTACK(flat_maps); /* for preprocessing of small maps */
     Eterm* next;
     SWord reds;
 #ifdef DEBUG
@@ -2950,7 +2950,6 @@ dec_term(ErtsDistExternal *edep,
         next     = ctx->u.dc.next;
         ep       = ctx->u.dc.ep;
 	factory  = &ctx->u.dc.factory;
-	maps_list = ctx->u.dc.maps_list;
 
         if (ctx->state != B2TDecode) {
             int n_limit = reds;
@@ -3026,15 +3025,18 @@ dec_term(ErtsDistExternal *edep,
             }
         }
 	PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK);
+        WSTACK_CHANGE_ALLOCATOR(flat_maps, ERTS_ALC_T_SAVED_ESTACK);
 	if (ctx->u.dc.hamt_array.pstart) {
 	    PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array);
 	}
+	if (ctx->u.dc.flat_maps.wstart) {
+	    WSTACK_RESTORE(flat_maps, &ctx->u.dc.flat_maps);
+	}
     }
     else {
         reds = ERTS_SWORD_MAX;
         next = objp;
         *next = (Eterm) (UWord) NULL;
-	maps_list = NULL;
     }
     hp = factory->hp;
 
@@ -3595,14 +3597,8 @@ dec_term_atom_common:
                      * vptr, last word for values
                      */
 
-                    /*
-                     * Use thing_word to link through decoded maps.
-                     * The list of maps is for later validation.
-                     */
-
-                    mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list);
-                    maps_list      = (Eterm *) mp;
-
+                    WSTACK_PUSH(flat_maps, (UWord)mp);
+                    mp->thing_word = MAP_HEADER_FLATMAP;
                     mp->size       = size;
                     mp->keys       = keys;
                     *objp          = make_flatmap(mp);
@@ -3851,7 +3847,9 @@ dec_term_atom_common:
                     ctx->u.dc.ep = ep;
                     ctx->u.dc.next = next;
                     ctx->u.dc.factory.hp = hp;
-		    ctx->u.dc.maps_list = maps_list;
+		    if (!WSTACK_ISEMPTY(flat_maps)) {
+			WSTACK_SAVE(flat_maps, &ctx->u.dc.flat_maps);
+		    }
 		    if (!PSTACK_IS_EMPTY(hamt_array)) {
 			PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array);
 		    }
@@ -3865,18 +3863,6 @@ dec_term_atom_common:
         }
     }
 
-    /* Iterate through all the maps and check for validity and sort keys
-     * - done here for when we know it is complete.
-     */
-
-    while (maps_list) {
-	next  = (Eterm *)(EXPAND_POINTER(*maps_list));
-	*maps_list = MAP_HEADER_FLATMAP;
-	if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list))
-	    goto error;
-	maps_list  = next;
-    }
-
     ASSERT(hp <= factory->hp_end
            || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp)));
     factory->hp = hp;
@@ -3885,20 +3871,31 @@ dec_term_atom_common:
      */
 
     if (!PSTACK_IS_EMPTY(hamt_array)) {
-	do {
-	    struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array);
-
-	    *hamt->objp = erts_hashmap_from_array(factory,
-						  hamt->leaf_array,
-						  hamt->size,
-						  1);
-	    if (is_non_value(*hamt->objp))
-		goto error_hamt;
-
-	    (void) PSTACK_POP(hamt_array);
-	} while (!PSTACK_IS_EMPTY(hamt_array));
-	PSTACK_DESTROY(hamt_array);
+        do {
+            struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array);
+
+            *hamt->objp = erts_hashmap_from_array(factory,
+                                                  hamt->leaf_array,
+                                                  hamt->size,
+                                                  1);
+            if (is_non_value(*hamt->objp))
+                goto error_hamt;
+
+            (void) PSTACK_POP(hamt_array);
+        } while (!PSTACK_IS_EMPTY(hamt_array));
+        PSTACK_DESTROY(hamt_array);
+    }
+
+    /* Iterate through all the (flat)maps and check for validity and sort keys
+     * - done here for when we know it is complete.
+     */
+
+    while(!WSTACK_ISEMPTY(flat_maps)) {
+        next = (Eterm *)WSTACK_POP(flat_maps);
+        if (!erts_validate_and_sort_flatmap((flatmap_t*)next))
+            goto error;
     }
+    WSTACK_DESTROY(flat_maps);
 
     ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL);
 
@@ -3924,6 +3921,7 @@ error_hamt:
 	ctx->state = B2TDecodeFail;
 	ctx->reds = reds;
     }
+    WSTACK_DESTROY(flat_maps);
         
     return NULL;
 }
-- 
cgit v1.2.3


From 5cf556df2e178ce78d0c4197a73654d593f58f92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 24 Nov 2015 16:05:04 +0100
Subject: Fix seq_trace refactoring bug

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 5907dd4567..a8b7211793 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11666,7 +11666,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 
     if ((state & ERTS_PSFLG_TRAP_EXIT)
 	&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
-        if (have_seqtrace(token))
+        if (have_seqtrace(token) && token_update)
 	    seq_trace_update_send(token_update);
 	if (is_value(exit_tuple))
 	    send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token);
-- 
cgit v1.2.3


From bc08768e5bad4baa2eaec4c6798ead9ded0a8889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 24 Nov 2015 18:35:11 +0100
Subject: Fix seq_trace token copy size

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a8b7211793..a814462668 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11547,7 +11547,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 #ifdef SHCOPY_SEND
         INITIALIZE_SHCOPY(info);
         term_size = copy_shared_calculate(exit_term, &info);
-	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
+	mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp);
         mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp);
         DESTROY_SHCOPY(info);
 #else
-- 
cgit v1.2.3


From ef45d2c9f874354b17c2aca96de7b3306a9eb943 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 8 Dec 2014 20:37:59 +0100
Subject: erts: Add enif_getenv

to read OS environment variables in a safe and portable way.
---
 erts/emulator/beam/erl_nif.c           | 1 +
 erts/emulator/beam/erl_nif.h           | 3 ++-
 erts/emulator/beam/erl_nif_api_funcs.h | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index add4a66f90..d7a2076d85 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1173,6 +1173,7 @@ ErlNifTid enif_thread_self(void) { return erl_drv_thread_self(); }
 int enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2) { return erl_drv_equal_tids(tid1,tid2); }
 void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); }
 int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); }
+int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); }
 
 int enif_fprintf(void* filep, const char* format, ...) 
 { 
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 7d880126f8..4cbfd8360b 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -48,9 +48,10 @@
 **           add ErlNifEntry options
 **           add ErlNifFunc flags
 ** 2.8: 18.0 add enif_has_pending_exception
+** 2.9: 18.2 enif_getenv
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 8
+#define ERL_NIF_MINOR_VERSION 9
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 2f2180e1aa..08b9afc6af 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -159,6 +159,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
 ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason));
+ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* value_size));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -310,6 +311,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif)
 #  define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception)
 #  define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception)
+#  define enif_getenv ERL_NIF_API_FUNC_MACRO(enif_getenv)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From 41e0c6e584d392ed0d5fbbc51a84418c4f7abcf5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 9 Dec 2014 11:53:16 +0100
Subject: erts: Refactor alloc_SUITE to use NIFs instead of drivers

---
 erts/emulator/beam/erl_nif.h | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 4cbfd8360b..5e39343e9b 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -232,6 +232,7 @@ typedef enum {
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
 typedef struct {
 #  include "erl_nif_api_funcs.h"
+   void* erts_alc_test;
 } TWinDynNifCallbacks;
 extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  undef ERL_NIF_API_FUNC_DECL
-- 
cgit v1.2.3


From 5c9557a814ac993a72d7b045ee374745d6dcab79 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 2 Oct 2015 18:14:28 +0200
Subject: erts: Improve alloc_SUITE:migration test

In the quest to improve code coverage in cpool_fetch
---
 erts/emulator/beam/erl_alloc.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 0aa45acd82..c3f4fe5a63 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3642,6 +3642,10 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 	    *(void**)a3 = orig_destroying_mbc;
 	    return offset;
 	}
+	case 0xf17: {
+	    ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
+	    return ts->allctr[0]->largest_mbc_size;
+	}
 	default:
 	    break;
 	}
-- 
cgit v1.2.3


From cb62c989e59f0ec8556f9f1d4e9a45b8d9ee32f6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 27 Nov 2015 17:06:39 +0100
Subject: erts: Fix rare case of faulty heap fragment deallocation

after major GC.

Can only be caused by distributed messages containing large maps.
Bad map hashing will increase the risk.
---
 erts/emulator/beam/erl_gc.c | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d2604f1595..2f21111a2e 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1237,6 +1237,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
     Uint n;
     Uint new_sz;
+    int done;
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
@@ -1440,6 +1441,8 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 
     *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
 
+    remove_message_buffers(p);
+
     {
 	ErlMessage *msgp;
 
@@ -1458,15 +1461,21 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	}
     }
 
-    adjust_after_fullsweep(p, need, objv, nobj);
-
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-    remove_message_buffers(p);
+    if (MBUF(p)) {
+        /* This is a very rare case when distributed messages copied above
+         * contained maps so big they did not fit on the heap causing the
+         * factory to create heap frags.
+         * Solution: Trigger a minor gc (without tenuring)
+         */
+        HIGH_WATER(p) = HEAP_START(p);
+        done = 0;
+    } else {
+        adjust_after_fullsweep(p, need, objv, nobj);
+        done = 1;
+    }
 
     ErtsGcQuickSanityCheck(p);
-    return 1;			/* We are done. */
+    return done;
 }
 
 static void
@@ -1955,7 +1964,18 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
     if (p->dictionary != NULL) {
 	disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used);
     }
-    disallow_heap_frag_ref_in_heap(p);
+    /* OTP-18: Actually we do allow references from heap to heap fragments now.
+       This can happen when doing "binary_to_term" with a "fat" map contained
+       in another term. A "fat" map is a hashmap with higher heap demand than
+       first estimated by "binary_to_term" causing the factory to allocate
+       additional heap (fragments) for the hashmap tree nodes.
+       Run map_SUITE:t_gc_rare_map_overflow to provoke this.
+
+       Inverted references like this does not matter however. The copy done
+       below by move_one_area() with move markers in the fragments and the
+       sweeping done later by the GC should make everything ok in the end.
+    */
+    /***disallow_heap_frag_ref_in_heap(p);***/
 #endif
 
     /*
-- 
cgit v1.2.3


From 99e6213c0f0cebaa01f8310b6950a814cf4b21ee Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Nov 2015 18:04:09 +0100
Subject: erts: Remove dead code erts_hash_merge

---
 erts/emulator/beam/hash.c | 50 -----------------------------------------------
 erts/emulator/beam/hash.h |  2 --
 2 files changed, 52 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index e0fde337f2..636aafc108 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -280,56 +280,6 @@ void* hash_put(Hash* h, void* tmpl)
     return (void*) b;
 }
 
-static void
-hash_insert_entry(Hash* h, HashBucket* entry)
-{
-    HashValue hval = entry->hvalue;
-    int ix = hval % h->size;
-    HashBucket* b = h->bucket[ix];
-
-    while (b != (HashBucket*) 0) {
-	if ((b->hvalue == hval) && (h->fun.cmp((void*)entry, (void*)b) == 0)) {
-	    abort();		/* Should not happen */
-	}
-	b = b->next;
-    }
-
-    if (h->bucket[ix] == NULL)
-	h->used++;
-
-    entry->next = h->bucket[ix];
-    h->bucket[ix] = entry;
-
-    if (h->used > h->size80percent)  /* rehash at 80% */
-	rehash(h, 1);
-}
-
-
-/*
- * Move all entries in src into dst; empty src.
- * Entries in src must not exist in dst.
- */
-void
-erts_hash_merge(Hash* src, Hash* dst)
-{
-    int limit = src->size;
-    HashBucket** bucket = src->bucket;
-    int i;
-
-    src->used = 0;
-    for (i = 0; i < limit; i++) {
-	HashBucket* b = bucket[i];
-	HashBucket* next;
-
-	bucket[i] = NULL;
-	while (b) {
-	    next = b->next;
-	    hash_insert_entry(dst, b);
-	    b = next;
-	}
-    }
-}
-
 /*
 ** Erase hash entry return template if erased
 ** return 0 if not erased
diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h
index 87fdb360e3..c9e75d7acf 100644
--- a/erts/emulator/beam/hash.h
+++ b/erts/emulator/beam/hash.h
@@ -93,6 +93,4 @@ void* hash_erase(Hash*, void*);
 void* hash_remove(Hash*, void*);
 void  hash_foreach(Hash*, void (*func)(void *, void *), void *);
 
-void erts_hash_merge(Hash* src, Hash* dst);
-
 #endif
-- 
cgit v1.2.3


From 4b4c3d525a06309b7e23c7c3ccf7a358bd0f33f3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Nov 2015 18:12:32 +0100
Subject: erts: Redesign grow/shrink thresholds of hash.c

1. Use load factor as indicator, not used buckets.
   Used buckets is a bad indicator as it makes the situation even
   worse with a bad hash function.
   Set grow_threshold to load factor of 160% as it roughly corresponds
   to the old 80% used bucket limit.

2. Never shrink table below initial size.
---
 erts/emulator/beam/hash.c | 60 ++++++++++++++++++++++++-----------------------
 erts/emulator/beam/hash.h |  9 +++----
 2 files changed, 36 insertions(+), 33 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index 636aafc108..9d247db039 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -66,6 +66,7 @@ void hash_get_info(HashInfo *hi, Hash *h)
     int i;
     int max_depth = 0;
     int objects = 0;
+    int used = 0;
 
     for (i = 0; i < size; i++) {
 	int depth = 0;
@@ -76,14 +77,18 @@ void hash_get_info(HashInfo *hi, Hash *h)
 	    depth++;
 	    b = b->next;
 	}
-	if (depth > max_depth)
-	    max_depth = depth;
+        if (depth) {
+            used++;
+            if (depth > max_depth)
+                max_depth = depth;
+        }
     }
+    ASSERT(objects == h->nobjs);
 
     hi->name  = h->name;
     hi->size  = h->size;
-    hi->used  = h->used;
-    hi->objs  = objects;
+    hi->used  = used;
+    hi->objs  = h->nobjs;
     hi->depth = max_depth;
 }
 
@@ -119,6 +124,15 @@ hash_table_sz(Hash *h)
 }
 
 
+static ERTS_INLINE void set_thresholds(Hash* h)
+{
+    h->grow_threshold = (8*h->size)/5;   /* grow at 160% load */
+    if (h->size_ix > h->min_size_ix)
+        h->shrink_threshold = h->size / 5;  /* shrink at 20% load */
+    else
+        h->shrink_threshold = -1;  /* never shrink below inital size */
+}
+
 /*
 ** init a pre allocated or static hash structure
 ** and allocate buckets.
@@ -145,10 +159,10 @@ Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions
     h->name = name;
     h->fun = fun;
     h->size = size;
-    h->size20percent = h->size/5;
-    h->size80percent = (4*h->size)/5;
-    h->ix = ix;
-    h->used = 0;
+    h->size_ix = ix;
+    h->min_size_ix = ix;
+    h->nobjs = 0;
+    set_thresholds(h);
     return h;
 }
 
@@ -199,32 +213,26 @@ static void rehash(Hash* h, int grow)
     int i;
 
     if (grow) {
-	if ((h_size_table[h->ix+1]) == -1)
+	if ((h_size_table[h->size_ix+1]) == -1)
 	    return;
-	h->ix++;
+	h->size_ix++;
     }
     else {
-	if (h->ix == 0)
+	if (h->size_ix == 0)
 	    return;
-	h->ix--;
+	h->size_ix--;
     }
-    h->size = h_size_table[h->ix];
-    h->size20percent = h->size/5;
-    h->size80percent = (4*h->size)/5;
+    h->size = h_size_table[h->size_ix];
     sz = h->size*sizeof(HashBucket*);
 
     new_bucket = (HashBucket **) erts_alloc(h->type, sz);
     sys_memzero(new_bucket, sz);
 
-    h->used = 0;
-
     for (i = 0; i < old_size; i++) {
 	HashBucket* b = h->bucket[i];
 	while (b != (HashBucket*) 0) {
 	    HashBucket* b_next = b->next;
 	    int ix = b->hvalue % h->size;
-	    if (new_bucket[ix] == NULL)
-		h->used++;
 	    b->next = new_bucket[ix];
 	    new_bucket[ix] = b;
 	    b = b_next;
@@ -232,6 +240,7 @@ static void rehash(Hash* h, int grow)
     }
     erts_free(h->type, (void *) h->bucket);
     h->bucket = new_bucket;
+    set_thresholds(h);
 }
 
 /*
@@ -268,14 +277,11 @@ void* hash_put(Hash* h, void* tmpl)
     }
     b = (HashBucket*) h->fun.alloc(tmpl);
 
-    if (h->bucket[ix] == NULL)
-	h->used++;
-
     b->hvalue = hval;
     b->next = h->bucket[ix];
     h->bucket[ix] = b;
 
-    if (h->used > h->size80percent)  /* rehash at 80% */
+    if (++h->nobjs > h->grow_threshold)
 	rehash(h, 1);
     return (void*) b;
 }
@@ -298,9 +304,7 @@ void* hash_erase(Hash* h, void* tmpl)
 	    else
 		h->bucket[ix] = b->next;
 	    h->fun.free((void*)b);
-	    if (h->bucket[ix] == NULL)
-		h->used--;
-	    if (h->used < h->size20percent)  /* rehash at 20% */
+	    if (--h->nobjs < h->shrink_threshold)
 		rehash(h, 0);
 	    return tmpl;
 	}
@@ -331,9 +335,7 @@ hash_remove(Hash *h, void *tmpl)
 		prev->next = b->next;
 	    else
 		h->bucket[ix] = b->next;
-	    if (h->bucket[ix] == NULL)
-		h->used--;
-	    if (h->used < h->size20percent)  /* rehash at 20% */
+	    if (--h->nobjs < h->shrink_threshold)
 		rehash(h, 0);
 	    return (void *) b;
 	}
diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h
index c9e75d7acf..dc7e9c10c5 100644
--- a/erts/emulator/beam/hash.h
+++ b/erts/emulator/beam/hash.h
@@ -72,10 +72,11 @@ typedef struct hash
     ErtsAlcType_t type;
     char* name;          /* Table name (static string, for debugging) */
     int size;		 /* Number of slots */
-    int size20percent;   /* 20 percent of number of slots */
-    int size80percent;   /* 80 percent of number of slots */
-    int ix;              /* Size index in size table */
-    int used;		 /* Number of slots used */
+    int shrink_threshold;
+    int grow_threshold;
+    int size_ix;         /* Size index in size table */
+    int min_size_ix;     /* Never shrink table smaller than this */
+    int nobjs;		 /* Number of objects in table */
     HashBucket** bucket; /* Vector of bucket pointers (objects) */
 } Hash;
 
-- 
cgit v1.2.3


From 2afa7580910050b9b087a188215f27553cc0aba3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 30 Nov 2015 12:08:15 +0100
Subject: erts: Remove unused include files from hash.c

Note that hash.c is quite "clean" from Erlang stuff
and is used by erl_child_setup as well.
---
 erts/emulator/beam/hash.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index 9d247db039..75d091d11c 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -27,8 +27,6 @@
 #endif
 
 #include "sys.h"
-#include "erl_vm.h"
-#include "global.h"
 #include "hash.h"
 
 /*
-- 
cgit v1.2.3


From 6fe04e5b66ce020651b2c150d26b9af8488b7dcd Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 30 Nov 2015 18:50:31 +0100
Subject: Remove ERTS_PRINT_INVALID from erts_print()

ERTS_PRINT_INVALID prevented file descriptor 0 to be used which
could cause an empty crash dump.
---
 erts/emulator/beam/sys.h   | 1 -
 erts/emulator/beam/utils.c | 3 ---
 2 files changed, 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index bb871b05ba..ec94e3a596 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -634,7 +634,6 @@ Uint erts_sys_misc_mem_sz(void);
 /* Io constants to erts_print and erts_putc */
 #define ERTS_PRINT_STDERR	(2)
 #define ERTS_PRINT_STDOUT	(1)
-#define ERTS_PRINT_INVALID	(0) /* Don't want to use 0 since CBUF was 0 */
 #define ERTS_PRINT_FILE		(-1)
 #define ERTS_PRINT_SBUF		(-2)
 #define ERTS_PRINT_SNBUF	(-3)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index e9d7c91ac9..5286391746 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -399,9 +399,6 @@ erts_print(int to, void *arg, char *format, ...)
 	case ERTS_PRINT_DSBUF:
 	    res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list);
 	    break;
-	case ERTS_PRINT_INVALID:
-	    res = -EINVAL;
-	    break;
 	default:
 	    res = erts_vfdprintf((int) to, format, arg_list);
 	    break;
-- 
cgit v1.2.3


From 33299ece737c635910e358d7e09dd8af6bce1a5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Wed, 2 Dec 2015 14:30:56 +0100
Subject: beam: Fix overflow bug in i_bs_add_jId

The test whether the result would fit in a smallnum could overflow into
a negative number that would fit a smallnum. A test that reproduces the
issue was added to bs_construct_SUITE.
---
 erts/emulator/beam/beam_emu.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 38def5d89f..73292885ce 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4069,7 +4069,7 @@ do {								\
 	     tmp_arg1 += Arg1;
 
 	 store_bs_add_result:
-	     if (MY_IS_SSMALL((Sint) tmp_arg1)) {
+	     if (tmp_arg1 <= MAX_SMALL) {
 		 tmp_arg1 = make_small(tmp_arg1);
 	     } else {
 		 /*
-- 
cgit v1.2.3


From 6b912d7fdd5fca46dee840b0bfa6a92915c0a093 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 3 Dec 2015 14:39:31 +0100
Subject: erts: Fix bug in heap_factory_undo for FACTORY_HEAP_FRAGS mode

Make sure a heap fragment is not deallocated before all off_heap
terms have been cleared.

The fix assumes/asserts that the off_heap-lists of all additional
heap fragments are empty.

I think this bug has been harmless as hashmap nodes, which is only
ones (?) that can cause a factory to produce more heap,
are not linked in off_heap-list.
---
 erts/emulator/beam/erl_message.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ef52823287..2a703fb102 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1372,13 +1372,16 @@ void erts_factory_undo(ErtsHeapFactory* factory)
         break;
 
     case FACTORY_HEAP_FRAGS:
+	erts_cleanup_offheap(factory->off_heap);
+	factory->off_heap->first = NULL;
+
         bp = factory->heap_frags;
         do {
             ErlHeapFragment* next_bp = bp->next;
 
-            erts_cleanup_offheap(&bp->off_heap);
+            ASSERT(bp->off_heap.first == NULL);
             ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
-                           ERTS_HEAP_FRAG_SIZE(bp->size));
+                           ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
             bp = next_bp;
         }while (bp != NULL);
 	break;
-- 
cgit v1.2.3


From 59aae51046e79b1c8a3cf2473fd4618e9d617ac6 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 30 Nov 2015 18:50:31 +0100
Subject: Remove ERTS_PRINT_INVALID from erts_print()

ERTS_PRINT_INVALID prevented file descriptor 0 to be used which
could cause an empty crash dump.
---
 erts/emulator/beam/sys.h   | 1 -
 erts/emulator/beam/utils.c | 3 ---
 2 files changed, 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c29d4b3777..f8ab0df082 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -577,7 +577,6 @@ Uint erts_sys_misc_mem_sz(void);
 /* Io constants to erts_print and erts_putc */
 #define ERTS_PRINT_STDERR	(2)
 #define ERTS_PRINT_STDOUT	(1)
-#define ERTS_PRINT_INVALID	(0) /* Don't want to use 0 since CBUF was 0 */
 #define ERTS_PRINT_FILE		(-1)
 #define ERTS_PRINT_SBUF		(-2)
 #define ERTS_PRINT_SNBUF	(-3)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index a8bbdb9354..ebc549fb31 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -343,9 +343,6 @@ erts_print(int to, void *arg, char *format, ...)
 	case ERTS_PRINT_DSBUF:
 	    res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list);
 	    break;
-	case ERTS_PRINT_INVALID:
-	    res = -EINVAL;
-	    break;
 	default:
 	    res = erts_vfdprintf((int) to, format, arg_list);
 	    break;
-- 
cgit v1.2.3


From f67a7375e19734c3f7d6947b0dcf608d0fe1c8fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 1 Dec 2015 18:02:54 +0100
Subject: erts: Use internal hash for process dictionaries

---
 erts/emulator/beam/erl_process_dict.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 8606371bdf..81469e8716 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -53,11 +53,11 @@
 /* Hash utility macros  */
 #define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition)
 
-#define MAKE_HASH(Term) 				\
-((is_small(Term)) ? unsigned_val(Term) :		\
- ((is_atom(Term)) ? 					\
-  (atom_tab(atom_val(term))->slot.bucket.hvalue) :	\
-  make_hash2(Term)))
+#define MAKE_HASH(Term)                                \
+    ((is_small(Term)) ? unsigned_val(Term) :           \
+     ((is_atom(Term)) ?                                \
+      (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \
+      make_internal_hash(Term)))
 
 #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
 
-- 
cgit v1.2.3


From c97f3332aeddf039ee2207196229b9ff07047c72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 2 Dec 2015 13:17:28 +0100
Subject: erts: Add i_get_hash instruction

Calculate hashvalue in load-time for constant process dictionary gets.
---
 erts/emulator/beam/beam_emu.c         | 10 +++++++
 erts/emulator/beam/beam_load.c        | 47 +++++++++++++++++++++++++++++
 erts/emulator/beam/erl_process_dict.c | 56 +++++++++++++++++++++++------------
 erts/emulator/beam/erl_process_dict.h |  1 +
 erts/emulator/beam/ops.tab            |  3 +-
 5 files changed, 97 insertions(+), 20 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 38def5d89f..d39cd9d8ea 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3735,6 +3735,16 @@ do {								\
 	StoreBifResult(1, result);
     }
 
+ OpCase(i_get_hash_cId):
+    {
+	Eterm arg;
+	Eterm result;
+
+	GetArg1(0, arg);
+	result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg);
+	StoreBifResult(2, result);
+    }
+
     {
 	Eterm case_end_val;
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index b70e5b9a2d..636672217b 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4258,6 +4258,53 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
     return op;
 }
 
+static int
+hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx)
+{
+    switch (Key.type) {
+    case TAG_a:
+	*hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue;
+	return 1;
+    case TAG_i:
+	*hx = Key.val;
+	return 1;
+    case TAG_n:
+	*hx = make_internal_hash(NIL);
+	return 1;
+    case TAG_q:
+	*hx = make_internal_hash(stp->literals[Key.val].term);
+	return 1;
+    default:
+	return 0;
+    }
+}
+
+
+static GenOp*
+gen_get(LoaderState* stp, GenOpArg Src, GenOpArg Dst)
+{
+    GenOp* op;
+    Uint32 hx = 0;
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    if (hash_internal_genop_arg(stp, Src, &hx)) {
+	op->arity = 3;
+	op->op = genop_i_get_hash_3;
+        op->a[0] = Src;
+	op->a[1].type = TAG_u;
+	op->a[1].val = (BeamInstr) hx;
+	op->a[2] = Dst;
+    } else {
+	op->arity = 2;
+	op->op = genop_i_get_2;
+        op->a[0] = Src;
+	op->a[1] = Dst;
+    }
+    return op;
+}
+
+
 static GenOp*
 gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
 		     GenOpArg Size, GenOpArg* Rest)
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 81469e8716..e497267b63 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -61,6 +61,9 @@
 
 #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
 
+#define pd_hash_value(Pdict, Key) \
+    pd_hash_value_to_ix(Pdict, MAKE_HASH((Key)))
+
 /* Memory allocation macros */
 #define PD_ALLOC(Sz)				\
      erts_alloc(ERTS_ALC_T_PROC_DICT, (Sz))
@@ -82,6 +85,7 @@
  */
 static void pd_hash_erase(Process *p, Eterm id, Eterm *ret);
 static void pd_hash_erase_all(Process *p);
+static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id);
 static Eterm pd_hash_get_keys(Process *p, Eterm value);
 static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd);
 static Eterm pd_hash_get_all(Process *p, ProcDict *pd);
@@ -93,7 +97,7 @@ static void grow(Process *p);
 static void array_shrink(ProcDict **ppd, unsigned int need);
 static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term);
 
-static unsigned int pd_hash_value(ProcDict *pdict, Eterm term);
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx);
 static unsigned int next_array_size(unsigned int need);
 
 /*
@@ -390,40 +394,55 @@ static void pd_hash_erase_all(Process *p)
     }
 }
 
+Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id)
+{
+    unsigned int hval;
+    ProcDict *pd = p->dictionary;
+
+    if (pd == NULL)
+	return am_undefined;
+    hval = pd_hash_value_to_ix(pd, hx);
+    return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id);
+}
+
 Eterm erts_pd_hash_get(Process *p, Eterm id) 
 {
     unsigned int hval;
-    Eterm tmp;
     ProcDict *pd = p->dictionary;
 
     if (pd == NULL)
 	return am_undefined;
     hval = pd_hash_value(pd, id);
-    tmp = ARRAY_GET(pd, hval);
-    if (is_boxed(tmp)) {	/* Tuple */
-	ASSERT(is_tuple(tmp));
-	if (EQ(tuple_val(tmp)[1], id)) {
-	    return tuple_val(tmp)[2];
+    return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id);
+}
+
+Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id)
+{
+    if (is_boxed(bucket)) {     /* Tuple */
+	ASSERT(is_tuple(bucket));
+	if (EQ(tuple_val(bucket)[1], id)) {
+	    return tuple_val(bucket)[2];
 	}
-    } else if (is_list(tmp)) {
-	for (; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) {
+    } else if (is_list(bucket)) {
+	for (; bucket != NIL && !EQ(tuple_val(TCAR(bucket))[1], id); bucket = TCDR(bucket)) {
 	    ;
 	}
-	if (tmp != NIL) {
-	    return tuple_val(TCAR(tmp))[2];
+	if (bucket != NIL) {
+	    return tuple_val(TCAR(bucket))[2];
 	}
-    } else if (is_not_nil(tmp)) {
+    } else if (is_not_nil(bucket)) {
 #ifdef DEBUG
 	erts_fprintf(stderr,
 		     "Process dictionary for process %T is broken, trying to "
 		     "display term found in line %d:\n"
-		     "%T\n", p->common.id, __LINE__, tmp);
+		     "%T\n", p->common.id, __LINE__, bucket);
 #endif
 	erl_exit(1, "Damaged process dictionary found during get/1.");
     }
     return am_undefined;
 }
 
+
 #define PD_GET_TKEY(Dst,Src)                            \
 do {                                                    \
     ASSERT(is_tuple((Src)));                            \
@@ -932,17 +951,16 @@ static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term)
 ** Basic utilities
 */
 
-static unsigned int pd_hash_value(ProcDict *pdict, Eterm term) 
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx) 
 { 
-    Uint hash, high;
-
-    hash = MAKE_HASH(term);
-    high = hash % (pdict->homeSize*2);
+    Uint high;
+    high = hx % (pdict->homeSize*2);
     if (high >= HASH_RANGE(pdict))
-	return hash % pdict->homeSize;
+	return hx % pdict->homeSize;
     return high;
 }
 
+
 static unsigned int next_array_size(unsigned int need)
 {
     static unsigned int tab[] =
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index cc53800eb5..9aa21b7c38 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -39,5 +39,6 @@ void erts_deep_dictionary_dump(int to, void *to_arg,
 Eterm erts_dictionary_copy(struct process *p, ProcDict *pd);
 
 Eterm erts_pd_hash_get(struct process *p, Eterm id);
+Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id);
 
 #endif
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 1d32e72247..86ae189c27 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1118,7 +1118,7 @@ call_bif e
 bif0 u$bif:erlang:self/0 Dst=d => self Dst
 bif0 u$bif:erlang:node/0 Dst=d => node Dst
 
-bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst
+bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => gen_get(Src, Dst)
 
 bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst)
 
@@ -1130,6 +1130,7 @@ bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst
 bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst
 bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst
 
+i_get_hash c I d
 i_get s d
 
 %macro: self Self
-- 
cgit v1.2.3


From ce8279d6a48d41f9de577825844f499bb3084b96 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 3 Dec 2015 15:34:14 +0100
Subject: erts: Fix bug for remote control message containing fat maps

that could cause the static factory to overflow

Fix: Introduce a new factory mode FACTORY_TMP
---
 erts/emulator/beam/dist.c        | 10 +++-------
 erts/emulator/beam/erl_message.c | 29 +++++++++++++++++++++++++++--
 erts/emulator/beam/erl_message.h |  4 +++-
 3 files changed, 33 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 4846133aa6..170690ca89 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1153,7 +1153,6 @@ int erts_net_message(Port *prt,
     Process* rp;
     DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE);
     Eterm* ctl = ctl_default;
-    ErlOffHeap off_heap;
     ErtsHeapFactory factory;
     Eterm* hp;
     Sint type;
@@ -1168,9 +1167,6 @@ int erts_net_message(Port *prt,
 #endif
 
     UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
-    /* Thanks to Luke Gorrie */
-    off_heap.first = NULL;
-    off_heap.overhead = 0;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
@@ -1231,7 +1227,7 @@ int erts_net_message(Port *prt,
     }
     hp = ctl;
 
-    erts_factory_static_init(&factory, ctl, ctl_len, &off_heap);
+    erts_factory_tmp_init(&factory, ctl, ctl_len, ERTS_ALC_T_DCTRL_BUF);
     arg = erts_decode_dist_ext(&factory, &ede);
     if (is_non_value(arg)) {
 #ifdef ERTS_DIST_MSG_DBG
@@ -1719,7 +1715,7 @@ int erts_net_message(Port *prt,
 	goto invalid_message;
     }
 
-    erts_cleanup_offheap(&off_heap);
+    erts_factory_close(&factory);
     if (ctl != ctl_default) {
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
@@ -1734,7 +1730,7 @@ int erts_net_message(Port *prt,
     }
  data_error:
     PURIFY_MSG("data error");
-    erts_cleanup_offheap(&off_heap);
+    erts_factory_close(&factory);
     if (ctl != ctl_default) {
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 2a703fb102..fa6b2fc613 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1174,6 +1174,9 @@ void erts_factory_message_init(ErtsHeapFactory* factory,
     ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end);
 }
 
+/* One static sized heap that must suffice.
+   No extra heap fragments will be allocated.
+*/
 void erts_factory_static_init(ErtsHeapFactory* factory,
 			     Eterm* hp,
 			     Uint size,
@@ -1188,6 +1191,23 @@ void erts_factory_static_init(ErtsHeapFactory* factory,
     factory->off_heap_saved.overhead = factory->off_heap->overhead;
 }
 
+/* A temporary heap with default buffer allocated/freed by client.
+ * factory_close is same as factory_undo
+ */
+void erts_factory_tmp_init(ErtsHeapFactory* factory, Eterm* hp, Uint size,
+			   Uint32 atype)
+{
+    factory->mode     = FACTORY_TMP;
+    factory->hp_start = hp;
+    factory->hp       = hp;
+    factory->hp_end   = hp + size;
+    factory->heap_frags = NULL;
+    factory->off_heap_saved.first    = NULL;
+    factory->off_heap_saved.overhead = 0;
+    factory->off_heap = &factory->off_heap_saved;
+    factory->alloc_type = atype;
+}
+
 /* When we know the term is an immediate and need no heap.
 */
 void erts_factory_dummy_init(ErtsHeapFactory* factory)
@@ -1231,6 +1251,7 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
 	return;
 
     case FACTORY_HEAP_FRAGS:
+    case FACTORY_TMP:
 	bp = factory->heap_frags;
 
         if (bp) {
@@ -1280,6 +1301,9 @@ void erts_factory_close(ErtsHeapFactory* factory)
 	    bp->used_size = factory->hp - bp->mem;
         }
 	break;
+    case FACTORY_TMP:
+	erts_factory_undo(factory);
+	break;
     case FACTORY_STATIC: break;
     case FACTORY_CLOSED: break;
     default:
@@ -1371,19 +1395,20 @@ void erts_factory_undo(ErtsHeapFactory* factory)
         }
         break;
 
+    case FACTORY_TMP:
     case FACTORY_HEAP_FRAGS:
 	erts_cleanup_offheap(factory->off_heap);
 	factory->off_heap->first = NULL;
 
         bp = factory->heap_frags;
-        do {
+        while (bp != NULL) {
             ErlHeapFragment* next_bp = bp->next;
 
             ASSERT(bp->off_heap.first == NULL);
             ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
                            ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
             bp = next_bp;
-        }while (bp != NULL);
+        }
 	break;
 
     case FACTORY_CLOSED: break;
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index fbdf3fb0e2..92ba3e571c 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -58,7 +58,8 @@ typedef struct {
         FACTORY_CLOSED = 0,
         FACTORY_HALLOC,
         FACTORY_HEAP_FRAGS,
-        FACTORY_STATIC
+        FACTORY_STATIC,
+        FACTORY_TMP
     } mode;
     Process* p;
     Eterm* hp_start;
@@ -75,6 +76,7 @@ void erts_factory_proc_init(ErtsHeapFactory*, Process*);
 void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size);
 void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*);
 void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*);
+void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype);
 void erts_factory_dummy_init(ErtsHeapFactory*);
 
 Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
-- 
cgit v1.2.3


From a2b28094081f1b185a31b33e3c1bcb377d6761bb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 3 Dec 2015 18:50:20 +0100
Subject: erts: Tweak hashmap heap size estimation

1. Change order between mul and div to not lose too much
   in integer divisions.

2. Fix estimation in DEBUG to really be an *under* estimation.
---
 erts/emulator/beam/erl_map.h | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index c391de3f11..4d9d74bc37 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -195,14 +195,17 @@ typedef struct hashmap_head_s {
    [one cons cell + one list term in parent node] per key
    [one header + one boxed term in parent node] per inner node
    [one header + one size word] for root node
+   Observed average number of nodes per key is about 0.35.
 */
-#define HASHMAP_HEAP_SIZE(KEYS,NODES) ((KEYS)*3 + (NODES)*2)
+#define HASHMAP_WORDS_PER_KEY 3
+#define HASHMAP_WORDS_PER_NODE 2
 #ifdef DEBUG
-#  define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (KEYS)
+#  define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \
+    (HASHMAP_WORDS_PER_NODE * (KEYS) * 3/10)   /* slightly under estimated */
 #else
-#  define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (2*(KEYS)/5)
+#  define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \
+    (HASHMAP_WORDS_PER_NODE * (KEYS) * 4/10)   /* slightly over estimated */
 #endif
 #define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \
-        HASHMAP_HEAP_SIZE(KEYS,HASHMAP_ESTIMATED_NODE_COUNT(KEYS))
-
+        ((KEYS)*HASHMAP_WORDS_PER_KEY + HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS))
 #endif
-- 
cgit v1.2.3


From 2ae91c3ade0538500ff4dbda29ad539e595f64df Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Oct 2015 20:10:46 +0200
Subject: erts: Change erts_internal:map_type/1 into term_type/1

to support other terms, not just maps
---
 erts/emulator/beam/atom.h    |  1 +
 erts/emulator/beam/bif.tab   |  2 +-
 erts/emulator/beam/erl_map.c | 25 +++++++++++++++++--------
 3 files changed, 19 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index ead56c83d8..2c002ca92f 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -129,6 +129,7 @@ typedef enum {
   (erts_is_atom_utf8_bytes((byte *) LSTR, sizeof(LSTR) - 1, (TERM)))
 #define ERTS_DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 #define ERTS_INIT_AM(S) AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
+#define ERTS_MAKE_AM(Str) am_atom_put(Str, sizeof(Str) - 1)
 
 int atom_table_size(void);	/* number of elements */
 int atom_table_sz(void);	/* table size in bytes, excluding stored objects */
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index c49a3ff313..07d4702b92 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -167,7 +167,7 @@ bif erts_internal:request_system_task/3
 bif erts_internal:check_process_code/2
 
 bif erts_internal:map_to_tuple_keys/1
-bif erts_internal:map_type/1
+bif erts_internal:term_type/1
 bif erts_internal:map_hashmap_children/1
 
 bif erts_internal:time_unit/0
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 29b3024644..ac10b9a3e3 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2698,29 +2698,38 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
 }
 
 /*
- * erts_internal:map_type/1
+ * erts_internal:term_type/1
  *
  * Used in erts_debug:size/1
  */
 
-BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
-    DECL_AM(hashmap);
-    DECL_AM(hashmap_node);
-    DECL_AM(flatmap);
+BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) {
     if (is_map(BIF_ARG_1)) {
         Eterm hdr = *(boxed_val(BIF_ARG_1));
         ASSERT(is_header(hdr));
         switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
             case HAMT_SUBTAG_HEAD_FLATMAP:
-                BIF_RET(AM_flatmap);
+                BIF_RET(ERTS_MAKE_AM("flatmap"));
             case HAMT_SUBTAG_HEAD_ARRAY:
             case HAMT_SUBTAG_HEAD_BITMAP:
-                BIF_RET(AM_hashmap);
+                BIF_RET(ERTS_MAKE_AM("hashmap"));
             case HAMT_SUBTAG_NODE_BITMAP:
-                BIF_RET(AM_hashmap_node);
+                BIF_RET(ERTS_MAKE_AM("hashmap_node"));
             default:
                 erl_exit(1, "bad header");
         }
+    } else if (is_immed(BIF_ARG_1)) {
+        if (is_small(BIF_ARG_1)) {
+            BIF_RET(ERTS_MAKE_AM("small"));
+        } else if (is_ifloat(BIF_ARG_1)) {
+            BIF_RET(ERTS_MAKE_AM("ifloat"));
+        }
+    } else if (is_boxed(BIF_ARG_1)) {
+        if (is_big(BIF_ARG_1)) {
+            BIF_RET(ERTS_MAKE_AM("big"));
+        } else if (is_hfloat(BIF_ARG_1)) {
+            BIF_RET(ERTS_MAKE_AM("hfloat"));
+        }
     }
     BIF_P->fvalue = BIF_ARG_1;
     BIF_ERROR(BIF_P, BADMAP);
-- 
cgit v1.2.3


From f6eacb9981b40604a031c9d61967b0c3a3588bdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 13 Nov 2015 17:41:36 +0100
Subject: erts: Let term_type/1 encompass all types

---
 erts/emulator/beam/erl_map.c | 101 +++++++++++++++++++++++++++++++------------
 1 file changed, 74 insertions(+), 27 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ac10b9a3e3..d0ffb11e79 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2704,35 +2704,82 @@ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) {
  */
 
 BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) {
-    if (is_map(BIF_ARG_1)) {
-        Eterm hdr = *(boxed_val(BIF_ARG_1));
-        ASSERT(is_header(hdr));
-        switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
-            case HAMT_SUBTAG_HEAD_FLATMAP:
-                BIF_RET(ERTS_MAKE_AM("flatmap"));
-            case HAMT_SUBTAG_HEAD_ARRAY:
-            case HAMT_SUBTAG_HEAD_BITMAP:
-                BIF_RET(ERTS_MAKE_AM("hashmap"));
-            case HAMT_SUBTAG_NODE_BITMAP:
-                BIF_RET(ERTS_MAKE_AM("hashmap_node"));
-            default:
-                erl_exit(1, "bad header");
-        }
-    } else if (is_immed(BIF_ARG_1)) {
-        if (is_small(BIF_ARG_1)) {
-            BIF_RET(ERTS_MAKE_AM("small"));
-        } else if (is_ifloat(BIF_ARG_1)) {
-            BIF_RET(ERTS_MAKE_AM("ifloat"));
-        }
-    } else if (is_boxed(BIF_ARG_1)) {
-        if (is_big(BIF_ARG_1)) {
-            BIF_RET(ERTS_MAKE_AM("big"));
-        } else if (is_hfloat(BIF_ARG_1)) {
-            BIF_RET(ERTS_MAKE_AM("hfloat"));
+    Eterm obj = BIF_ARG_1;
+    switch (primary_tag(obj)) {
+        case TAG_PRIMARY_LIST:
+            BIF_RET(ERTS_MAKE_AM("list"));
+        case TAG_PRIMARY_BOXED: {
+            Eterm hdr = *boxed_val(obj);
+            ASSERT(is_header(hdr));
+            switch (hdr & _TAG_HEADER_MASK) {
+                case ARITYVAL_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("tuple"));
+                case EXPORT_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("export"));
+                case FUN_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("fun"));
+                case MAP_SUBTAG:
+                    switch (MAP_HEADER_TYPE(hdr)) {
+                        case MAP_HEADER_TAG_FLATMAP_HEAD :
+                            BIF_RET(ERTS_MAKE_AM("flatmap"));
+                        case MAP_HEADER_TAG_HAMT_HEAD_BITMAP :
+                        case MAP_HEADER_TAG_HAMT_HEAD_ARRAY :
+                            BIF_RET(ERTS_MAKE_AM("hashmap"));
+                        case MAP_HEADER_TAG_HAMT_NODE_BITMAP :
+                            BIF_RET(ERTS_MAKE_AM("hashmap_node"));
+                        default:
+                            erl_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr));
+                    }
+                case REFC_BINARY_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("refc_binary"));
+                case HEAP_BINARY_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("heap_binary"));
+                case SUB_BINARY_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("sub_binary"));
+                case BIN_MATCHSTATE_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("matchstate"));
+                case POS_BIG_SUBTAG:
+                case NEG_BIG_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("bignum"));
+                case REF_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("reference"));
+                case EXTERNAL_REF_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("external_reference"));
+                case EXTERNAL_PID_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("external_pid"));
+                case EXTERNAL_PORT_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("external_port"));
+                case FLOAT_SUBTAG:
+                    BIF_RET(ERTS_MAKE_AM("hfloat"));
+                default:
+                    erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr);
+            }
         }
+        case TAG_PRIMARY_IMMED1:
+            switch (obj & _TAG_IMMED1_MASK) {
+                case _TAG_IMMED1_SMALL:
+                    BIF_RET(ERTS_MAKE_AM("fixnum"));
+                case _TAG_IMMED1_PID:
+                    BIF_RET(ERTS_MAKE_AM("pid"));
+                case _TAG_IMMED1_PORT:
+                    BIF_RET(ERTS_MAKE_AM("port"));
+                case _TAG_IMMED1_IMMED2:
+                    switch (obj & _TAG_IMMED2_MASK) {
+                        case _TAG_IMMED2_ATOM:
+                            BIF_RET(ERTS_MAKE_AM("atom"));
+                        case _TAG_IMMED2_CATCH:
+                            BIF_RET(ERTS_MAKE_AM("catch"));
+                        case _TAG_IMMED2_NIL:
+                            BIF_RET(ERTS_MAKE_AM("nil"));
+                        default:
+                            erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
+                    }
+                default:
+                    erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
+            }
+        default:
+            erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj);
     }
-    BIF_P->fvalue = BIF_ARG_1;
-    BIF_ERROR(BIF_P, BADMAP);
 }
 
 /*
-- 
cgit v1.2.3


From bc40e21224e871c9bcbb5c2f27854c308bc4fe01 Mon Sep 17 00:00:00 2001
From: Luca Favatella <lucafavatella@users.noreply.github.com>
Date: Sun, 6 Dec 2015 21:37:42 +0000
Subject: Fix compilation with `--enable-vm-probes`

... broken by 3ac08f9b.

Compilation error:
```
beam/erl_message.c: In function 'erts_send_message':
beam/erl_message.c:753:56: error: macro "copy_struct" requires 4 arguments, but only 3 given
   utag = copy_struct(DT_UTAG(sender), dt_utag_size, ohp);
                                                        ^
beam/erl_message.c:753:10: error: 'copy_struct' undeclared (first use in this function)
   utag = copy_struct(DT_UTAG(sender), dt_utag_size, ohp);
          ^
beam/erl_message.c:753:10: note: each undeclared identifier is reported only once for each function it appears in
```
---
 erts/emulator/beam/erl_message.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 1811651a58..d593108f8e 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -750,7 +750,7 @@ erts_send_message(Process* sender,
 	    if (is_immed(DT_UTAG(sender)))
 		utag = DT_UTAG(sender);
 	    else
-		utag = copy_struct(DT_UTAG(sender), dt_utag_size, ohp);
+		utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp);
 #ifdef DTRACE_TAG_HARDDEBUG
 	    erts_fprintf(stderr,
 			 "Dtrace -> (%T) Spreading tag (%T) with "
-- 
cgit v1.2.3


From ac529b0326496e52f3289464f9410001bc3bde6d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 16 Nov 2015 11:21:49 +0100
Subject: Fix memory leaks

---
 erts/emulator/beam/erl_gc.c      |  8 +++++++-
 erts/emulator/beam/erl_process.c | 17 +++++++++++------
 2 files changed, 18 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6cb37752bc..3bb1f601aa 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -734,6 +734,12 @@ erts_garbage_collect_hibernate(Process* p)
 			    p->arg_reg,
 			    p->arity);
 
+    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
+		   (p->abandoned_heap
+		    ? p->abandoned_heap
+		    : p->heap),
+		   p->heap_sz * sizeof(Eterm));
+
     p->heap = heap;
     p->high_water = htop;
     p->htop = htop;
@@ -1435,7 +1441,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 		   (p->abandoned_heap
 		    ? p->abandoned_heap
 		    : HEAP_START(p)),
-		   (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
+		   p->heap_sz * sizeof(Eterm));
     p->abandoned_heap = NULL;
     p->flags &= ~F_ABANDONED_HEAP_USE;
     HEAP_START(p) = n_heap;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9acce8acb6..ebb4d323e6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11259,6 +11259,7 @@ erts_cleanup_empty_process(Process* p)
 static void
 delete_process(Process* p)
 {
+    Eterm *heap;
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
 
     /* Cleanup psd */
@@ -11283,16 +11284,17 @@ delete_process(Process* p)
      * Release heaps. Clobber contents in DEBUG build.
      */
 
-
-#ifdef DEBUG
-    sys_memset(p->heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm));
-#endif
-
 #ifdef HIPE
     hipe_delete_process(&p->hipe);
 #endif
 
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) p->heap, p->heap_sz*sizeof(Eterm));
+    heap = p->abandoned_heap ? p->abandoned_heap : p->heap;
+
+#ifdef DEBUG
+    sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm));
+#endif
+
+    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm));
     if (p->old_heap != NULL) {
 
 #ifdef DEBUG
@@ -11311,6 +11313,9 @@ delete_process(Process* p)
 	free_message_buffer(p->mbuf);
     }
 
+    if (p->msg_frag)
+	erts_cleanup_messages(p->msg_frag);
+
     erts_erase_dicts(p);
 
     /* free all pending messages */
-- 
cgit v1.2.3


From ccbb5ba4d0198f3ba21b83ae1aca3fe52a7f2c80 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 17 Nov 2015 15:21:56 +0100
Subject: Remove unused variable

---
 erts/emulator/beam/erl_process.c | 4 +---
 erts/emulator/beam/erl_process.h | 1 -
 2 files changed, 1 insertion(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ebb4d323e6..ebe9361b8d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -335,7 +335,6 @@ static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info;
 
 static Uint last_reductions;
 static Uint last_exact_reductions;
-Uint erts_default_process_flags;
 Eterm erts_system_monitor;
 Eterm erts_system_monitor_long_gc;
 Uint erts_system_monitor_long_schedule;
@@ -682,7 +681,6 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab)
 
     last_reductions = 0;
     last_exact_reductions = 0;
-    erts_default_process_flags = 0;
 }
 
 void
@@ -10737,7 +10735,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 		   Eterm args,	/* Arguments for function (must be well-formed list). */
 		   ErlSpawnOpts* so) /* Options for spawn. */
 {
-    Uint flags = erts_default_process_flags;
+    Uint flags = 0;
     ErtsRunQueue *rq = NULL;
     Process *p;
     Sint arity;			/* Number of arguments. */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 884027f482..b80aa12872 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1248,7 +1248,6 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
 Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
 #endif
 
-extern Uint erts_default_process_flags;
 extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
 /* If any of the erts_system_monitor_* variables are set (enabled),
 ** erts_system_monitor must be != NIL, to allow testing on just
-- 
cgit v1.2.3


From cc3cc025f43a00281d2f836e12c2e1627cd50145 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 17 Nov 2015 15:25:22 +0100
Subject: Fix process_info(_, off_heap_message_queue)

---
 erts/emulator/beam/erl_bif_info.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 1eb106a551..3fe8aed50e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1497,7 +1497,7 @@ process_info_aux(Process *BIF_P,
     }
 
     case am_off_heap_message_queue:
-	res = BIF_P->flags & F_OFF_HEAP_MSGQ ? am_true : am_false;
+	res = rp->flags & F_OFF_HEAP_MSGQ ? am_true : am_false;
 	hp = HAlloc(BIF_P, 3);
 	break;
 
-- 
cgit v1.2.3


From a5dd6499d53ed596e2f8aa17ee35ff87cd32fe60 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 19 Nov 2015 17:07:06 +0100
Subject: Distinguish between GC disabled by BIFs and other disabled GC

Processes remember heap fragments that are known to be fully
live due to creation in a just called BIF that yields in the
live_hf_end field. This field must not be used if we have not
disabled GC in a BIF. F_DELAY_GC has been introduced in order
to distinguish between to two different scenarios.

- F_DISABLE_GC should *only* be used by BIFs. This when
  the BIF needs to yield while preventig a GC.
- F_DELAY_GC should only be used when GC is temporarily
  disabled while the process is scheduled. A process must
  not be scheduled out while F_DELAY_GC is set.
---
 erts/emulator/beam/beam_emu.c    | 14 +++++++-------
 erts/emulator/beam/erl_gc.c      | 15 ++++++++++++---
 erts/emulator/beam/erl_process.c |  2 ++
 erts/emulator/beam/erl_process.h | 14 +++++++++++++-
 4 files changed, 34 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 208a16dfd0..95a22e6c54 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1843,8 +1843,8 @@ void process_main(void)
       * in the queue. This since messages with data outside
       * the heap will be corrupted by a GC.
       */
-     ASSERT(!(c_p->flags & F_DISABLE_GC));
-     c_p->flags |= F_DISABLE_GC;
+     ASSERT(!(c_p->flags & F_DELAY_GC));
+     c_p->flags |= F_DELAY_GC;
 
  loop_rec__:
      PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -1858,7 +1858,7 @@ void process_main(void)
 	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	     SWAPOUT;
-	     c_p->flags &= ~F_DISABLE_GC;
+	     c_p->flags &= ~F_DELAY_GC;
 	     goto do_schedule; /* Will be rescheduled for exit */
 	 }
 	 ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
@@ -1868,7 +1868,7 @@ void process_main(void)
 	 else
 #endif
 	 {
-	     c_p->flags &= ~F_DISABLE_GC;
+	     c_p->flags &= ~F_DELAY_GC;
 	     SET_I((BeamInstr *) Arg(0));
 	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
 	 }
@@ -1997,7 +1997,7 @@ void process_main(void)
      CANCEL_TIMER(c_p);
 
      erts_save_message_in_proc(c_p, msgp);
-     c_p->flags &= ~F_DISABLE_GC;
+     c_p->flags &= ~F_DELAY_GC;
 
      if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) {
 	 /*
@@ -2019,7 +2019,7 @@ void process_main(void)
      */
  OpCase(loop_rec_end_f): {
 
-     ASSERT(c_p->flags & F_DISABLE_GC);
+     ASSERT(c_p->flags & F_DELAY_GC);
 
      SET_I((BeamInstr *) Arg(0));
      SAVE_MESSAGE(c_p);
@@ -2028,7 +2028,7 @@ void process_main(void)
 	 goto loop_rec__;
      }
 
-     c_p->flags &= ~F_DISABLE_GC;
+     c_p->flags &= ~F_DELAY_GC;
      c_p->i = I;
      SWAPOUT;
      c_p->arity = 0;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 3bb1f601aa..3784367195 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -440,8 +440,15 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
 
     ERTS_HOLE_CHECK(p);
 
-    if (p->live_hf_end == ERTS_INVALID_HFRAG_PTR)
+    if ((p->flags & F_DISABLE_GC)
+	&& p->live_hf_end == ERTS_INVALID_HFRAG_PTR) {
+	/*
+	 * A BIF yielded with disabled GC. Remember
+	 * heap fragments created by the BIF until we
+	 * do next GC.
+	 */
 	p->live_hf_end = live_hf_end;
+    }
 
     if (need == 0)
 	return 1;
@@ -564,10 +571,12 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
-    if (p->flags & F_DISABLE_GC)
+    if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
 	return delay_garbage_collection(p, live_hf_end, need);
 
-    if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
+    if (p->abandoned_heap)
+	live_hf_end = ERTS_INVALID_HFRAG_PTR;
+    else if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
 	live_hf_end = p->live_hf_end;
 
     esdp = erts_get_scheduler_data();
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ebe9361b8d..d17355207b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9262,6 +9262,8 @@ Process *schedule(Process *p, int calls)
     } else {
     sched_out_proc:
 
+	ASSERT(!(p->flags & F_DELAY_GC));
+
 #ifdef ERTS_SMP
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 	esdp = p->scheduler_data;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b80aa12872..41b367435d 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1288,10 +1288,22 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
-#define F_DISABLE_GC         (1 << 11) /* Disable GC */
+#define F_DISABLE_GC         (1 << 11) /* Disable GC (see below) */
 #define F_OFF_HEAP_MSGQ      (1 << 12) /* Off heap msg queue */
 #define F_OFF_HEAP_MSGQ_CHNG (1 << 13) /* Off heap msg queue changing */
 #define F_ABANDONED_HEAP_USE (1 << 14) /* Have usage of abandoned heap */
+#define F_DELAY_GC           (1 << 15) /* Similar to disable GC (see below) */
+
+/*
+ * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
+ * GC of the process, but it is important to use the right
+ * one:
+ * - F_DISABLE_GC should *only* be used by BIFs. This when
+ *   the BIF needs to yield while preventig a GC.
+ * - F_DELAY_GC should only be used when GC is temporarily
+ *   disabled while the process is scheduled. A process must
+ *   not be scheduled out while F_DELAY_GC is set.
+ */
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
-- 
cgit v1.2.3


From 1cd97bc82d042bc713473932af7d6061065f6527 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 20 Nov 2015 16:57:32 +0100
Subject: Always use literal_alloc

---
 erts/emulator/beam/erl_alloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 019aa0f16c..5574a3b713 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -290,7 +290,7 @@ static void
 set_default_literal_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
-    ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
+    ip->enable			= 1;
     ip->thr_spec		= 0;
     ip->atype			= BESTFIT;
     ip->init.bf.ao		= 1;
-- 
cgit v1.2.3


From 19c4689eea86f26c5af9b8f712c227ce4f62310b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Nov 2015 15:57:55 +0100
Subject: Replace off_heap_message_queue option with message_queue_data option

The message_queue_data option can have the values
 - off_heap
 - on_heap
 - mixed
---
 erts/emulator/beam/atom.names     |   5 +-
 erts/emulator/beam/bif.c          |  30 +++---
 erts/emulator/beam/erl_bif_info.c |  37 ++++++--
 erts/emulator/beam/erl_gc.c       | 195 ++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_init.c     |  21 ++--
 erts/emulator/beam/erl_message.c  | 160 +++++++++++++++++++++++--------
 erts/emulator/beam/erl_message.h  |   2 +-
 erts/emulator/beam/erl_process.c  |   4 +
 erts/emulator/beam/erl_process.h  |  21 ++--
 9 files changed, 388 insertions(+), 87 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index ea04495574..7424e47ec3 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -350,6 +350,7 @@ atom memory_internal
 atom memory_types
 atom message
 atom message_binary
+atom message_queue_data
 atom message_queue_len
 atom messages
 atom merge_trap
@@ -361,6 +362,7 @@ atom min_heap_size
 atom min_bin_vheap_size
 atom minor_version
 atom Minus='-'
+atom mixed
 atom module
 atom module_info
 atom monitored_by
@@ -423,11 +425,12 @@ atom notify
 atom notsup
 atom nouse_stdio
 atom objects
-atom off_heap_message_queue
+atom off_heap
 atom offset
 atom ok
 atom old_heap_block_size
 atom old_heap_size
+atom on_heap
 atom on_load
 atom open
 atom open_error
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index f0340540cb..410a6cecac 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -910,13 +910,22 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
 		    so.priority = PRIORITY_LOW;
 		else
 		    goto error;
-	    } else if (arg == am_off_heap_message_queue) {
-		if (val == am_true)
-		    so.flags |= SPO_OFF_HEAP_MSGQ;
-		else if (val == am_false)
+	    } else if (arg == am_message_queue_data) {
+		switch (val) {
+		case am_mixed:
+		    so.flags &= ~(SPO_OFF_HEAP_MSGQ|SPO_ON_HEAP_MSGQ);
+		    break;
+		case am_on_heap:
 		    so.flags &= ~SPO_OFF_HEAP_MSGQ;
-		else
+		    so.flags |= SPO_ON_HEAP_MSGQ;
+		    break;
+		case am_off_heap:
+		    so.flags &= ~SPO_ON_HEAP_MSGQ;
+		    so.flags |= SPO_OFF_HEAP_MSGQ;
+		    break;
+		default:
 		    goto error;
+		}
 	    } else if (arg == am_min_heap_size && is_small(val)) {
 		Sint min_heap_size = signed_val(val);
 		if (min_heap_size < 0) {
@@ -1695,15 +1704,10 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        }
        BIF_RET(old_value);
    }
-   else if (BIF_ARG_1 == am_off_heap_message_queue) {
-       int enable;
-       if (BIF_ARG_2 == am_true)
-	   enable = 1;
-       else if (BIF_ARG_2 == am_false)
-	   enable = 0;
-       else
+   else if (BIF_ARG_1 == am_message_queue_data) {
+       old_value = erts_change_message_queue_management(BIF_P, BIF_ARG_2);
+       if (is_non_value(old_value))
 	   goto error;
-       old_value = erts_change_off_heap_message_queue_state(BIF_P, enable);
        BIF_RET(old_value);
    }
    else if (BIF_ARG_1 == am_sensitive) {
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 3fe8aed50e..bb75b8abb6 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -589,7 +589,7 @@ static Eterm pi_args[] = {
     am_min_bin_vheap_size,
     am_current_location,
     am_current_stacktrace,
-    am_off_heap_message_queue
+    am_message_queue_data
 };
 
 #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
@@ -637,7 +637,7 @@ pi_arg2ix(Eterm arg)
     case am_min_bin_vheap_size:			return 28;
     case am_current_location:			return 29;
     case am_current_stacktrace:			return 30;
-    case am_off_heap_message_queue:		return 31;
+    case am_message_queue_data:			return 31;
     default:					return -1;
     }
 }
@@ -1496,8 +1496,22 @@ process_info_aux(Process *BIF_P,
 	break;
     }
 
-    case am_off_heap_message_queue:
-	res = rp->flags & F_OFF_HEAP_MSGQ ? am_true : am_false;
+    case am_message_queue_data:
+	switch (rp->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) {
+	case F_OFF_HEAP_MSGQ:
+	    res = am_off_heap;
+	    break;
+	case F_ON_HEAP_MSGQ:
+	    res = am_on_heap;
+	    break;
+	case 0:
+	    res = am_mixed;
+	    break;
+	default:
+	    res = am_error;
+	    ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
+	    break;
+	}
 	hp = HAlloc(BIF_P, 3);
 	break;
 
@@ -2662,9 +2676,18 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(am_true);
     }
 #endif
-    else if (BIF_ARG_1 == am_off_heap_message_queue) {
-	BIF_RET(erts_default_spo_flags & SPO_OFF_HEAP_MSGQ
-		? am_true : am_false);
+    else if (BIF_ARG_1 == am_message_queue_data) {
+	switch (erts_default_spo_flags & (SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ)) {
+	case SPO_OFF_HEAP_MSGQ:
+	    BIF_RET(am_off_heap);
+	case SPO_ON_HEAP_MSGQ:
+	    BIF_RET(am_on_heap);
+	case 0:
+	    BIF_RET(am_mixed);
+	default:
+	    ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
+	    BIF_RET(am_error);
+	}
     }
     else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) {
 	Uint  sz;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 3784367195..b63567e563 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -145,6 +145,7 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
 			   Eterm* objv, int nobj);
 static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
 static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size);
+static void move_msgq_to_heap(Process *p);
 
 static void init_gc_info(ErtsGCInfo *gcip);
 
@@ -520,6 +521,14 @@ young_gen_usage(Process *p)
     Eterm *aheap;
 
     hsz = p->mbuf_sz;
+
+    if (p->flags & F_ON_HEAP_MSGQ) {
+	ErtsMessage *mp;
+	for (mp = p->msg.first; mp; mp = mp->next)
+	    if (mp->data.attached)
+		hsz += erts_msg_attached_data_size(mp);
+    }
+
     aheap = p->abandoned_heap;
     if (!aheap)
 	hsz += p->htop - p->heap;
@@ -1040,10 +1049,13 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
         do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm),
 		 new_sz, objv, nobj);
 
+	if (p->flags & F_ON_HEAP_MSGQ)
+	    move_msgq_to_heap(p);
+
 	new_mature = p->old_htop - prev_old_htop;
 
 	size_after = new_mature;
-        size_after += HEAP_TOP(p) - HEAP_START(p);
+        size_after += HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz;
         *recl += (size_before - size_after);
 
 	ErtsGcQuickSanityCheck(p);
@@ -1461,9 +1473,14 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     HIGH_WATER(p) = HEAP_TOP(p);
 
+    remove_message_buffers(p);
+
+    if (p->flags & F_ON_HEAP_MSGQ)
+	move_msgq_to_heap(p);
+
     ErtsGcQuickSanityCheck(p);
 
-    size_after = HEAP_TOP(p) - HEAP_START(p);
+    size_after = HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz;
     *recl += size_before - size_after;
 
     adjusted = adjust_after_fullsweep(p, need, objv, nobj);
@@ -1471,8 +1488,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 #ifdef HARDDEBUG
     disallow_heap_frag_ref_in_heap(p);
 #endif
-    remove_message_buffers(p);
-
     ErtsGcQuickSanityCheck(p);
 
     return gc_cost(size_after, adjusted ? size_after : 0);
@@ -2000,6 +2015,173 @@ collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
     return n_htop;
 }
 
+static ERTS_INLINE void
+copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
+	      ErlHeapFragment *bp, Eterm *refs, int nrefs)
+{
+    Uint sz;
+    int i;
+    Sint offs;
+    struct erl_off_heap_header* oh;
+    Eterm *fhp, *hp;
+
+    OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+    sz = bp->used_size;
+
+    fhp = bp->mem;
+    hp = *hpp;
+    offs = hp - fhp;
+
+    oh = NULL;
+    while (sz--) {
+	Uint cpy_sz;
+	Eterm val = *fhp++;
+
+	switch (primary_tag(val)) {
+	case TAG_PRIMARY_IMMED1:
+	    *hp++ = val;
+	    break;
+	case TAG_PRIMARY_LIST:
+	case TAG_PRIMARY_BOXED:
+	    *hp++ = offset_ptr(val, offs);
+	    break;
+	case TAG_PRIMARY_HEADER:
+	    *hp++ = val;
+	    switch (val & _HEADER_SUBTAG_MASK) {
+	    case ARITYVAL_SUBTAG:
+		break;
+	    case REFC_BINARY_SUBTAG:
+	    case FUN_SUBTAG:
+	    case EXTERNAL_PID_SUBTAG:
+	    case EXTERNAL_PORT_SUBTAG:
+	    case EXTERNAL_REF_SUBTAG:
+		oh = (struct erl_off_heap_header*) (hp-1);
+		cpy_sz = thing_arityval(val);
+		goto cpy_words;
+	    default:
+		cpy_sz = header_arity(val);
+
+	    cpy_words:
+		ASSERT(sz >= cpy_sz);
+		sz -= cpy_sz;
+		while (cpy_sz >= 8) {
+		    cpy_sz -= 8;
+		    *hp++ = *fhp++;
+		    *hp++ = *fhp++;
+		    *hp++ = *fhp++;
+		    *hp++ = *fhp++;
+		    *hp++ = *fhp++;
+		    *hp++ = *fhp++;
+		    *hp++ = *fhp++;
+		    *hp++ = *fhp++;
+		}
+		switch (cpy_sz) {
+		case 7: *hp++ = *fhp++;
+		case 6: *hp++ = *fhp++;
+		case 5: *hp++ = *fhp++;
+		case 4: *hp++ = *fhp++;
+		case 3: *hp++ = *fhp++;
+		case 2: *hp++ = *fhp++;
+		case 1: *hp++ = *fhp++;
+		default: break;
+		}
+		if (oh) {
+		    /* Add to offheap list */
+		    oh->next = off_heap->first;
+		    off_heap->first = oh;
+		    ASSERT(*hpp <= (Eterm*)oh);
+		    ASSERT(hp > (Eterm*)oh);
+		    oh = NULL;
+		}
+		break;
+	    }
+	    break;
+	}
+    }
+
+    ASSERT(bp->used_size == hp - *hpp);
+    *hpp = hp;
+
+    for (i = 0; i < nrefs; i++) {
+	if (is_not_immed(refs[i]))
+	    refs[i] = offset_ptr(refs[i], offs);
+    }
+    bp->off_heap.first = NULL;
+}
+
+static void
+move_msgq_to_heap(Process *p)
+{
+    ErtsMessage **mpp = &p->msg.first;
+
+    while (*mpp) {
+	ErtsMessage *mp = *mpp;
+
+	if (mp->data.attached) {
+	    ErlHeapFragment *bp;
+	    ErtsHeapFactory factory;
+
+	    erts_factory_proc_prealloc_init(&factory, p,
+					    erts_msg_attached_data_size(mp));
+
+	    if (is_non_value(ERL_MESSAGE_TERM(mp))) {
+		if (mp->data.dist_ext) {
+		    ASSERT(mp->data.dist_ext->heap_size >= 0);
+		    if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) {
+			bp = erts_dist_ext_trailer(mp->data.dist_ext);
+			ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
+							    bp->used_size,
+							    &factory.hp,
+							    factory.off_heap);
+			erts_cleanup_offheap(&bp->off_heap);
+		    }
+		    ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory,
+								mp->data.dist_ext);
+		    erts_free_dist_ext_copy(mp->data.dist_ext);
+		    mp->data.dist_ext = NULL;
+		}
+	    }
+	    else {
+
+		if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG)
+		    bp = &mp->hfrag;
+		else
+		    bp = mp->data.heap_frag;
+
+		if (bp->next)
+		    erts_move_multi_frags(&factory.hp, factory.off_heap, bp,
+					  mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0);
+		else
+		    copy_one_frag(&factory.hp, factory.off_heap, bp,
+				  mp->m, ERL_MESSAGE_REF_ARRAY_SZ);
+
+		if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
+		    mp->data.heap_frag = NULL;
+		    free_message_buffer(bp);
+		}
+		else {
+		    ErtsMessage *tmp = erts_alloc_message(0, NULL);
+		    sys_memcpy((void *) tmp->m, (void *) mp->m,
+			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); 
+		    tmp->next = mp->next;
+		    if (p->msg.save == &mp->next)
+			p->msg.save = &tmp->next;
+		    if (p->msg.last == &mp->next)
+			p->msg.last = &tmp->next;
+		    *mpp = tmp;
+		    mp->next = NULL;
+		    erts_cleanup_messages(mp);
+		    mp = tmp;
+		}
+	    }
+
+	    erts_factory_close(&factory);
+	}
+
+	mpp = &(*mpp)->next;
+    }
+}
+
 static Uint
 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 {
@@ -2089,9 +2271,8 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
     case F_OFF_HEAP_MSGQ_CHNG:
     case 0: {
 	/*
-	 * Off heap message queue disabled, i.e. we may
-	 * have references from the message queue to the
-	 * heap...
+	 * We do not have off heap message queue enabled, i.e. we
+	 * need to add message queue to rootset...
 	 */
 	ErtsMessage *mp;
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f396a0a156..628915eb23 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -630,7 +630,8 @@ void erts_usage(void)
 
     erts_fprintf(stderr, "-W<i|w|e>      set error logger warnings mapping,\n");
     erts_fprintf(stderr, "               see error_logger documentation for details\n");
-    erts_fprintf(stderr, "-xohmq  bool   set default off_heap_message_queue flag for processes\n");
+    erts_fprintf(stderr, "-xmqd  val     set default message queue data flag for processes,\n");
+    erts_fprintf(stderr, "               valid values are: off_heap | on_heap | mixed\n");
     erts_fprintf(stderr, "-zdbbl size    set the distribution buffer busy limit in kilobytes\n");
     erts_fprintf(stderr, "               valid range is [1-%d]\n", INT_MAX/1024);
     erts_fprintf(stderr, "-zdntgc time   set delayed node table gc in seconds\n");
@@ -2018,15 +2019,21 @@ erl_start(int argc, char **argv)
 
 	case 'x': {
 	    char *sub_param = argv[i]+2;
-	    if (has_prefix("ohmq", sub_param)) {
-		arg = get_arg(sub_param+4, argv[i+1], &i);
-		if (sys_strcmp(arg, "true") == 0)
-		    erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ;
-		else if (sys_strcmp(arg, "false") == 0)
+	    if (has_prefix("mqd", sub_param)) {
+		arg = get_arg(sub_param+3, argv[i+1], &i);
+		if (sys_strcmp(arg, "mixed") == 0)
+		    erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ);
+		else if (sys_strcmp(arg, "on_heap") == 0) {
 		    erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ;
+		    erts_default_spo_flags |= SPO_ON_HEAP_MSGQ;
+		}
+		else if (sys_strcmp(arg, "off_heap") == 0) {
+		    erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ;
+		    erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ;
+		}
 		else {
 		    erts_fprintf(stderr,
-				 "Invalid off_heap_message_queue flag: %s\n", arg);
+				 "Invalid message_queue_data flag: %s\n", arg);
 		    erts_usage();
 		}
 	    } else {
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 79739501a8..797212450c 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -629,9 +629,24 @@ erts_try_alloc_message_on_heap(Process *pp,
 #endif
     else {
     in_message_fragment:
-
-	mp = erts_alloc_message(sz, hpp);
-	*ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
+	if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) {
+	    mp = erts_alloc_message(sz, hpp);
+	    *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
+	}
+	else {
+	    mp = erts_alloc_message(0, NULL);
+	    if (!sz) {
+		*hpp = NULL;
+		*ohpp = NULL;
+	    }
+	    else {
+		ErlHeapFragment *bp;
+		bp = new_message_buffer(sz);
+		*hpp = &bp->mem[0];
+		mp->data.heap_frag = bp;
+		*ohpp = &bp->off_heap;
+	    }
+	}
 	*on_heap_p = 0;
     }
 
@@ -976,12 +991,12 @@ erts_complete_off_heap_message_queue_change(Process *c_p)
     ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ);
 
     /*
-     * This job was first initiated when the process changed
-     * "off heap message queue" state from false to true. Since
-     * then ERTS_PSFLG_OFF_HEAP_MSGQ has been set. However, the
-     * state change might have been changed again (multiple times)
-     * since then. Check users last requested state (the flag
-     * F_OFF_HEAP_MSGQ), and make the state consistent with that.
+     * This job was first initiated when the process changed to off heap
+     * message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ
+     * has been set. However, the management state might have been changed
+     * again (multiple times) since then. Check users last requested state
+     * (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state
+     * consistent with that.
      */
 
     if (!(c_p->flags & F_OFF_HEAP_MSGQ))
@@ -1022,8 +1037,9 @@ change_off_heap_msgq(void *vcohmq)
 }
 
 Eterm
-erts_change_off_heap_message_queue_state(Process *c_p, int enable)
+erts_change_message_queue_management(Process *c_p, Eterm new_state)
 {
+    Eterm res;
 
 #ifdef DEBUG
     if (c_p->flags & F_OFF_HEAP_MSGQ) {
@@ -1042,57 +1058,117 @@ erts_change_off_heap_message_queue_state(Process *c_p, int enable)
     }
 #endif
 
-    if (c_p->flags & F_OFF_HEAP_MSGQ) {
-	/* Off heap message queue is enabled */
+    switch (c_p->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) {
 
-	if (!enable) {
+    case F_OFF_HEAP_MSGQ:
+	res = am_off_heap;
+
+	switch (new_state) {
+	case am_off_heap:
+	    break;
+	case am_on_heap:
+	    c_p->flags |= F_ON_HEAP_MSGQ;
+	    erts_smp_atomic32_read_bor_nob(&c_p->state,
+					   ERTS_PSFLG_ON_HEAP_MSGQ);
+	    /* fall through */
+	case am_mixed:
 	    c_p->flags &= ~F_OFF_HEAP_MSGQ;
 	    /*
 	     * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ
-	     * if a change is ongoing. It will be adjusted when the
-	     * change completes...
+	     * if a off heap change is ongoing. It will be adjusted
+	     * when the change completes...
 	     */
 	    if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
 		/* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */
 		erts_smp_atomic32_read_band_nob(&c_p->state,
 						~ERTS_PSFLG_OFF_HEAP_MSGQ);
 	    }
+	    break;
+	default:
+	    res = THE_NON_VALUE; /* badarg */
+	    break;
 	}
+	break;
+
+    case F_ON_HEAP_MSGQ:
+	res = am_on_heap;
 
-	return am_true; /* Old state */
+	switch (new_state) {
+	case am_on_heap:
+	    break;
+	case am_mixed:
+	    c_p->flags &= ~F_ON_HEAP_MSGQ;
+	    erts_smp_atomic32_read_band_nob(&c_p->state,
+					    ~ERTS_PSFLG_ON_HEAP_MSGQ);
+	    break;
+	case am_off_heap:
+	    c_p->flags &= ~F_ON_HEAP_MSGQ;
+	    erts_smp_atomic32_read_band_nob(&c_p->state,
+					    ~ERTS_PSFLG_ON_HEAP_MSGQ);
+	    goto change_to_off_heap;
+	default:
+	    res = THE_NON_VALUE; /* badarg */
+	    break;
+	}
+	break;
+
+    case 0:
+	res = am_mixed;
+
+	switch (new_state) {
+	case am_mixed:
+	    break;
+	case am_on_heap:
+	    c_p->flags |= F_ON_HEAP_MSGQ;
+	    erts_smp_atomic32_read_bor_nob(&c_p->state,
+					   ERTS_PSFLG_ON_HEAP_MSGQ);
+	    break;
+	case am_off_heap:
+	    goto change_to_off_heap;
+	default:
+	    res = THE_NON_VALUE; /* badarg */
+	    break;
+	}
+	break;
+
+    default:
+	res = am_error;
+	ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
+	break;
     }
 
-    /* Off heap message queue is disabled */
+    return res;
+
+change_to_off_heap:
 
-    if (enable) {
-	c_p->flags |= F_OFF_HEAP_MSGQ;
+    c_p->flags |= F_OFF_HEAP_MSGQ;
+
+    /*
+     * We do not have to schedule a change if
+     * we have an ongoing off heap change...
+     */
+    if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
+	ErtsChangeOffHeapMessageQueue *cohmq;
 	/*
-	 * We do not have to schedule a change if
-	 * we have an ongoing change...
+	 * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait
+	 * thread progress before completing the change in
+	 * order to ensure that all senders observe that
+	 * messages should be passed off heap. When the
+	 * change has completed, GC does not need to inspect
+	 * the message queue at all.
 	 */
-	if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
-	    ErtsChangeOffHeapMessageQueue *cohmq;
-	    /*
-	     * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait
-	     * thread progress before completing the change in
-	     * order to ensure that all senders observe that
-	     * messages should be passed off heap. When the
-	     * change has completed, GC does not need to inspect
-	     * the message queue at all.
-	     */
-	    erts_smp_atomic32_read_bor_nob(&c_p->state,
-					   ERTS_PSFLG_OFF_HEAP_MSGQ);
-	    c_p->flags |= F_OFF_HEAP_MSGQ_CHNG;
-	    cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG,
-			       sizeof(ErtsChangeOffHeapMessageQueue));
-	    cohmq->pid = c_p->common.id;
-	    erts_schedule_thr_prgr_later_op(change_off_heap_msgq,
-					    (void *) cohmq,
-					    &cohmq->lop);
-	}
+	erts_smp_atomic32_read_bor_nob(&c_p->state,
+				       ERTS_PSFLG_OFF_HEAP_MSGQ);
+	c_p->flags |= F_OFF_HEAP_MSGQ_CHNG;
+	cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG,
+			   sizeof(ErtsChangeOffHeapMessageQueue));
+	cohmq->pid = c_p->common.id;
+	erts_schedule_thr_prgr_later_op(change_off_heap_msgq,
+					(void *) cohmq,
+					&cohmq->lop);
     }
 
-    return am_false; /* Old state */
+    return res;
 }
 
 int
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 740ae46a0f..e241926638 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -277,7 +277,7 @@ void erts_cleanup_offheap(ErlOffHeap *offheap);
 void erts_save_message_in_proc(Process *p, ErtsMessage *msg);
 Sint erts_move_messages_off_heap(Process *c_p);
 Sint erts_complete_off_heap_message_queue_change(Process *c_p);
-Eterm erts_change_off_heap_message_queue_state(Process *c_p, int enable);
+Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state);
 
 int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int);
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d17355207b..7a31aa3e33 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10779,6 +10779,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	state |= ERTS_PSFLG_OFF_HEAP_MSGQ;
 	flags |= F_OFF_HEAP_MSGQ;
     }
+    else if (so->flags & SPO_ON_HEAP_MSGQ) {
+	state |= ERTS_PSFLG_ON_HEAP_MSGQ;
+	flags |= F_ON_HEAP_MSGQ;
+    }
 
     if (!rq)
 	rq = erts_get_runq_proc(parent);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 41b367435d..f0798d8c2d 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1145,14 +1145,15 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
 #define ERTS_PSFLG_OFF_HEAP_MSGQ	ERTS_PSFLG_BIT(18)
+#define ERTS_PSFLG_ON_HEAP_MSGQ		ERTS_PSFLG_BIT(19)
 #ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(19)
-#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(20)
-#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(21)
-#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(22)
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 23)
+#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(21)
+#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(22)
+#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(23)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 24)
 #else
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 19)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 20)
 #endif
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
@@ -1201,6 +1202,7 @@ void erts_check_for_holes(Process* p);
 #define SPO_MONITOR 4
 #define SPO_SYSTEM_PROC 8
 #define SPO_OFF_HEAP_MSGQ 16
+#define SPO_ON_HEAP_MSGQ 32
 
 extern int erts_default_spo_flags;
 
@@ -1290,9 +1292,10 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC (see below) */
 #define F_OFF_HEAP_MSGQ      (1 << 12) /* Off heap msg queue */
-#define F_OFF_HEAP_MSGQ_CHNG (1 << 13) /* Off heap msg queue changing */
-#define F_ABANDONED_HEAP_USE (1 << 14) /* Have usage of abandoned heap */
-#define F_DELAY_GC           (1 << 15) /* Similar to disable GC (see below) */
+#define F_ON_HEAP_MSGQ       (1 << 13) /* Off heap msg queue */
+#define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */
+#define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */
+#define F_DELAY_GC           (1 << 16) /* Similar to disable GC (see below) */
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
-- 
cgit v1.2.3


From 9805ea44b04b4d7db65fc6d6369addb223040032 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 9 Dec 2015 10:20:38 +0100
Subject: Fix check_process_code()

---
 erts/emulator/beam/beam_bif_load.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index c3ebf71a01..8e6123eacd 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -837,10 +837,10 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	    if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
 		return am_true;
 	    /* Should not contain any constants... */
-	    ASSERT(!any_heap_ref_ptrs(&hfrag->mem[0],
-				      &hfrag->mem[hfrag->used_size],
-				      mod_start,
-				      mod_size));
+	    ASSERT(!any_heap_refs(&hfrag->mem[0],
+				  &hfrag->mem[hfrag->used_size],
+				  mod_start,
+				  mod_size));
 	}
     }
 
@@ -881,7 +881,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 
 	    hp = &hfrag->mem[0];
 	    hp_end = &hfrag->mem[hfrag->used_size];
-	    if (any_heap_ref_ptrs(hp, hp_end, mod_start, lit_bsize))
+	    if (any_heap_refs(hp, hp_end, mod_start, lit_bsize))
 		goto try_literal_gc;
 	}
 
@@ -902,7 +902,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 
 		hp = &hfrag->mem[0];
 		hp_end = &hfrag->mem[hfrag->used_size];
-		ASSERT(!any_heap_ref_ptrs(hp, hp_end, mod_start, lit_bsize));
+		ASSERT(!any_heap_refs(hp, hp_end, mod_start, lit_bsize));
 	    }
 	}
 
-- 
cgit v1.2.3


From 343f461a2810ce3440b1d7c55cda996038071d87 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 8 Dec 2015 18:45:32 +0100
Subject: erts: Optimize hashing in process dictionary

by limiting table sizes to powers of 2.

This will change the default size from 10 to 8.
---
 erts/emulator/beam/erl_init.c         |  4 ++--
 erts/emulator/beam/erl_process_dict.c | 21 +++++++++++++++++++--
 erts/emulator/beam/erl_process_dict.h |  1 +
 3 files changed, 22 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index d9c3b0dcf4..0a34d91a01 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -197,7 +197,7 @@ Uint32 verbose;             /* See erl_debug.h for information about verbose */
 
 int erts_atom_table_size = ATOM_LIMIT;	/* Maximum number of atoms */
 
-int erts_pd_initial_size = 10;
+int erts_pd_initial_size = 8;  /* must be power of 2 */
 
 int erts_modified_timing_level;
 
@@ -1479,7 +1479,7 @@ erl_start(int argc, char **argv)
 		VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE));
 	    } else if (has_prefix("pds", sub_param)) {
 		arg = get_arg(sub_param+3, argv[i+1], &i);
-		if ((erts_pd_initial_size = atoi(arg)) <= 0) {
+		if (!erts_pd_set_initial_size(atoi(arg))) {
 		    erts_fprintf(stderr, "bad initial process dictionary size %s\n", arg);
 		    erts_usage();
 		}
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index e497267b63..c56a722542 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -80,6 +80,8 @@
 #define ARRAY_GET(PDict, Index) (((PDict)->size > (Index)) ? \
 				 (PDict)->data[Index] : NIL)
 
+#define IS_POW2(X) ((X) && !((X) & ((X)-1)))
+
 /*
  * Forward decalarations
  */
@@ -138,6 +140,16 @@ static void pd_check(ProcDict *pd);
 ** External interface
 */
 
+int
+erts_pd_set_initial_size(int size)
+{
+    if (size <= 0)
+	return 0;
+
+    erts_pd_initial_size = 1 << erts_fit_in_bits_uint(size-1);
+    return 1;
+}
+
 /*
  * Called from break handler
  */
@@ -399,6 +411,7 @@ Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id)
     unsigned int hval;
     ProcDict *pd = p->dictionary;
 
+    ASSERT(hx == MAKE_HASH(id));
     if (pd == NULL)
 	return am_undefined;
     hval = pd_hash_value_to_ix(pd, hx);
@@ -570,6 +583,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	/* Create it */
 	array_put(&(p->dictionary), INITIAL_SIZE - 1, NIL);
 	p->dictionary->homeSize = INITIAL_SIZE;
+	ASSERT(IS_POW2(p->dictionary->homeSize));
     }	
     hval = pd_hash_value(p->dictionary, id);
     old = ARRAY_GET(p->dictionary, hval);
@@ -954,9 +968,12 @@ static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term)
 static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx) 
 { 
     Uint high;
-    high = hx % (pdict->homeSize*2);
+
+    ASSERT(IS_POW2(pdict->homeSize));
+
+    high = hx & (pdict->homeSize*2 - 1);
     if (high >= HASH_RANGE(pdict))
-	return hx % pdict->homeSize;
+	return hx & (pdict->homeSize - 1);
     return high;
 }
 
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index 9aa21b7c38..ab50d45c63 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -31,6 +31,7 @@ typedef struct proc_dict {
     Eterm data[1]; /* The beginning of an array of erlang terms */
 } ProcDict;
 
+int erts_pd_set_initial_size(int size);
 Uint erts_dicts_mem_size(struct process *p);
 void erts_erase_dicts(struct process *p);
 void erts_dictionary_dump(int to, void *to_arg, ProcDict *pd);
-- 
cgit v1.2.3


From eb2e118ee2be5c008fc4417a443c351807adf123 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 8 Dec 2015 20:28:38 +0100
Subject: erts: Optimize away function "array_put" in proc dict

Replace heave array_put() with a dumb array index assignment ARRAY_PUT
and instead introduce ensure_array_size() to be called when
we know the array might need to grow.

This change also ensures the entire HASH_RANGE is always
allocated. No need for ARRAY_GET to check index any more.
---
 erts/emulator/beam/erl_process_dict.c | 104 ++++++++++++++++------------------
 1 file changed, 48 insertions(+), 56 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index c56a722542..0934183b6a 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -77,8 +77,10 @@
 #define TCDR(Term) CDR(list_val(Term))
 
 /* Array access macro */ 
-#define ARRAY_GET(PDict, Index) (((PDict)->size > (Index)) ? \
-				 (PDict)->data[Index] : NIL)
+#define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->size), \
+				 (PDict)->data[Index])
+#define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->size), \
+                                      (PDict)->data[Index] = (Val))
 
 #define IS_POW2(X) ((X) && !((X) & ((X)-1)))
 
@@ -97,7 +99,7 @@ static void shrink(Process *p, Eterm* ret);
 static void grow(Process *p);
 
 static void array_shrink(ProcDict **ppd, unsigned int need);
-static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term);
+static void ensure_array_size(ProcDict**, unsigned int size);
 
 static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx);
 static unsigned int next_array_size(unsigned int need);
@@ -361,7 +363,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret)
     if (is_boxed(old)) {	/* Tuple */
 	ASSERT(is_tuple(old));
 	if (EQ(tuple_val(old)[1], id)) {
-	    array_put(&(p->dictionary), hval, NIL);
+	    ARRAY_PUT(p->dictionary, hval, NIL);
 	    --(p->dictionary->numElements);
 	    *ret = tuple_val(old)[2];
 	}
@@ -381,7 +383,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret)
 	old = ARRAY_GET(p->dictionary, hval);
 	ASSERT(is_list(old));
 	if (is_nil(TCDR(old))) {
-	    array_put(&p->dictionary, hval, TCAR(old));
+	    ARRAY_PUT(p->dictionary, hval, TCAR(old));
 	}
     } else if (is_not_nil(old)) {
 #ifdef DEBUG
@@ -581,9 +583,8 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 
     if (p->dictionary == NULL) {
 	/* Create it */
-	array_put(&(p->dictionary), INITIAL_SIZE - 1, NIL);
+        ensure_array_size(&p->dictionary, INITIAL_SIZE);
 	p->dictionary->homeSize = INITIAL_SIZE;
-	ASSERT(IS_POW2(p->dictionary->homeSize));
     }	
     hval = pd_hash_value(p->dictionary, id);
     old = ARRAY_GET(p->dictionary, hval);
@@ -635,19 +636,19 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
      * Update the dictionary.
      */
     if (is_nil(old)) {
-	array_put(&(p->dictionary), hval, tpl);
+	ARRAY_PUT(p->dictionary, hval, tpl);
 	++(p->dictionary->numElements);
     } else if (is_boxed(old)) {
 	ASSERT(is_tuple(old));
 	if (EQ(tuple_val(old)[1],id)) {
-	    array_put(&(p->dictionary), hval, tpl);
+	    ARRAY_PUT(p->dictionary, hval, tpl);
 	    return tuple_val(old)[2];
 	} else {
 	    hp = HeapOnlyAlloc(p, 4);
 	    tmp = CONS(hp, old, NIL);
 	    hp += 2;
 	    ++(p->dictionary->numElements);
-	    array_put(&(p->dictionary), hval, CONS(hp, tpl, tmp));
+	    ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, tmp));
 	    hp += 2;
 	    ASSERT(hp <= hp_limit);
 	}
@@ -657,7 +658,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	     * New key. Simply prepend the tuple to the beginning of the list.
 	     */
 	    hp = HeapOnlyAlloc(p, 2);
-	    array_put(&(p->dictionary), hval, CONS(hp, tpl, old));
+	    ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, old));
 	    hp += 2;
 	    ASSERT(hp <= hp_limit);
 	    ++(p->dictionary->numElements);
@@ -692,7 +693,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	    nlist = CONS(hp, tpl, nlist);
 	    hp += 2;
 	    ASSERT(hp <= hp_limit);
-	    array_put(&(p->dictionary), hval, nlist);
+	    ARRAY_PUT(p->dictionary, hval, nlist);
 	    return tuple_val(TCAR(tmp))[2];
 	}
     } else {
@@ -741,7 +742,7 @@ static void shrink(Process *p, Eterm* ret)
 	lo = ARRAY_GET(pd, pd->splitPosition);
 	if (hi != NIL) {
 	    if (lo == NIL) {
-		array_put(&(p->dictionary), pd->splitPosition, hi);
+		ARRAY_PUT(p->dictionary, pd->splitPosition, hi);
 	    } else {
 		int needed = 4;
 		if (is_list(hi) && is_list(lo)) {
@@ -760,13 +761,13 @@ static void shrink(Process *p, Eterm* ret)
 			hp = HeapOnlyAlloc(p, 4);
 			tmp = CONS(hp, hi, NIL);
 			hp += 2;
-			array_put(&(p->dictionary), pd->splitPosition, 
+			ARRAY_PUT(p->dictionary, pd->splitPosition,
 				  CONS(hp,lo,tmp));
 			hp += 2;
 			ASSERT(hp <= hp_limit);
 		    } else { /* hi is a list */
 			hp = HeapOnlyAlloc(p, 2);
-			array_put(&(p->dictionary), pd->splitPosition, 
+			ARRAY_PUT(p->dictionary, pd->splitPosition,
 				  CONS(hp, lo, hi));
 			hp += 2;
 			ASSERT(hp <= hp_limit);
@@ -774,7 +775,7 @@ static void shrink(Process *p, Eterm* ret)
 		} else { /* lo is a list */
 		    if (is_tuple(hi)) {
 			hp = HeapOnlyAlloc(p, 2);
-			array_put(&(p->dictionary), pd->splitPosition, 
+			ARRAY_PUT(p->dictionary, pd->splitPosition,
 				  CONS(hp, hi, lo));
 			hp += 2;
 			ASSERT(hp <= hp_limit);
@@ -786,12 +787,12 @@ static void shrink(Process *p, Eterm* ret)
 			    hp += 2;
 			}
 			ASSERT(hp <= hp_limit);
-			array_put(&(p->dictionary), pd->splitPosition, lo);
+			ARRAY_PUT(p->dictionary, pd->splitPosition, lo);
 		    }
 		}
 	    }
 	}
-	array_put(&(p->dictionary), (pd->splitPosition + pd->homeSize), NIL);
+	ARRAY_PUT(p->dictionary, (pd->splitPosition + pd->homeSize), NIL);
     }
     if (HASH_RANGE(p->dictionary) <= (p->dictionary->size / 4)) {
 	array_shrink(&(p->dictionary), (HASH_RANGE(p->dictionary) * 3) / 2);
@@ -808,7 +809,7 @@ static void grow(Process *p)
     unsigned int pos;
     unsigned int homeSize;
     int needed = 0;
-    ProcDict *pd;
+    ProcDict *pd = p->dictionary;
 #ifdef DEBUG
     Eterm *hp_limit;
 #endif
@@ -817,16 +818,18 @@ static void grow(Process *p)
     if (steps == 0)
 	steps = 1;
     /* Dont grow over MAX_HASH */
-    if ((MAX_HASH - steps) <= HASH_RANGE(p->dictionary)) {
+    if ((MAX_HASH - steps) <= HASH_RANGE(pd)) {
 	return;
     }
 
+    ensure_array_size(&p->dictionary, HASH_RANGE(pd) + steps);
+    pd = p->dictionary;
+
     /*
      * Calculate total number of heap words needed, and garbage collect
      * if necessary.
      */
 
-    pd = p->dictionary;
     pos = pd->splitPosition;
     homeSize = pd->homeSize;
     for (i = 0; i < steps; ++i) {
@@ -855,7 +858,6 @@ static void grow(Process *p)
      */
 
     for (i = 0; i < steps; ++i) {
-	ProcDict *pd = p->dictionary;
 	if (pd->splitPosition == pd->homeSize) {
 	    pd->homeSize *= 2;
 	    pd->splitPosition = 0;
@@ -865,9 +867,8 @@ static void grow(Process *p)
 	l = ARRAY_GET(pd, pos);
 	if (is_tuple(l)) {
 	    if (pd_hash_value(pd, tuple_val(l)[1]) != pos) {
-		array_put(&(p->dictionary), pos + 
-			  p->dictionary->homeSize, l);
-		array_put(&(p->dictionary), pos, NIL);
+		ARRAY_PUT(pd, pos + pd->homeSize, l);
+		ARRAY_PUT(pd, pos, NIL);
 	    }
 	} else {
 	    l2 = NIL;
@@ -889,10 +890,8 @@ static void grow(Process *p)
 	    if (l2 != NIL && TCDR(l2) == NIL)
 		l2 = TCAR(l2);
 	    ASSERT(hp <= hp_limit);
-	    /* After array_put pd is no longer valid */
-	    array_put(&(p->dictionary), pos, l1);
-	    array_put(&(p->dictionary), pos + 
-		      p->dictionary->homeSize, l2);
+	    ARRAY_PUT(pd, pos, l1);
+	    ARRAY_PUT(pd, pos + pd->homeSize, l2);
 	}
     }
 
@@ -912,7 +911,7 @@ static void array_shrink(ProcDict **ppd, unsigned int need)
     HDEBUGF(("array_shrink: size = %d, used = %d, need = %d",
 	     (*ppd)->size, (*ppd)->used, need));
 
-    if (siz > (*ppd)->size)
+    if (siz >= (*ppd)->size)
 	return; /* Only shrink */
 
     *ppd = PD_REALLOC(((void *) *ppd),
@@ -925,40 +924,33 @@ static void array_shrink(ProcDict **ppd, unsigned int need)
 }
     
 			
-static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term)
+static void ensure_array_size(ProcDict **ppdict, unsigned int size)
 {
+    ProcDict *pd = *ppdict;
     unsigned int i;
-    Eterm ret;
-    if (*ppdict == NULL) {
-	Uint siz = next_array_size(ndx+1);
-	ProcDict *p;
 
-        p = PD_ALLOC(PD_SZ2BYTES(siz));
+    if (pd == NULL) {
+	Uint siz = next_array_size(size);
+
+        pd = PD_ALLOC(PD_SZ2BYTES(siz));
 	for (i = 0; i < siz; ++i) 
-	    p->data[i] = NIL;
-	p->size = siz;
-	p->homeSize = p->splitPosition = p->numElements = p->used = 0;
-	*ppdict = p;
-    } else if (ndx >= (*ppdict)->size) {
-	Uint osize = (*ppdict)->size;
-	Uint nsize = next_array_size(ndx+1);
-	*ppdict = PD_REALLOC(((void *) *ppdict),
+	    pd->data[i] = NIL;
+	pd->size = siz;
+	pd->homeSize = pd->splitPosition = pd->numElements = pd->used = 0;
+        *ppdict = pd;
+    } else if (size > pd->size) {
+	Uint osize = pd->size;
+	Uint nsize = next_array_size(size);
+	pd = PD_REALLOC(((void *) pd),
 			     PD_SZ2BYTES(osize),
 			     PD_SZ2BYTES(nsize));
 	for (i = osize; i < nsize; ++i)
-	    (*ppdict)->data[i] = NIL;
-	(*ppdict)->size = nsize;
+	    pd->data[i] = NIL;
+	pd->size = nsize;
+        *ppdict = pd;
     }
-    ret = (*ppdict)->data[ndx];
-    (*ppdict)->data[ndx] = term;
-    if ((ndx + 1) > (*ppdict)->used)
-	(*ppdict)->used = ndx + 1;
-#ifdef HARDDEBUG
-    HDEBUGF(("array_put: (*ppdict)->size = %d, (*ppdict)->used = %d, ndx = %d",
-	     (*ppdict)->size, (*ppdict)->used, ndx));
-    erts_fprintf(stderr, "%T", term);
-#endif /* HARDDEBUG */
-    return ret;
+    if (size > pd->used)
+	pd->used = size;
 }
 
 /*
-- 
cgit v1.2.3


From bb5f71a7573158056dd9c80228c95833f970ec0b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 9 Dec 2015 19:28:15 +0100
Subject: erts: Add proc dict macros ERTS_PD_START/SIZE

---
 erts/emulator/beam/beam_bif_load.c    |  4 ++--
 erts/emulator/beam/erl_debug.c        |  6 +++---
 erts/emulator/beam/erl_gc.c           | 10 +++++-----
 erts/emulator/beam/erl_process_dict.h |  3 +++
 4 files changed, 13 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 0e192b1ebd..c804a09f87 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -851,8 +851,8 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	}
 
 	if (rp->dictionary != NULL) {
-	    Eterm* start = rp->dictionary->data;
-	    Eterm* end = start + rp->dictionary->used;
+	    Eterm* start = ERTS_PD_START(rp->dictionary);
+	    Eterm* end = start + ERTS_PD_SIZE(rp->dictionary);
 
 	    if (any_heap_ref_ptrs(start, end, mod_start, mod_size)) {
 		goto need_gc;
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 2dcfb79f00..a2af3adf70 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -416,7 +416,7 @@ void verify_process(Process *p)
     erts_check_heap(p);
 
     if (p->dictionary)
-        VERIFY_AREA("dictionary",p->dictionary->data, p->dictionary->used);
+        VERIFY_AREA("dictionary", ERTS_PD_START(p->dictionary), ERTS_PD_SIZE(p->dictionary));
     VERIFY_ETERM("seq trace token",p->seq_trace_token);
     VERIFY_ETERM("group leader",p->group_leader);
     VERIFY_ETERM("fvalue",p->fvalue);
@@ -542,8 +542,8 @@ static void print_process_memory(Process *p)
     }
 
     if (p->dictionary != NULL) {
-        int n = p->dictionary->used;
-        Eterm *ptr = p->dictionary->data;
+        int n = ERTS_PD_SIZE(p->dictionary);
+        Eterm *ptr = ERTS_PD_START(p->dictionary);
         erts_printf("  Dictionary: ");
         while (n--) erts_printf("0x%0*lx ",PTR_SIZE,(unsigned long)ptr++);
         erts_printf("\n");
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d2604f1595..b743b7e8f6 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1953,7 +1953,7 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
 #ifdef HARDDEBUG
     disallow_heap_frag_ref(p, n_htop, p->stop, STACK_START(p) - p->stop);
     if (p->dictionary != NULL) {
-	disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used);
+	disallow_heap_frag_ref(p, n_htop, ERTS_PD_START(p->dictionary), ERTS_PD_SIZE(p->dictionary));
     }
     disallow_heap_frag_ref_in_heap(p);
 #endif
@@ -1993,8 +1993,8 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
     ++n;
 
     if (p->dictionary != NULL) {
-        roots[n].v = p->dictionary->data;
-        roots[n].sz = p->dictionary->used;
+        roots[n].v = ERTS_PD_START(p->dictionary);
+        roots[n].sz = ERTS_PD_SIZE(p->dictionary);
         ++n;
     }
     if (nobj > 0) {
@@ -2589,8 +2589,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 	       Eterm* objv, int nobj)
 {
     if (p->dictionary)  {
-	offset_heap(p->dictionary->data, 
-		    p->dictionary->used, 
+	offset_heap(ERTS_PD_START(p->dictionary),
+		    ERTS_PD_SIZE(p->dictionary),
 		    offs, area, area_size);
     }
 
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index ab50d45c63..3ad070d914 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -31,6 +31,9 @@ typedef struct proc_dict {
     Eterm data[1]; /* The beginning of an array of erlang terms */
 } ProcDict;
 
+#define ERTS_PD_START(PD) ((PD)->data)
+#define ERTS_PD_SIZE(PD)  ((PD)->used)
+
 int erts_pd_set_initial_size(int size);
 Uint erts_dicts_mem_size(struct process *p);
 void erts_erase_dicts(struct process *p);
-- 
cgit v1.2.3


From 4179ca9f10cdc78e882ce4496cf0a1261a0129af Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 9 Dec 2015 19:34:45 +0100
Subject: erts: Remove ProcDict.used

(homeSize + splitPosition) will do just fine
---
 erts/emulator/beam/erl_process_dict.c | 20 +++++++++-----------
 erts/emulator/beam/erl_process_dict.h |  3 +--
 2 files changed, 10 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 0934183b6a..34c1f0c993 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -164,9 +164,9 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd)
     /*PD_CHECK(pd);*/
     if (pd == NULL)
 	return;
-    erts_print(to, to_arg, "(size = %d, used = %d, homeSize = %d, "
+    erts_print(to, to_arg, "(size = %d, homeSize = %d, "
 	       "splitPosition = %d, numElements = %d)\n",
-	       pd->size, pd->used, pd->homeSize, 
+	       pd->size, pd->homeSize,
 	       pd->splitPosition, (unsigned int) pd->numElements);
     for (i = 0; i < HASH_RANGE(pd); ++i) {
 	erts_print(to, to_arg, "%d: %T\n", i, ARRAY_GET(pd, i));
@@ -908,8 +908,8 @@ static void array_shrink(ProcDict **ppd, unsigned int need)
 {
     unsigned int siz = next_array_size(need);
 
-    HDEBUGF(("array_shrink: size = %d, used = %d, need = %d",
-	     (*ppd)->size, (*ppd)->used, need));
+    HDEBUGF(("array_shrink: size = %d, need = %d",
+	     (*ppd)->size, need));
 
     if (siz >= (*ppd)->size)
 	return; /* Only shrink */
@@ -919,8 +919,6 @@ static void array_shrink(ProcDict **ppd, unsigned int need)
 		      PD_SZ2BYTES(siz));
 
     (*ppd)->size = siz;
-    if ((*ppd)->size < (*ppd)->used)
-	(*ppd)->used = (*ppd)->size;
 }
     
 			
@@ -936,7 +934,7 @@ static void ensure_array_size(ProcDict **ppdict, unsigned int size)
 	for (i = 0; i < siz; ++i) 
 	    pd->data[i] = NIL;
 	pd->size = siz;
-	pd->homeSize = pd->splitPosition = pd->numElements = pd->used = 0;
+	pd->homeSize = pd->splitPosition = pd->numElements = 0;
         *ppdict = pd;
     } else if (size > pd->size) {
 	Uint osize = pd->size;
@@ -949,8 +947,6 @@ static void ensure_array_size(ProcDict **ppdict, unsigned int size)
 	pd->size = nsize;
         *ppdict = pd;
     }
-    if (size > pd->used)
-	pd->used = size;
 }
 
 /*
@@ -1029,12 +1025,14 @@ static unsigned int next_array_size(unsigned int need)
 static void pd_check(ProcDict *pd) 
 {
     unsigned int i;
+    unsigned int used;
     Uint num;
     if (pd == NULL)
 	return;
-    ASSERT(pd->size >= pd->used);
+    used = HASH_RANGE(pd);
+    ASSERT(pd->size >= used);
     ASSERT(HASH_RANGE(pd) <= MAX_HASH);
-    for (i = 0, num = 0; i < pd->used; ++i) {
+    for (i = 0, num = 0; i < used; ++i) {
 	Eterm t = pd->data[i];
 	if (is_nil(t)) {
 	    continue;
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index 3ad070d914..eb26a1ffcc 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -24,7 +24,6 @@
 
 typedef struct proc_dict {
     unsigned int size;
-    unsigned int used;
     unsigned int homeSize;
     unsigned int splitPosition;
     Uint numElements;
@@ -32,7 +31,7 @@ typedef struct proc_dict {
 } ProcDict;
 
 #define ERTS_PD_START(PD) ((PD)->data)
-#define ERTS_PD_SIZE(PD)  ((PD)->used)
+#define ERTS_PD_SIZE(PD)  ((PD)->homeSize + (PD)->splitPosition)
 
 int erts_pd_set_initial_size(int size);
 Uint erts_dicts_mem_size(struct process *p);
-- 
cgit v1.2.3


From 4804ea2aa50af490ab3998466269efa540abd90d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Dec 2015 15:25:22 +0100
Subject: erts: Add sizeMask for faster proc dict indexing

---
 erts/emulator/beam/erl_process_dict.c | 26 +++++++++++++++-----------
 erts/emulator/beam/erl_process_dict.h |  1 +
 2 files changed, 16 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 34c1f0c993..97a2828b4e 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -585,6 +585,9 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	/* Create it */
         ensure_array_size(&p->dictionary, INITIAL_SIZE);
 	p->dictionary->homeSize = INITIAL_SIZE;
+        p->dictionary->sizeMask = p->dictionary->homeSize*2 - 1;
+        p->dictionary->splitPosition = 0;
+        p->dictionary->numElements = 0;
     }	
     hval = pd_hash_value(p->dictionary, id);
     old = ARRAY_GET(p->dictionary, hval);
@@ -718,6 +721,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 
 static void shrink(Process *p, Eterm* ret) 
 {
+    ProcDict *pd = p->dictionary;
     unsigned int range = HASH_RANGE(p->dictionary);
     unsigned int steps = (range*3) / 10;
     Eterm hi, lo, tmp;
@@ -732,8 +736,8 @@ static void shrink(Process *p, Eterm* ret)
     }
 
     for (i = 0; i < steps; ++i) {
-	ProcDict *pd = p->dictionary;
 	if (pd->splitPosition == 0) {
+            pd->sizeMask = pd->homeSize - 1;
 	    pd->homeSize /= 2;
 	    pd->splitPosition = pd->homeSize;
 	}
@@ -742,7 +746,7 @@ static void shrink(Process *p, Eterm* ret)
 	lo = ARRAY_GET(pd, pd->splitPosition);
 	if (hi != NIL) {
 	    if (lo == NIL) {
-		ARRAY_PUT(p->dictionary, pd->splitPosition, hi);
+		ARRAY_PUT(pd, pd->splitPosition, hi);
 	    } else {
 		int needed = 4;
 		if (is_list(hi) && is_list(lo)) {
@@ -761,13 +765,13 @@ static void shrink(Process *p, Eterm* ret)
 			hp = HeapOnlyAlloc(p, 4);
 			tmp = CONS(hp, hi, NIL);
 			hp += 2;
-			ARRAY_PUT(p->dictionary, pd->splitPosition,
+			ARRAY_PUT(pd, pd->splitPosition,
 				  CONS(hp,lo,tmp));
 			hp += 2;
 			ASSERT(hp <= hp_limit);
 		    } else { /* hi is a list */
 			hp = HeapOnlyAlloc(p, 2);
-			ARRAY_PUT(p->dictionary, pd->splitPosition,
+			ARRAY_PUT(pd, pd->splitPosition,
 				  CONS(hp, lo, hi));
 			hp += 2;
 			ASSERT(hp <= hp_limit);
@@ -775,7 +779,7 @@ static void shrink(Process *p, Eterm* ret)
 		} else { /* lo is a list */
 		    if (is_tuple(hi)) {
 			hp = HeapOnlyAlloc(p, 2);
-			ARRAY_PUT(p->dictionary, pd->splitPosition,
+			ARRAY_PUT(pd, pd->splitPosition,
 				  CONS(hp, hi, lo));
 			hp += 2;
 			ASSERT(hp <= hp_limit);
@@ -787,12 +791,12 @@ static void shrink(Process *p, Eterm* ret)
 			    hp += 2;
 			}
 			ASSERT(hp <= hp_limit);
-			ARRAY_PUT(p->dictionary, pd->splitPosition, lo);
+			ARRAY_PUT(pd, pd->splitPosition, lo);
 		    }
 		}
 	    }
 	}
-	ARRAY_PUT(p->dictionary, (pd->splitPosition + pd->homeSize), NIL);
+	ARRAY_PUT(pd, (pd->splitPosition + pd->homeSize), NIL);
     }
     if (HASH_RANGE(p->dictionary) <= (p->dictionary->size / 4)) {
 	array_shrink(&(p->dictionary), (HASH_RANGE(p->dictionary) * 3) / 2);
@@ -860,7 +864,8 @@ static void grow(Process *p)
     for (i = 0; i < steps; ++i) {
 	if (pd->splitPosition == pd->homeSize) {
 	    pd->homeSize *= 2;
-	    pd->splitPosition = 0;
+            pd->sizeMask = pd->homeSize*2 - 1;
+            pd->splitPosition = 0;
 	}
 	pos = pd->splitPosition;
 	++pd->splitPosition; /* For the hashes */
@@ -934,7 +939,6 @@ static void ensure_array_size(ProcDict **ppdict, unsigned int size)
 	for (i = 0; i < siz; ++i) 
 	    pd->data[i] = NIL;
 	pd->size = siz;
-	pd->homeSize = pd->splitPosition = pd->numElements = 0;
         *ppdict = pd;
     } else if (size > pd->size) {
 	Uint osize = pd->size;
@@ -959,9 +963,9 @@ static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx)
 
     ASSERT(IS_POW2(pdict->homeSize));
 
-    high = hx & (pdict->homeSize*2 - 1);
+    high = hx & pdict->sizeMask;
     if (high >= HASH_RANGE(pdict))
-	return hx & (pdict->homeSize - 1);
+	return hx & (pdict->sizeMask >> 1);
     return high;
 }
 
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index eb26a1ffcc..fd59c969cf 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -23,6 +23,7 @@
 #include "sys.h"
 
 typedef struct proc_dict {
+    unsigned int sizeMask;
     unsigned int size;
     unsigned int homeSize;
     unsigned int splitPosition;
-- 
cgit v1.2.3


From efa7a08d1b16aeb280c87f035a97d31ca6eebba6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 11 Dec 2015 14:17:57 +0100
Subject: erts: Fix faulty cleanup when receiving broken dist msg

Bug introduced in ce8279d6a48d41f9.

Thank you valgrind.
---
 erts/emulator/beam/dist.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 170690ca89..7be2b77a3b 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1231,11 +1231,11 @@ int erts_net_message(Port *prt,
     arg = erts_decode_dist_ext(&factory, &ede);
     if (is_non_value(arg)) {
 #ifdef ERTS_DIST_MSG_DBG
-	erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n");
+	erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext(CTL) failed:\n");
 	bw(buf, orig_len);
 #endif
 	PURIFY_MSG("data error");
-	goto data_error;
+	goto decode_error;
     }
     ctl_len = t - buf;
 
@@ -1728,12 +1728,13 @@ int erts_net_message(Port *prt,
 	erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg);
 	erts_send_error_to_logger_nogl(dsbufp);
     }
- data_error:
+decode_error:
     PURIFY_MSG("data error");
     erts_factory_close(&factory);
     if (ctl != ctl_default) {
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
+data_error:
     UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     erts_deliver_port_exit(prt, dep->cid, am_killed, 0);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-- 
cgit v1.2.3


From eda21ee8a6b87a7ec03822e255d82827db1aee12 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 11 Dec 2015 19:37:51 +0100
Subject: erts: Fix correct node name for DTRACE

broken by 949de78331b9c4ecb9.
---
 erts/emulator/beam/erl_node_tables.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index a7d0511bf9..707de39556 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -752,6 +752,10 @@ erts_set_this_node(Eterm sysname, Uint creation)
     erts_this_dist_entry = erts_this_node->dist_entry;
 
     erts_refc_inc(&erts_this_dist_entry->refc, 2);
+
+    erts_this_node_sysname = erts_this_node_sysname_BUFFER;
+    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
+		  "%T", sysname);
 }
 
 Uint
-- 
cgit v1.2.3


From 4b98e710b9c45481c1cdc7a4ee68f7ce7fca908a Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Mon, 13 Jul 2015 15:38:41 +0200
Subject: erts: Add support for asynchronous open_port

OTP-13086
---
 erts/emulator/beam/atom.names     |  1 +
 erts/emulator/beam/bif.tab        |  2 +-
 erts/emulator/beam/erl_bif_port.c | 54 ++++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_driver.h   |  4 +++
 erts/emulator/beam/erl_port.h     |  5 ++++
 erts/emulator/beam/io.c           | 60 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 114 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 967cf013f0..f20d99f114 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -209,6 +209,7 @@ atom dsend_continue_trap
 atom dunlink
 atom duplicate_bag
 atom dupnames
+atom einval
 atom elib_malloc
 atom emulator
 atom enable_trace
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 65f8d6f1f5..884555dee2 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -115,7 +115,7 @@ bif erlang:time_offset/0
 bif erlang:time_offset/1
 bif erlang:timestamp/0
 
-bif erlang:open_port/2
+bif erts_internal:open_port/2
 
 bif erlang:pid_to_list/1
 bif erlang:ports/0
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index e47d7bcbbb..839abd0424 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -41,6 +41,7 @@
 #include "external.h"
 #include "packet_parser.h"
 #include "erl_bits.h"
+#include "erl_bif_unique.h"
 #include "dtrace-wrapper.h"
 
 static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump);
@@ -50,10 +51,10 @@ static void free_args(char **);
 
 char *erts_default_arg0 = "default";
 
-BIF_RETTYPE open_port_2(BIF_ALIST_2)
+BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
 {
     Port *port;
-    Eterm port_id;
+    Eterm res;
     char *str;
     int err_type, err_num;
 
@@ -61,27 +62,58 @@ BIF_RETTYPE open_port_2(BIF_ALIST_2)
     if (!port) {
 	if (err_type == -3) {
 	    ASSERT(err_num == BADARG || err_num == SYSTEM_LIMIT);
-	    BIF_ERROR(BIF_P, err_num);
+	    if (err_num == BADARG)
+                res = am_badarg;
+            else if (err_num == SYSTEM_LIMIT)
+                res = am_system_limit;
+            else
+                /* this is only here to silence gcc, it should not happen */
+                BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
 	} else if (err_type == -2) {
 	    str = erl_errno_id(err_num);
+            res = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1);
 	} else {
-	    str = "einval";
+	    res = am_einval;
 	}
-	BIF_P->fvalue = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1);
-	BIF_ERROR(BIF_P, EXC_ERROR);
-    }
+        BIF_RET(res);
+    }
+
+    if (port->drv_ptr->flags & ERL_DRV_FLAG_USE_INIT_ACK) {
+        /* Copied from erl_port_task.c */
+        port->async_open_port = erts_alloc(ERTS_ALC_T_PRTSD,
+                                           sizeof(*port->async_open_port));
+        erts_make_ref_in_array(port->async_open_port->ref);
+        port->async_open_port->to = BIF_P->common.id;
+
+        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
+        if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
+	    /* need to exit caller instead */
+	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
+	    KILL_CATCHES(BIF_P);
+	    BIF_P->freason = EXC_EXIT;
+            erts_port_release(port);
+            BIF_RET(am_badarg);
+	}
+
+	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(BIF_P);
+	BIF_P->msg.save = BIF_P->msg.last;
 
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE);
+
+        res = erts_proc_store_ref(BIF_P, port->async_open_port->ref);
+    } else {
+        res = port->common.id;
+        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+    }
 
-    port_id = port->common.id;
     erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id);
-    erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port_id);
+    erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port->common.id);
 
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 
     erts_port_release(port);
 
-    BIF_RET(port_id);
+    BIF_RET(res);
 }
 
 static ERTS_INLINE Port *
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index e71b87803b..5f2115ef7f 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -163,6 +163,7 @@ typedef struct {
 #define ERL_DRV_FLAG_USE_PORT_LOCKING	(1 << 0)
 #define ERL_DRV_FLAG_SOFT_BUSY		(1 << 1)
 #define ERL_DRV_FLAG_NO_BUSY_MSGQ	(1 << 2)
+#define ERL_DRV_FLAG_USE_INIT_ACK	(1 << 3)
 
 /*
  * Integer types
@@ -690,6 +691,9 @@ EXTERN char *driver_dl_error(void);
 EXTERN int erl_drv_putenv(char *key, char *value);
 EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 
+/* spawn start init ack */
+EXTERN void erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res);
+
 #endif /* !ERL_DRIVER_TYPES_ONLY */
 
 #ifdef WIN32_DYNAMIC_ERL_DRIVER
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index acd68ef0ad..bc4b412594 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -187,6 +187,11 @@ struct _erl_drv_port {
 
     ErtsPrtSD *psd;		 /* Port specific data */
     int reds; /* Only used while executing driver callbacks */
+
+    struct {
+        Eterm to;
+        Uint32 ref[ERTS_MAX_REF_NUMBERS];
+    } *async_open_port;         /* Reference used with async open port */
 };
 
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index fdd26fcc4b..409df846e9 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -84,6 +84,7 @@ static void deliver_result(Eterm sender, Eterm pid, Eterm res);
 static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *);
 static void terminate_port(Port *p);
 static void pdl_init(void);
+static int driver_failure_term(ErlDrvPort ix, Eterm term, int eof);
 #ifdef ERTS_SMP
 static void driver_monitor_lock_pdl(Port *p);
 static void driver_monitor_unlock_pdl(Port *p);
@@ -383,6 +384,7 @@ static Port *create_port(char *name,
     ERTS_PTMR_INIT(prt);
     erts_port_task_handle_init(&prt->timeout_task);
     prt->psd = NULL;
+    prt->async_open_port = NULL;
     prt->drv_data = (SWord) 0;
     prt->os_pid = -1;
 
@@ -2732,6 +2734,61 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp)
 					  port_sig_link);
 }
 
+static void
+init_ack_send_reply(Port *port, Eterm resp)
+{
+
+    if (!is_internal_port(resp)) {
+        Process *rp = erts_proc_lookup_raw(port->async_open_port->to);
+        erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+        erts_remove_link(&ERTS_P_LINKS(port), port->async_open_port->to);
+        erts_remove_link(&ERTS_P_LINKS(rp), port->common.id);
+        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+    }
+    port_sched_op_reply(port->async_open_port->to,
+                        port->async_open_port->ref,
+                        resp);
+
+    erts_free(ERTS_ALC_T_PRTSD, port->async_open_port);
+    port->async_open_port = NULL;
+}
+
+void
+erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res) {
+    Port *port = erts_drvport2port(ix);
+    SWord err_type = (SWord)res;
+    Eterm resp;
+
+    if (port == ERTS_INVALID_ERL_DRV_PORT && port->async_open_port)
+        return;
+
+    if (port->async_open_port) {
+        switch(err_type) {
+        case -3:
+            resp = am_badarg;
+            break;
+        case -2: {
+            char *str = erl_errno_id(errno);
+            resp = erts_atom_put((byte *) str, strlen(str),
+                                 ERTS_ATOM_ENC_LATIN1, 1);
+            break;
+        }
+        case -1:
+            resp = am_einval;
+            break;
+        default:
+            resp = port->common.id;
+            break;
+        }
+
+        init_ack_send_reply(port, resp);
+
+        if (err_type == -1 || err_type == -2 || err_type == -3)
+            driver_failure_term(ix, am_normal, 0);
+        port->drv_data = err_type;
+    }
+}
+
 void erts_init_io(int port_tab_size,
 		  int port_tab_size_ignore_files,
 		  int legacy_port_tab)
@@ -6972,6 +7029,9 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+
+    if (prt->async_open_port)
+        init_ack_send_reply(prt, prt->common.id);
     if (eof)
 	flush_linebuf_messages(prt, state);
     if (state & ERTS_PORT_SFLG_CLOSING) {
-- 
cgit v1.2.3


From 91c1876f70870f450f7e8fd3b02f2396067e3cb8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Tue, 14 Jul 2015 11:07:33 +0200
Subject: erts: Add erl_drv_set_pid

OTP-13087
---
 erts/emulator/beam/erl_driver.h |  3 +++
 erts/emulator/beam/io.c         | 11 +++++++++++
 2 files changed, 14 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 5f2115ef7f..3d663655a2 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -694,6 +694,9 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 /* spawn start init ack */
 EXTERN void erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res);
 
+/* set the pid seen in port_info */
+EXTERN void erl_drv_set_os_pid(ErlDrvPort ix, ErlDrvSInt pid);
+
 #endif /* !ERL_DRIVER_TYPES_ONLY */
 
 #ifdef WIN32_DYNAMIC_ERL_DRIVER
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 409df846e9..2a3759212e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2789,6 +2789,17 @@ erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res) {
     }
 }
 
+void
+erl_drv_set_os_pid(ErlDrvPort ix, ErlDrvSInt pid) {
+    Port *port = erts_drvport2port(ix);
+
+    if (port == ERTS_INVALID_ERL_DRV_PORT)
+        return;
+
+    port->os_pid = (SWord)pid;
+
+}
+
 void erts_init_io(int port_tab_size,
 		  int port_tab_size_ignore_files,
 		  int legacy_port_tab)
-- 
cgit v1.2.3


From 31b8dd2b4b5259d9ed2178017c3580c42bf62ec6 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Wed, 12 Aug 2015 10:21:09 +0200
Subject: erts: Bump driver minor version

---
 erts/emulator/beam/erl_driver.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 3d663655a2..ef7792ecab 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -125,7 +125,7 @@ typedef struct {
 
 #define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
 #define ERL_DRV_EXTENDED_MAJOR_VERSION	3
-#define ERL_DRV_EXTENDED_MINOR_VERSION	2
+#define ERL_DRV_EXTENDED_MINOR_VERSION	3
 
 /*
  * The emulator will refuse to load a driver with a major version
-- 
cgit v1.2.3


From 17ecb94437bac6726044d299dde48d02dd2b2e9c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Tue, 14 Jul 2015 15:20:16 +0200
Subject: erts: Refactor out erts functions from hash

---
 erts/emulator/beam/atom.c            |  3 +++
 erts/emulator/beam/erl_fun.c         |  3 +++
 erts/emulator/beam/erl_node_tables.c | 12 ++++++++----
 erts/emulator/beam/erl_node_tables.h |  1 +
 erts/emulator/beam/export.c          |  3 +++
 erts/emulator/beam/hash.c            | 30 +++++++++++++++---------------
 erts/emulator/beam/hash.h            | 20 +++++++++++++-------
 erts/emulator/beam/index.h           |  4 ++++
 erts/emulator/beam/module.c          |  3 +++
 erts/emulator/beam/register.c        |  3 +++
 10 files changed, 56 insertions(+), 26 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index fe91134ef4..fd2adac676 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -435,6 +435,9 @@ init_atom_table(void)
     f.cmp  = (HCMP_FUN) atom_cmp;
     f.alloc = (HALLOC_FUN) atom_alloc;
     f.free = (HFREE_FUN) atom_free;
+    f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+    f.meta_free = (HMFREE_FUN) erts_free;
+    f.meta_print = (HMPRINT_FUN) erts_print;
 
     atom_text_pos = NULL;
     atom_text_end = NULL;
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 4268e2d40a..cff476694c 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -66,6 +66,9 @@ erts_init_fun_table(void)
     f.cmp  = (HCMP_FUN) fun_cmp;
     f.alloc = (HALLOC_FUN) fun_alloc;
     f.free = (HFREE_FUN) fun_free;
+    f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+    f.meta_free = (HMFREE_FUN) erts_free;
+    f.meta_print = (HMPRINT_FUN) erts_print;
 
     hash_init(ERTS_ALC_T_FUN_TABLE, &erts_fun_table, "fun_table", 16, f);
 }
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 62a44f7129..e5bd0d58f0 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -793,10 +793,14 @@ void erts_init_node_tables(int dd_sec)
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    f.hash  = (H_FUN)			dist_table_hash;
-    f.cmp   = (HCMP_FUN)		dist_table_cmp;
-    f.alloc = (HALLOC_FUN)		dist_table_alloc;
-    f.free  = (HFREE_FUN)		dist_table_free;
+    f.hash       = (H_FUN)		dist_table_hash;
+    f.cmp        = (HCMP_FUN)		dist_table_cmp;
+    f.alloc      = (HALLOC_FUN)		dist_table_alloc;
+    f.free       = (HFREE_FUN)		dist_table_free;
+    f.meta_alloc = (HMALLOC_FUN) 	erts_alloc;
+    f.meta_free  = (HMFREE_FUN) 	erts_free;
+    f.meta_print = (HMPRINT_FUN) 	erts_print;
+
 
     erts_this_dist_entry = erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry));
     dist_entries = 1;
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 64278d2ea0..fb2f2a5407 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -41,6 +41,7 @@
 
 #include "sys.h"
 #include "hash.h"
+#include "erl_alloc.h"
 #include "erl_process.h"
 #include "erl_monitors.h"
 #include "erl_smp.h"
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2420df36b5..581efe6eec 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -184,6 +184,9 @@ init_export_table(void)
     f.cmp  = (HCMP_FUN) export_cmp;
     f.alloc = (HALLOC_FUN) export_alloc;
     f.free = (HFREE_FUN) export_free;
+    f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+    f.meta_free = (HMFREE_FUN) erts_free;
+    f.meta_print = (HMPRINT_FUN) erts_print;
 
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
 	erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_tables[i], "export_list",
diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index e0fde337f2..acdec28e19 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -98,11 +98,11 @@ void hash_info(int to, void *arg, Hash* h)
 
     hash_get_info(&hi, h);
 
-    erts_print(to, arg, "=hash_table:%s\n", hi.name);
-    erts_print(to, arg, "size: %d\n",       hi.size);
-    erts_print(to, arg, "used: %d\n",       hi.used);
-    erts_print(to, arg, "objs: %d\n",       hi.objs);
-    erts_print(to, arg, "depth: %d\n",      hi.depth);
+    h->fun.meta_print(to, arg, "=hash_table:%s\n", hi.name);
+    h->fun.meta_print(to, arg, "size: %d\n",       hi.size);
+    h->fun.meta_print(to, arg, "used: %d\n",       hi.used);
+    h->fun.meta_print(to, arg, "objs: %d\n",       hi.objs);
+    h->fun.meta_print(to, arg, "depth: %d\n",      hi.depth);
 }
 
 
@@ -123,22 +123,22 @@ hash_table_sz(Hash *h)
 ** init a pre allocated or static hash structure
 ** and allocate buckets.
 */
-Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions fun)
+Hash* hash_init(int type, Hash* h, char* name, int size, HashFunctions fun)
 {
     int sz;
     int ix = 0;
 
-    h->type = type;
+    h->meta_alloc_type = type;
 
     while (h_size_table[ix] != -1 && h_size_table[ix] < size)
 	ix++;
     if (h_size_table[ix] == -1)
-	erl_exit(1, "panic: too large hash table size (%d)\n", size);
+	return NULL;
 
     size = h_size_table[ix];
     sz = size*sizeof(HashBucket*);
 
-    h->bucket = (HashBucket**) erts_alloc(h->type, sz);
+    h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz);
 
     sys_memzero(h->bucket, sz);
     h->is_allocated = 0;
@@ -155,11 +155,11 @@ Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions
 /*
 ** Create a new hash table
 */
-Hash* hash_new(ErtsAlcType_t type, char* name, int size, HashFunctions fun)
+Hash* hash_new(int type, char* name, int size, HashFunctions fun)
 {
     Hash* h;
 
-    h = erts_alloc(type, sizeof(Hash));
+    h = fun.meta_alloc(type, sizeof(Hash));
 
     h = hash_init(type, h, name, size, fun);
     h->is_allocated =  1;
@@ -183,9 +183,9 @@ void hash_delete(Hash* h)
 	    b = b_next;
 	}
     }
-    erts_free(h->type, h->bucket);
+    h->fun.meta_free(h->meta_alloc_type, h->bucket);
     if (h->is_allocated)
-	erts_free(h->type, (void*) h);
+	h->fun.meta_free(h->meta_alloc_type, (void*) h);
 }
 
 /*
@@ -213,7 +213,7 @@ static void rehash(Hash* h, int grow)
     h->size80percent = (4*h->size)/5;
     sz = h->size*sizeof(HashBucket*);
 
-    new_bucket = (HashBucket **) erts_alloc(h->type, sz);
+    new_bucket = (HashBucket **) h->fun.meta_alloc(h->meta_alloc_type, sz);
     sys_memzero(new_bucket, sz);
 
     h->used = 0;
@@ -230,7 +230,7 @@ static void rehash(Hash* h, int grow)
 	    b = b_next;
 	}
     }
-    erts_free(h->type, (void *) h->bucket);
+    h->fun.meta_free(h->meta_alloc_type, (void *) h->bucket);
     h->bucket = new_bucket;
 }
 
diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h
index 87fdb360e3..e98f6c32e1 100644
--- a/erts/emulator/beam/hash.h
+++ b/erts/emulator/beam/hash.h
@@ -29,14 +29,17 @@
 #include "sys.h"
 #endif
 
-#include "erl_alloc.h"
-
 typedef unsigned long HashValue;
+typedef struct hash Hash;
 
 typedef int (*HCMP_FUN)(void*, void*);
 typedef HashValue (*H_FUN)(void*);
 typedef void* (*HALLOC_FUN)(void*);
 typedef void (*HFREE_FUN)(void*);
+/* Meta functions */
+typedef void* (*HMALLOC_FUN)(int,size_t);
+typedef void (*HMFREE_FUN)(int,void*);
+typedef int (*HMPRINT_FUN)(int,void*,char*, ...);
 
 /*
 ** This bucket must be placed in top of 
@@ -55,6 +58,9 @@ typedef struct hash_functions
     HCMP_FUN cmp;
     HALLOC_FUN alloc;
     HFREE_FUN free;
+    HMALLOC_FUN meta_alloc;
+    HMFREE_FUN meta_free;
+    HMPRINT_FUN meta_print;
 } HashFunctions;
 
 typedef struct {
@@ -65,11 +71,11 @@ typedef struct {
   int   depth;
 } HashInfo;
 
-typedef struct hash
+struct hash
 {
     HashFunctions fun;   /* Function block */
     int is_allocated;    /* 0 iff hash structure is on stack or is static */
-    ErtsAlcType_t type;
+    int meta_alloc_type; /* argument to pass to meta_alloc and meta_free */
     char* name;          /* Table name (static string, for debugging) */
     int size;		 /* Number of slots */
     int size20percent;   /* 20 percent of number of slots */
@@ -77,10 +83,10 @@ typedef struct hash
     int ix;              /* Size index in size table */
     int used;		 /* Number of slots used */
     HashBucket** bucket; /* Vector of bucket pointers (objects) */
-} Hash;
+};
 
-Hash* hash_new(ErtsAlcType_t, char*, int, HashFunctions);
-Hash* hash_init(ErtsAlcType_t, Hash*, char*, int, HashFunctions);
+Hash* hash_new(int, char*, int, HashFunctions);
+Hash* hash_init(int, Hash*, char*, int, HashFunctions);
 
 void  hash_delete(Hash*);
 void  hash_get_info(HashInfo*, Hash*);
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 14fab41026..99b2bdfab0 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -30,6 +30,10 @@
 #include "hash.h"
 #endif
 
+#ifndef ERL_ALLOC_H__
+#include "erl_alloc.h"
+#endif
+
 typedef struct index_slot 
 {
     HashBucket bucket;
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 86dd3b5aac..2db6f957c6 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -103,6 +103,9 @@ void init_module_table(void)
     f.cmp  = (HCMP_FUN) module_cmp;
     f.alloc = (HALLOC_FUN) module_alloc;
     f.free = (HFREE_FUN) module_free;
+    f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+    f.meta_free = (HMFREE_FUN) erts_free;
+    f.meta_print = (HMPRINT_FUN) erts_print;
 
     for (i = 0; i < ERTS_NUM_CODE_IX; i++) {
 	erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code",
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 7ade8bca0f..fdb6cbc813 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -151,6 +151,9 @@ void init_register_table(void)
     f.cmp  = (HCMP_FUN) reg_cmp;
     f.alloc = (HALLOC_FUN) reg_alloc;
     f.free = (HFREE_FUN) reg_free;
+    f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+    f.meta_free = (HMFREE_FUN) erts_free;
+    f.meta_print = (HMPRINT_FUN) erts_print;
 
     hash_init(ERTS_ALC_T_REG_TABLE, &process_reg, "process_reg",
 	      PREG_HASH_SIZE, f);
-- 
cgit v1.2.3


From 14c7fefd51be035a44bfe42127fb4b9df92d760b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 6 Jul 2015 17:13:52 +0200
Subject: erts: Create forker process for spawn driver

Instead of forking from the beam process, we create a separate
process in which all forks are done. This has several advantages:

1) performance:
  * don't have to close all fd's in the world
  * fork only has to copy stuff from a small process
  * work is done in a completely seperate process
  * a 3x performance increase has been measured,
    can be made even greater (10x) if we cache the
    environment in child setup

2) stability
  * the exec is done in another process than beam, which means that
    if the file that we exec to is on an nfs that is not available
    right now we will not block a scheduler until the nfs returns.

3) simplicity
  * don't have to deal with SIGCHLD in the erts

Unfortunately, this solution also implies some badness.

1) There will always be a seperate process running together with
   beam on unix. This could be confusing and undesirable.
2) We have to transfer the entire environment to child_setup
   for each command.

OTP-13088
---
 erts/emulator/beam/erl_init.c       |  1 +
 erts/emulator/beam/erl_lock_check.c |  4 +--
 erts/emulator/beam/erl_port.h       |  7 +++-
 erts/emulator/beam/erl_process.c    | 37 --------------------
 erts/emulator/beam/erl_process.h    |  6 ----
 erts/emulator/beam/global.h         |  1 +
 erts/emulator/beam/io.c             | 67 ++++++++++++++++++++++++++++++++-----
 erts/emulator/beam/sys.h            | 10 +-----
 8 files changed, 69 insertions(+), 64 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 5c209a4af2..69c0de81be 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -388,6 +388,7 @@ erl_init(int ncpu,
     erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread
 			      initializations */
 #endif
+    erl_sys_late_init();
 #ifdef HIPE
     hipe_mode_switch_init(); /* Must be after init_load/beam_catches/init */
 #endif
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 84bee976ff..34c0144cbc 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -113,9 +113,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"environ",				NULL			},
 #endif
     {	"efile_drv",				"address"		},
-#if defined(ENABLE_CHILD_WAITER_THREAD) || defined(ERTS_SMP)
-    {	"child_status",				NULL			},
-#endif
     {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
     {	"safe_hash",				"address"		},
@@ -187,6 +184,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #ifdef ERTS_SMP
     {	"os_monotonic_time",			NULL			},
 #endif
+    {   "forker_hash_mtx",                      NULL                    },
     {	"erts_alloc_hard_debug",		NULL			},
     {	"hard_dbg_mseg",		        NULL	                },
     {	"erts_mmap",				NULL			}
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index bc4b412594..fa97707a87 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -692,7 +692,7 @@ erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep)
     Port *prt = ERTS_ErlDrvPort2Port(drvport);
     erts_aint32_t state;
     ASSERT(prt);
-    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
+//    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return ERTS_INVALID_ERL_DRV_PORT;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
@@ -949,4 +949,9 @@ ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm
 ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
 ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);
 
+/*
+ * Signals from ports to ports. Used by sys drivers.
+ */
+int erl_drv_port_control(Eterm, char, char*, ErlDrvSizeT);
+
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 15a6d5d651..03cb3dc254 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -497,9 +497,6 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 #if HAVE_ERTS_MSEG
     valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
 #endif
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-    valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
-#endif
 #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS
     valid |= ERTS_SSI_AUX_WORK_REAP_PORTS;
 #endif
@@ -586,8 +583,6 @@ erts_pre_init_process(void)
 	= "MISC_THR_PRGR";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX]
 	= "MISC";
-    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX]
-	= "CHECK_CHILDREN";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX]
 	= "SET_TMO";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX]
@@ -2100,34 +2095,6 @@ erts_debug_wait_completed(Process *c_p, int flags)
 }
 
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-void
-erts_smp_notify_check_children_needed(void)
-{
-    int i;
-    for (i = 0; i < erts_no_schedulers; i++)
-	set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i),
-				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    for (i = 0; i < erts_no_dirty_cpu_schedulers; i++)
-	set_aux_work_flags_wakeup_nob(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(i),
-				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
-    for (i = 0; i < erts_no_dirty_io_schedulers; i++)
-	set_aux_work_flags_wakeup_nob(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(i),
-				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
-#endif
-}
-
-static ERTS_INLINE erts_aint32_t
-handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
-{
-    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
-    erts_check_children();
-    return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
-}
-
-#endif
-
 static void
 notify_reap_ports_relb(void)
 {
@@ -2281,10 +2248,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
 		    handle_misc_aux_work);
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN,
-		    handle_check_children);
-#endif
 
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
 		    handle_setup_aux_work_timer);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index e7c5614b9c..ef58fef102 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -293,7 +293,6 @@ typedef enum {
     ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
     ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
     ERTS_SSI_AUX_WORK_MISC_IX,
-    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
     ERTS_SSI_AUX_WORK_SET_TMO_IX,
     ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
     ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
@@ -326,8 +325,6 @@ typedef enum {
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
 #define ERTS_SSI_AUX_WORK_MISC \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
-#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
-    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
 #define ERTS_SSI_AUX_WORK_SET_TMO \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
 #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
@@ -1643,11 +1640,8 @@ Eterm erts_multi_scheduling_blockers(Process *);
 void erts_start_schedulers(void);
 void erts_alloc_notify_delayed_dealloc(int);
 void erts_alloc_ensure_handle_delayed_dealloc_call(int);
-#ifdef ERTS_SMP
 void erts_notify_canceled_timer(ErtsSchedulerData *, int);
 #endif
-void erts_smp_notify_check_children_needed(void);
-#endif
 #if ERTS_USE_ASYNC_READY_Q
 void erts_notify_check_async_ready_queue(void *);
 #endif
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b4d02dd1dd..4352d5758b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1322,6 +1322,7 @@ extern void erts_match_prog_foreach_offheap(Binary *b,
 
 extern erts_driver_t vanilla_driver;
 extern erts_driver_t spawn_driver;
+extern erts_driver_t forker_driver;
 extern erts_driver_t fd_driver;
 
 int erts_beam_jump_table(void);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 2a3759212e..6ec7a9ba1e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -54,6 +54,7 @@
 extern ErlDrvEntry fd_driver_entry;
 extern ErlDrvEntry vanilla_driver_entry;
 extern ErlDrvEntry spawn_driver_entry;
+extern ErlDrvEntry forker_driver_entry;
 extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */
 
 erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */
@@ -71,6 +72,7 @@ const Port erts_invalid_port = {{ERTS_INVALID_PORT}};
 
 erts_driver_t vanilla_driver;
 erts_driver_t spawn_driver;
+erts_driver_t forker_driver;
 erts_driver_t fd_driver;
 
 int erts_port_synchronous_ops = 0;
@@ -306,12 +308,9 @@ static Port *create_port(char *name,
     size_t port_size, busy_port_queue_size, size;
     erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED;
     erts_aint32_t x_pts_flgs = 0;
-#ifdef DEBUG
-    /* Make sure the debug flags survives until port is freed */
-    state |= ERTS_PORT_SFLG_PORT_DEBUG;
-#endif
 
 #ifdef ERTS_SMP
+    ErtsRunQueue *runq;
     if (!driver_lock) {
 	/* Align size for mutex following port struct */
 	port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port));
@@ -321,6 +320,12 @@ static Port *create_port(char *name,
 #endif
 	port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port));
 
+#ifdef DEBUG
+    /* Make sure the debug flags survives until port is freed */
+    state |= ERTS_PORT_SFLG_PORT_DEBUG;
+#endif
+
+
     busy_port_queue_size
 	= ((driver->flags & ERL_DRV_FLAG_NO_BUSY_MSGQ)
 	   ? 0
@@ -356,8 +361,12 @@ static Port *create_port(char *name,
 	p += sizeof(erts_mtx_t);
 	state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
     }
-    erts_smp_atomic_set_nob(&prt->run_queue,
-			    (erts_aint_t) erts_get_runq_current(NULL));
+    if (erts_get_scheduler_data())
+        runq = erts_get_runq_current(NULL);
+    else
+        runq = ERTS_RUNQ_IX(0);
+    erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
+
     prt->xports = NULL;
 #else
     erts_atomic32_init_nob(&prt->refc, 1);
@@ -1535,6 +1544,26 @@ erts_schedule_proc2port_signal(Process *c_p,
     return ERTS_PORT_OP_SCHEDULED;
 }
 
+static int
+erts_schedule_port2port_signal(Eterm port_num, ErtsProc2PortSigData *sigdp,
+                               int task_flags,
+                               ErtsProc2PortSigCallback callback)
+{
+    Port *prt = erts_port_lookup_raw(port_num);
+
+    if (!prt)
+        return -1;
+
+    sigdp->caller = ERTS_INVALID_PID;
+
+    return erts_port_task_schedule(prt->common.id,
+                                   NULL,
+                                   ERTS_PORT_TASK_PROC_SIG,
+                                   sigdp,
+                                   callback,
+                                   task_flags);
+}
+
 static ERTS_INLINE void
 send_badsig(Port *prt) {
     ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
@@ -2862,6 +2891,7 @@ void erts_init_io(int port_tab_size,
     init_driver(&fd_driver, &fd_driver_entry, NULL);
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
     init_driver(&spawn_driver, &spawn_driver_entry, NULL);
+    init_driver(&forker_driver, &forker_driver_entry, NULL);
     erts_init_static_drivers();
     for (dp = driver_tab; *dp != NULL; dp++)
 	erts_add_driver_entry(*dp, NULL, 1);
@@ -2923,6 +2953,7 @@ void erts_lcnt_enable_io_lock_count(int enable) {
 
     lcnt_enable_drv_lock_count(&vanilla_driver, enable);
     lcnt_enable_drv_lock_count(&spawn_driver, enable);
+    lcnt_enable_drv_lock_count(&forker_driver, enable);
     lcnt_enable_drv_lock_count(&fd_driver, enable);
     /* enable lock counting in all drivers */
     for (dp = driver_list; dp; dp = dp->next) {
@@ -3964,7 +3995,7 @@ port_sig_control(Port *prt,
 	    Uint hsz, rsz;
 	    int control_flags;
 
-	    rp = erts_proc_lookup_raw(sigdp->caller);
+	    rp = sigdp->caller == ERTS_INVALID_PID ? NULL : erts_proc_lookup_raw(sigdp->caller);
 	    if (!rp)
 		goto done;
 
@@ -4007,7 +4038,8 @@ port_sig_control(Port *prt,
 
     /* failure */
 
-    port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+    if (sigdp->caller != ERTS_INVALID_PID)
+        port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
 
 done:
 
@@ -4017,6 +4049,23 @@ done:
     return ERTS_PORT_REDS_CONTROL;
 }
 
+/*
+ * This is an asynchronous control call. I.e. it will not return anything
+ * to the caller.
+ */
+int
+erl_drv_port_control(Eterm port_num, char cmd, char* buff, ErlDrvSizeT size)
+{
+    ErtsProc2PortSigData *sigdp = erts_port_task_alloc_p2p_sig_data();
+
+    sigdp->flags = ERTS_P2P_SIG_TYPE_CONTROL | ERTS_P2P_SIG_DATA_FLG_REPLY;
+    sigdp->u.control.binp = NULL;
+    sigdp->u.control.command = cmd;
+    sigdp->u.control.bufp = buff;
+    sigdp->u.control.size = size;
+
+    return erts_schedule_port2port_signal(port_num, sigdp, 0, port_sig_control);
+}
 
 ErtsPortOpResult
 erts_port_control(Process* c_p,
@@ -4794,6 +4843,8 @@ print_port_info(Port *p, int to, void *arg)
 	erts_print(to, arg, "Port is a file: %s\n",p->name);
     } else if (p->drv_ptr == &spawn_driver) {
 	erts_print(to, arg, "Port controls external process: %s\n",p->name);
+    } else if (p->drv_ptr == &forker_driver) {
+	erts_print(to, arg, "Port controls forker process: %s\n",p->name);
     } else {
 	erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name);
     }
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 34011147d9..522d899c94 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -61,15 +61,6 @@
 #  define NO_FPE_SIGNALS
 #endif
 
-#ifdef DISABLE_CHILD_WAITER_THREAD
-#undef ENABLE_CHILD_WAITER_THREAD
-#endif
-
-#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD)
-#undef ENABLE_CHILD_WAITER_THREAD
-#define ENABLE_CHILD_WAITER_THREAD 1
-#endif
-
 #define ERTS_I64_LITERAL(X) X##LL
 
 #if defined (__WIN32__)
@@ -731,6 +722,7 @@ void erts_sys_main_thread(void);
 extern int erts_sys_prepare_crash_dump(int secs);
 extern void erts_sys_pre_init(void);
 extern void erl_sys_init(void);
+extern void erl_sys_late_init(void);
 extern void erl_sys_args(int *argc, char **argv);
 extern void erl_sys_schedule(int);
 void sys_tty_reset(int);
-- 
cgit v1.2.3


From 05823de18bd48b70b398a6b6fd756a0f71587283 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 27 Aug 2015 10:53:07 +0200
Subject: erts: Fix forker driver ifdefs for win32

---
 erts/emulator/beam/io.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 6ec7a9ba1e..fc247a3260 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -54,7 +54,9 @@
 extern ErlDrvEntry fd_driver_entry;
 extern ErlDrvEntry vanilla_driver_entry;
 extern ErlDrvEntry spawn_driver_entry;
+#ifndef __WIN32__
 extern ErlDrvEntry forker_driver_entry;
+#endif
 extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */
 
 erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */
@@ -72,7 +74,9 @@ const Port erts_invalid_port = {{ERTS_INVALID_PORT}};
 
 erts_driver_t vanilla_driver;
 erts_driver_t spawn_driver;
+#ifndef __WIN32__
 erts_driver_t forker_driver;
+#endif
 erts_driver_t fd_driver;
 
 int erts_port_synchronous_ops = 0;
@@ -2891,7 +2895,9 @@ void erts_init_io(int port_tab_size,
     init_driver(&fd_driver, &fd_driver_entry, NULL);
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
     init_driver(&spawn_driver, &spawn_driver_entry, NULL);
+#ifndef __WIN32__
     init_driver(&forker_driver, &forker_driver_entry, NULL);
+#endif
     erts_init_static_drivers();
     for (dp = driver_tab; *dp != NULL; dp++)
 	erts_add_driver_entry(*dp, NULL, 1);
@@ -2953,7 +2959,9 @@ void erts_lcnt_enable_io_lock_count(int enable) {
 
     lcnt_enable_drv_lock_count(&vanilla_driver, enable);
     lcnt_enable_drv_lock_count(&spawn_driver, enable);
+#ifndef __WIN32__
     lcnt_enable_drv_lock_count(&forker_driver, enable);
+#endif
     lcnt_enable_drv_lock_count(&fd_driver, enable);
     /* enable lock counting in all drivers */
     for (dp = driver_list; dp; dp = dp->next) {
@@ -4843,8 +4851,10 @@ print_port_info(Port *p, int to, void *arg)
 	erts_print(to, arg, "Port is a file: %s\n",p->name);
     } else if (p->drv_ptr == &spawn_driver) {
 	erts_print(to, arg, "Port controls external process: %s\n",p->name);
+#ifndef __WIN32__
     } else if (p->drv_ptr == &forker_driver) {
 	erts_print(to, arg, "Port controls forker process: %s\n",p->name);
+#endif
     } else {
 	erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name);
     }
-- 
cgit v1.2.3


From 0ad8c5f46bc0173c09fa5e7e91f917de82389068 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Fri, 11 Sep 2015 16:35:48 +0200
Subject: erts: Move os_pid to port hash to child setup

Had to move the hashing because of a race that can otherwise happen
where a new os_pid value was inserted into the hash before the
previous value had been removed.

Also replaced the protocol inbetween erts and child setup to be
a binary protocol. This was done in order to deal with the varying
size of Eterm.
---
 erts/emulator/beam/erl_lock_check.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 34c0144cbc..f7b4bd8041 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -184,7 +184,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #ifdef ERTS_SMP
     {	"os_monotonic_time",			NULL			},
 #endif
-    {   "forker_hash_mtx",                      NULL                    },
     {	"erts_alloc_hard_debug",		NULL			},
     {	"hard_dbg_mseg",		        NULL	                },
     {	"erts_mmap",				NULL			}
-- 
cgit v1.2.3


From e56d78b761680d83ab61045a62d4fc7bdb312b3c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 28 Sep 2015 11:02:40 +0200
Subject: erts: Fix memory leak at async open port

When an async open port fails early, it would sometimes
leak memory.
---
 erts/emulator/beam/io.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index fc247a3260..df8a213a1d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -479,6 +479,11 @@ erts_port_free(Port *prt)
 
     erts_port_task_fini_sched(&prt->sched);
 
+    if (prt->async_open_port) {
+        erts_free(ERTS_ALC_T_PRTSD, prt->async_open_port);
+        prt->async_open_port = NULL;
+    }
+
 #ifdef ERTS_SMP
     ASSERT(prt->lock);
     if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK)
-- 
cgit v1.2.3


From b96e62d346eea60b2300dce22d8d892387de4681 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 14 Oct 2015 09:17:58 +0200
Subject: erts: It is not possible to exit the forker driver

---
 erts/emulator/beam/io.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index df8a213a1d..8d9199162c 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2408,6 +2408,11 @@ erts_port_exit(Process *c_p,
 		      | ERTS_PORT_SIG_FLG_BROKEN_LINK
 		      | ERTS_PORT_SIG_FLG_FORCE_SCHED)) == 0);
 
+#ifndef __WIN32__
+    if (prt->drv_ptr == &forker_driver)
+        return ERTS_PORT_OP_DROPPED;
+#endif
+
     if (!(flags & ERTS_PORT_SIG_FLG_FORCE_SCHED)) {
 	ErtsTryImmDrvCallState try_call_state
 	    = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p,
-- 
cgit v1.2.3


From 5801defc866e34c6effd49b9dec995b5dac164f3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Dec 2015 16:21:59 +0100
Subject: erts: Refactor proc dict with 'usedSlots'

which is same as old homeSize + splitPosition.
---
 erts/emulator/beam/erl_process_dict.c | 50 ++++++++++++++++++++---------------
 erts/emulator/beam/erl_process_dict.h |  4 +--
 2 files changed, 31 insertions(+), 23 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 97a2828b4e..b3d7627f08 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -51,7 +51,7 @@
 #define INITIAL_SIZE        (erts_pd_initial_size)
 
 /* Hash utility macros  */
-#define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition)
+#define HASH_RANGE(PDict) ((PDict)->usedSlots)
 
 #define MAKE_HASH(Term)                                \
     ((is_small(Term)) ? unsigned_val(Term) :           \
@@ -164,9 +164,9 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd)
     /*PD_CHECK(pd);*/
     if (pd == NULL)
 	return;
-    erts_print(to, to_arg, "(size = %d, homeSize = %d, "
+    erts_print(to, to_arg, "(size = %d, usedSlots = %d, "
 	       "splitPosition = %d, numElements = %d)\n",
-	       pd->size, pd->homeSize,
+	       pd->arraySize, pd->usedSlots,
 	       pd->splitPosition, (unsigned int) pd->numElements);
     for (i = 0; i < HASH_RANGE(pd); ++i) {
 	erts_print(to, to_arg, "%d: %T\n", i, ARRAY_GET(pd, i));
@@ -584,8 +584,8 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
     if (p->dictionary == NULL) {
 	/* Create it */
         ensure_array_size(&p->dictionary, INITIAL_SIZE);
-	p->dictionary->homeSize = INITIAL_SIZE;
-        p->dictionary->sizeMask = p->dictionary->homeSize*2 - 1;
+	p->dictionary->usedSlots = INITIAL_SIZE;
+        p->dictionary->sizeMask = INITIAL_SIZE*2 - 1;
         p->dictionary->splitPosition = 0;
         p->dictionary->numElements = 0;
     }	
@@ -737,12 +737,13 @@ static void shrink(Process *p, Eterm* ret)
 
     for (i = 0; i < steps; ++i) {
 	if (pd->splitPosition == 0) {
-            pd->sizeMask = pd->homeSize - 1;
-	    pd->homeSize /= 2;
-	    pd->splitPosition = pd->homeSize;
+            ASSERT(IS_POW2(pd->usedSlots));
+            pd->sizeMask = pd->usedSlots - 1;
+	    pd->splitPosition = pd->usedSlots / 2;
 	}
 	--(pd->splitPosition);
-	hi = ARRAY_GET(pd, (pd->splitPosition + pd->homeSize));
+        /* Must wait to decrement 'usedSlots' for GC rootset below */
+	hi = ARRAY_GET(pd, pd->usedSlots - 1);
 	lo = ARRAY_GET(pd, pd->splitPosition);
 	if (hi != NIL) {
 	    if (lo == NIL) {
@@ -754,7 +755,7 @@ static void shrink(Process *p, Eterm* ret)
 		}
 		if (HeapWordsLeft(p) < needed) {
 		    BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1));
-		    hi = pd->data[(pd->splitPosition + pd->homeSize)];
+		    hi = pd->data[pd->usedSlots - 1];
 		    lo = pd->data[pd->splitPosition];
 		}
 #ifdef DEBUG
@@ -796,7 +797,8 @@ static void shrink(Process *p, Eterm* ret)
 		}
 	    }
 	}
-	ARRAY_PUT(pd, (pd->splitPosition + pd->homeSize), NIL);
+        --pd->usedSlots;
+	ARRAY_PUT(pd, pd->usedSlots, NIL);
     }
     if (HASH_RANGE(p->dictionary) <= (p->dictionary->size / 4)) {
 	array_shrink(&(p->dictionary), (HASH_RANGE(p->dictionary) * 3) / 2);
@@ -806,7 +808,7 @@ static void shrink(Process *p, Eterm* ret)
 static void grow(Process *p)
 {
     unsigned int i,j;
-    unsigned int steps = p->dictionary->homeSize / 5;
+    unsigned int steps = (p->dictionary->usedSlots / 4) & 0xf;
     Eterm l1,l2;
     Eterm l;
     Eterm *hp;
@@ -835,7 +837,7 @@ static void grow(Process *p)
      */
 
     pos = pd->splitPosition;
-    homeSize = pd->homeSize;
+    homeSize = pd->usedSlots - pd->splitPosition;
     for (i = 0; i < steps; ++i) {
 	if (pos == homeSize) {
 	    homeSize *= 2;
@@ -860,19 +862,21 @@ static void grow(Process *p)
     /*
      * Now grow.
      */
-
+    homeSize = pd->usedSlots - pd->splitPosition;
     for (i = 0; i < steps; ++i) {
-	if (pd->splitPosition == pd->homeSize) {
-	    pd->homeSize *= 2;
-            pd->sizeMask = pd->homeSize*2 - 1;
+	if (pd->splitPosition == homeSize) {
+	    homeSize *= 2;
+            pd->sizeMask = homeSize*2 - 1;
             pd->splitPosition = 0;
 	}
 	pos = pd->splitPosition;
 	++pd->splitPosition; /* For the hashes */
+        ++pd->usedSlots;
+        ASSERT(pos + homeSize == pd->usedSlots - 1);
 	l = ARRAY_GET(pd, pos);
 	if (is_tuple(l)) {
 	    if (pd_hash_value(pd, tuple_val(l)[1]) != pos) {
-		ARRAY_PUT(pd, pos + pd->homeSize, l);
+		ARRAY_PUT(pd, pos + homeSize, l);
 		ARRAY_PUT(pd, pos, NIL);
 	    }
 	} else {
@@ -896,7 +900,7 @@ static void grow(Process *p)
 		l2 = TCAR(l2);
 	    ASSERT(hp <= hp_limit);
 	    ARRAY_PUT(pd, pos, l1);
-	    ARRAY_PUT(pd, pos + pd->homeSize, l2);
+	    ARRAY_PUT(pd, pos + homeSize, l2);
 	}
     }
 
@@ -961,7 +965,9 @@ static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx)
 { 
     Uint high;
 
-    ASSERT(IS_POW2(pdict->homeSize));
+    ASSERT(IS_POW2(pdict->sizeMask+1));
+    ASSERT(HASH_RANGE(pdict) >= (pdict->sizeMask >> 1));
+    ASSERT(HASH_RANGE(pdict) <= (pdict->sizeMask + 1));
 
     high = hx & pdict->sizeMask;
     if (high >= HASH_RANGE(pdict))
@@ -1043,12 +1049,14 @@ static void pd_check(ProcDict *pd)
 	} else if (is_tuple(t)) {
 	    ++num;
 	    ASSERT(arityval(*tuple_val(t)) == 2);
+            ASSERT(pd_hash_value(pd, tuple_val(t)[1]) == i);
 	    continue;
 	} else if (is_list(t)) {
 	    while (t != NIL) {
 		++num;
 		ASSERT(is_tuple(TCAR(t)));
 		ASSERT(arityval(*(tuple_val(TCAR(t)))) == 2);
+                ASSERT(pd_hash_value(pd, tuple_val(TCAR(t))[1]) == i);
 		t = TCDR(t);
 	    }
 	    continue;
@@ -1059,7 +1067,7 @@ static void pd_check(ProcDict *pd)
 	}
     }
     ASSERT(num == pd->numElements);
-    ASSERT(pd->splitPosition <= pd->homeSize);
+    ASSERT(pd->usedSlots >= pd->splitPosition*2);
 }
 
 #endif /* DEBUG */
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index fd59c969cf..f0d57da1e6 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -25,14 +25,14 @@
 typedef struct proc_dict {
     unsigned int sizeMask;
     unsigned int size;
-    unsigned int homeSize;
+    unsigned int usedSlots;
     unsigned int splitPosition;
     Uint numElements;
     Eterm data[1]; /* The beginning of an array of erlang terms */
 } ProcDict;
 
 #define ERTS_PD_START(PD) ((PD)->data)
-#define ERTS_PD_SIZE(PD)  ((PD)->homeSize + (PD)->splitPosition)
+#define ERTS_PD_SIZE(PD)  ((PD)->usedSlots)
 
 int erts_pd_set_initial_size(int size);
 Uint erts_dicts_mem_size(struct process *p);
-- 
cgit v1.2.3


From 0f32d250876e7bb226fb96e07fb31734ba7d16f2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Dec 2015 17:47:37 +0100
Subject: erts: Rename proc dict size to arraySize

for naming style consistency.
---
 erts/emulator/beam/erl_process_dict.c | 26 +++++++++++++-------------
 erts/emulator/beam/erl_process_dict.h |  2 +-
 2 files changed, 14 insertions(+), 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index b3d7627f08..e384f4c3f6 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -77,9 +77,9 @@
 #define TCDR(Term) CDR(list_val(Term))
 
 /* Array access macro */ 
-#define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->size), \
+#define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \
 				 (PDict)->data[Index])
-#define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->size), \
+#define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->arraySize), \
                                       (PDict)->data[Index] = (Val))
 
 #define IS_POW2(X) ((X) && !((X) & ((X)-1)))
@@ -221,7 +221,7 @@ erts_dicts_mem_size(Process *p)
 {
     Uint size = 0;
     if (p->dictionary)
-	size += PD_SZ2BYTES(p->dictionary->size);
+	size += PD_SZ2BYTES(p->dictionary->arraySize);
     return size;
 }
 
@@ -800,7 +800,7 @@ static void shrink(Process *p, Eterm* ret)
         --pd->usedSlots;
 	ARRAY_PUT(pd, pd->usedSlots, NIL);
     }
-    if (HASH_RANGE(p->dictionary) <= (p->dictionary->size / 4)) {
+    if (HASH_RANGE(p->dictionary) <= (p->dictionary->arraySize / 4)) {
 	array_shrink(&(p->dictionary), (HASH_RANGE(p->dictionary) * 3) / 2);
     }
 }
@@ -918,16 +918,16 @@ static void array_shrink(ProcDict **ppd, unsigned int need)
     unsigned int siz = next_array_size(need);
 
     HDEBUGF(("array_shrink: size = %d, need = %d",
-	     (*ppd)->size, need));
+	     (*ppd)->arraySize, need));
 
-    if (siz >= (*ppd)->size)
+    if (siz >= (*ppd)->arraySize)
 	return; /* Only shrink */
 
     *ppd = PD_REALLOC(((void *) *ppd),
-		      PD_SZ2BYTES((*ppd)->size),
+		      PD_SZ2BYTES((*ppd)->arraySize),
 		      PD_SZ2BYTES(siz));
 
-    (*ppd)->size = siz;
+    (*ppd)->arraySize = siz;
 }
     
 			
@@ -942,17 +942,17 @@ static void ensure_array_size(ProcDict **ppdict, unsigned int size)
         pd = PD_ALLOC(PD_SZ2BYTES(siz));
 	for (i = 0; i < siz; ++i) 
 	    pd->data[i] = NIL;
-	pd->size = siz;
+	pd->arraySize = siz;
         *ppdict = pd;
-    } else if (size > pd->size) {
-	Uint osize = pd->size;
+    } else if (size > pd->arraySize) {
+	Uint osize = pd->arraySize;
 	Uint nsize = next_array_size(size);
 	pd = PD_REALLOC(((void *) pd),
 			     PD_SZ2BYTES(osize),
 			     PD_SZ2BYTES(nsize));
 	for (i = osize; i < nsize; ++i)
 	    pd->data[i] = NIL;
-	pd->size = nsize;
+	pd->arraySize = nsize;
         *ppdict = pd;
     }
 }
@@ -1040,7 +1040,7 @@ static void pd_check(ProcDict *pd)
     if (pd == NULL)
 	return;
     used = HASH_RANGE(pd);
-    ASSERT(pd->size >= used);
+    ASSERT(pd->arraySize >= used);
     ASSERT(HASH_RANGE(pd) <= MAX_HASH);
     for (i = 0, num = 0; i < used; ++i) {
 	Eterm t = pd->data[i];
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index f0d57da1e6..dac214c8a1 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -24,8 +24,8 @@
 
 typedef struct proc_dict {
     unsigned int sizeMask;
-    unsigned int size;
     unsigned int usedSlots;
+    unsigned int arraySize;
     unsigned int splitPosition;
     Uint numElements;
     Eterm data[1]; /* The beginning of an array of erlang terms */
-- 
cgit v1.2.3


From 46a1a3b8c8819a117d7f48a864d8e0f5e08ac548 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 14 Dec 2015 20:02:54 +0100
Subject: erts: Add 'fill_heap' to erts_debug:state_internal_state

to make it easy to provoke GC inside/after a BIF or instruction.
---
 erts/emulator/beam/erl_bif_info.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b44382cde8..caa1cb7608 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4085,6 +4085,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		BIF_RET(am_ok);
 	    }
 	}
+        else if (ERTS_IS_ATOM_STR("fill_heap", BIF_ARG_1)) {
+            UWord left = HeapWordsLeft(BIF_P);
+            if (left > 1) {
+                Eterm* hp = HAlloc(BIF_P, left);
+                *hp = make_pos_bignum_header(left - 1);
+            }
+            if (BIF_ARG_2 == am_true) {
+                FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
+            }
+            BIF_RET(am_ok);
+        }
     }
 
     BIF_ERROR(BIF_P, BADARG);
-- 
cgit v1.2.3


From 1a101efd5654d3d4a3e96523fbbf4eb871e61d1b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Dec 2015 18:08:41 +0100
Subject: erts: Fix bug in check_process_code for literals

---
 erts/emulator/beam/beam_bif_load.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index c925a8c812..956454d9b3 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -826,6 +826,9 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
     erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
 
+    literals = (char*) modp->old.code_hdr->literals_start;
+    lit_bsize = (char*) modp->old.code_hdr->literals_end - literals;
+
     for (msgp = rp->msg.first; msgp; msgp = msgp->next) {
 	if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
 	    hfrag = &msgp->hfrag;
@@ -839,14 +842,11 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	    /* Should not contain any constants... */
 	    ASSERT(!any_heap_refs(&hfrag->mem[0],
 				  &hfrag->mem[hfrag->used_size],
-				  mod_start,
-				  mod_size));
+                                  literals,
+				  lit_bsize));
 	}
     }
 
-    literals = (char*) modp->old.code_hdr->literals_start;
-    lit_bsize = (char*) modp->old.code_hdr->literals_end - literals;
-
     while (1) {
 
 	/* Check heap, stack etc... */
@@ -881,7 +881,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 
 	    hp = &hfrag->mem[0];
 	    hp_end = &hfrag->mem[hfrag->used_size];
-	    if (any_heap_refs(hp, hp_end, mod_start, lit_bsize))
+	    if (any_heap_refs(hp, hp_end, literals, lit_bsize))
 		goto try_literal_gc;
 	}
 
@@ -902,7 +902,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 
 		hp = &hfrag->mem[0];
 		hp_end = &hfrag->mem[hfrag->used_size];
-		ASSERT(!any_heap_refs(hp, hp_end, mod_start, lit_bsize));
+		ASSERT(!any_heap_refs(hp, hp_end, literals, lit_bsize));
 	    }
 	}
 
-- 
cgit v1.2.3


From 6b60ed6d30dccaeab1207178d5e786aaa14b3201 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 16 Dec 2015 11:14:40 +0100
Subject: Fix offset_mqueue

---
 erts/emulator/beam/erl_gc.c | 48 ++++++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index b63567e563..e182782323 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2773,33 +2773,37 @@ offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
 {
     ErtsMessage* mp = p->msg.first;
 
-    while (mp != NULL) {
-        Eterm mesg = ERL_MESSAGE_TERM(mp);
-	if (is_value(mesg)) {
-	    switch (primary_tag(mesg)) {
-	    case TAG_PRIMARY_LIST:
-	    case TAG_PRIMARY_BOXED:
-		if (ErtsInArea(ptr_val(mesg), area, area_size)) {
-		    ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
+    if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ) {
+
+	while (mp != NULL) {
+	    Eterm mesg = ERL_MESSAGE_TERM(mp);
+	    if (is_value(mesg)) {
+		switch (primary_tag(mesg)) {
+		case TAG_PRIMARY_LIST:
+		case TAG_PRIMARY_BOXED:
+		    if (ErtsInArea(ptr_val(mesg), area, area_size)) {
+			ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
+		    }
+		    break;
 		}
-		break;
 	    }
-	}
-	mesg = ERL_MESSAGE_TOKEN(mp);
-	if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
-	    ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
-        }
+	    mesg = ERL_MESSAGE_TOKEN(mp);
+	    if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
+		ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
+	    }
 #ifdef USE_VM_PROBES
-	mesg = ERL_MESSAGE_DT_UTAG(mp);
-	if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
-	    ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs);
-        }
+	    mesg = ERL_MESSAGE_DT_UTAG(mp);
+	    if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
+		ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs);
+	    }
 #endif	
 	
-        ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) ||
-		is_tuple(ERL_MESSAGE_TOKEN(mp)) ||
-		is_atom(ERL_MESSAGE_TOKEN(mp))));
-        mp = mp->next;
+	    ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) ||
+		    is_tuple(ERL_MESSAGE_TOKEN(mp)) ||
+		    is_atom(ERL_MESSAGE_TOKEN(mp))));
+	    mp = mp->next;
+	}
+
     }
 }
 
-- 
cgit v1.2.3


From 47a7d8b9d701b81355a02f4bad8d16a327bc1588 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Mon, 21 Dec 2015 23:07:10 -0500
Subject: Do not allow aux work on dirty schedulers

The nature of aux work is such that dirty schedulers should not
attempt to perform it. Modify the code to ensure that dirty schedulers
avoid aux work.

Also fix an incorrect assumption about the size of a Uint in the
ErtsDirtySchedId type.
---
 erts/emulator/beam/erl_process.c | 75 +++++++++++++++++++---------------------
 erts/emulator/beam/erl_process.h |  2 +-
 2 files changed, 37 insertions(+), 40 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d583118e7b..eae05f1651 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2239,6 +2239,7 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     erts_aint32_t aux_work = orig_aux_work;
     erts_aint32_t ignore = 0;
 
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
 #ifdef ERTS_SMP
     haw_thr_prgr_current_reset(awdp);
 #endif
@@ -2972,14 +2973,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    ErtsMonotonicTime current_time;
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
-	    if (aux_work) {
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+	    if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		if (!thr_prgr_active) {
 		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 		    sched_wall_time_change(esdp, 1);
 		}
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
-		if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)
-		    && erts_thr_progress_update(esdp))
+		if (aux_work && erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
 	    }
 
@@ -3131,19 +3131,16 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 #endif
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
-	    if (aux_work) {
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		    if (!working)
-			sched_wall_time_change(esdp, working = 1);
+	    if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		if (!working)
+		    sched_wall_time_change(esdp, working = 1);
 #ifdef ERTS_SMP
-		    if (!thr_prgr_active)
-			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+		if (!thr_prgr_active)
+		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 #endif
-		}
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
 #ifdef ERTS_SMP
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work &&
-		    erts_thr_progress_update(esdp))
+		if (aux_work && erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
 #endif
 	    }
@@ -6798,18 +6795,19 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			 & ERTS_RUNQ_FLGS_QMASK);
 		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 		if (aux_work|qmask) {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			sched_wall_time_change(esdp, 1);
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
+			}
+			if (aux_work)
+			    aux_work = handle_aux_work(&esdp->aux_work_data,
+						       aux_work,
+						       1);
+
+			if (aux_work && erts_thr_progress_update(esdp))
+			    erts_thr_progress_leader_update(esdp);
 		    }
-		    if (aux_work)
-			aux_work = handle_aux_work(&esdp->aux_work_data,
-						   aux_work,
-						   1);
-
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
-			(aux_work && erts_thr_progress_update(esdp)))
-			erts_thr_progress_leader_update(esdp);
 		    if (qmask) {
 #ifdef ERTS_DIRTY_SCHEDULERS
 			if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
@@ -7026,17 +7024,18 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			 & ERTS_RUNQ_FLGS_QMASK);
 		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 		if (aux_work|qmask) {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			sched_wall_time_change(esdp, 1);
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
+			}
+			if (aux_work)
+			    aux_work = handle_aux_work(&esdp->aux_work_data,
+						       aux_work,
+						       1);
+			if (aux_work && erts_thr_progress_update(esdp))
+			    erts_thr_progress_leader_update(esdp);
 		    }
-		    if (aux_work)
-			aux_work = handle_aux_work(&esdp->aux_work_data,
-						   aux_work,
-						   1);
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work &&
-			erts_thr_progress_update(esdp))
-			erts_thr_progress_leader_update(esdp);
 		    if (qmask) {
 			erts_smp_runq_lock(esdp->run_queue);
 			evacuate_run_queue(esdp->run_queue, &sbp);
@@ -9407,10 +9406,9 @@ Process *schedule(Process *p, int calls)
 	    suspend_scheduler(esdp);
 #endif
 
-	{
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    erts_aint32_t aux_work;
-	    int leader_update = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0
-		: erts_thr_progress_update(esdp);
+	    int leader_update = erts_thr_progress_update(esdp);
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
 	    if (aux_work | leader_update | ERTS_SCHED_FAIR) {
 		erts_smp_runq_unlock(rq);
@@ -9423,8 +9421,7 @@ Process *schedule(Process *p, int calls)
 		erts_smp_runq_lock(rq);
 	    }
 
-	    ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)
-			       || !erts_thr_progress_is_blocking());
+	    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
 	}
 	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 10c6fa4a67..f5f50b77ba 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -607,7 +607,7 @@ typedef enum {
 typedef union {
     struct {
 	ErtsDirtySchedulerType type: 1;
-	unsigned num: 31;
+	Uint num: sizeof(Uint)*8 - 1;
     } s;
     Uint no;
 } ErtsDirtySchedId;
-- 
cgit v1.2.3


From 77f0f265f860130102acc1db31d99f8dd9e690c5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 22 Dec 2015 18:32:10 +0100
Subject: Use monotonic time for call_time trace

---
 erts/emulator/beam/beam_bp.c | 99 ++++++++++++++------------------------------
 erts/emulator/beam/beam_bp.h |  7 +---
 2 files changed, 32 insertions(+), 74 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 016d0aaa32..9860968687 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -75,6 +75,16 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 erts_smp_atomic32_t erts_active_bp_index;
 erts_smp_atomic32_t erts_staging_bp_index;
 
+/*
+ * Inlined helpers
+ */
+
+static ERTS_INLINE ErtsMonotonicTime
+get_mtime(Process *c_p)
+{
+    return erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(c_p));
+}
+
 /* *************************************************************************
 ** Local prototypes
 */
@@ -97,9 +107,6 @@ static int clear_function_break(BeamInstr *pc, Uint break_flags);
 
 static BpDataTime* get_time_break(BeamInstr *pc);
 static GenericBpData* check_break(BeamInstr *pc, Uint break_flags);
-static void bp_time_diff(bp_data_time_item_t *item,
-			 process_breakpoint_time_t *pbt,
-			 Uint ms, Uint s, Uint us);
 
 static void bp_meta_unref(BpMetaPid* bmp);
 static void bp_count_unref(BpCount* bcp);
@@ -110,13 +117,8 @@ static void uninstall_breakpoint(BeamInstr* pc);
 /* bp_hash */
 #define BP_TIME_ADD(pi0, pi1)                       \
     do {                                            \
-	Uint r;                                     \
 	(pi0)->count   += (pi1)->count;             \
-	(pi0)->s_time  += (pi1)->s_time;            \
-	(pi0)->us_time += (pi1)->us_time;           \
-	r = (pi0)->us_time / 1000000;               \
-	(pi0)->s_time  += r;                        \
-	(pi0)->us_time  = (pi0)->us_time % 1000000; \
+	(pi0)->time    += (pi1)->time;              \
     } while(0)
 
 static void bp_hash_init(bp_time_hash_t *hash, Uint n);
@@ -948,7 +950,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 void
 erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 {
-    Uint ms,s,us;
+    ErtsMonotonicTime time;
     process_breakpoint_time_t *pbt = NULL;
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
@@ -961,7 +963,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
      * from the process psd  */
 
     pbt = ERTS_PROC_GET_CALL_TIME(c_p);
-    get_sys_now(&ms, &s, &us);
+    time = get_mtime(c_p);
 
     /* get pbt
      * timestamp = t0
@@ -976,7 +978,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
     } else {
 	ASSERT(pbt->pc);
 	/* add time to previous code */
-	bp_time_diff(&sitem, pbt, ms, s, us);
+	sitem.time = time - pbt->time;
 	sitem.pid = c_p->common.id;
 	sitem.count = 0;
 
@@ -1002,8 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
     /* Add count to this code */
     sitem.pid     = c_p->common.id;
     sitem.count   = 1;
-    sitem.s_time  = 0;
-    sitem.us_time = 0;
+    sitem.time    = 0;
 
     /* this breakpoint */
     ASSERT(bdt);
@@ -1020,15 +1021,13 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
     }
 
     pbt->pc = I;
-    pbt->ms = ms;
-    pbt->s  = s;
-    pbt->us = us;
+    pbt->time = time;
 }
 
 void
 erts_trace_time_return(Process *p, BeamInstr *pc)
 {
-    Uint ms,s,us;
+    ErtsMonotonicTime time;
     process_breakpoint_time_t *pbt = NULL;
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
@@ -1041,7 +1040,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
      * from the process psd  */
 
     pbt = ERTS_PROC_GET_CALL_TIME(p);
-    get_sys_now(&ms,&s,&us);
+    time = get_mtime(p);
 
     /* get pbt
      * lookup bdt from code
@@ -1057,7 +1056,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 	 */
 	ASSERT(pbt->pc);
 
-	bp_time_diff(&sitem, pbt, ms, s, us);
+	sitem.time = time - pbt->time;
 	sitem.pid   = p->common.id;
 	sitem.count = 0;
 
@@ -1080,9 +1079,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 	}
 
 	pbt->pc = pc;
-	pbt->ms = ms;
-	pbt->s  = s;
-	pbt->us = us;
+	pbt->time = time;
     }
 }
 
@@ -1183,10 +1180,14 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) {
 		for(ix = 0; ix < hash.n; ix++) {
 		    item = &(hash.item[ix]);
 		    if (item->pid != NIL) {
+			ErtsMonotonicTime sec, usec;
+			usec = ERTS_MONOTONIC_TO_USEC(item->time);
+			sec = usec / 1000000;
+			usec = usec - sec*1000000;
 			t = TUPLE4(hp, item->pid,
 				make_small(item->count),
-				make_small(item->s_time),
-				make_small(item->us_time));
+				   make_small((Uint) sec),
+				   make_small((Uint) usec));
 			hp += 5;
 			*retval = CONS(hp, t, *retval); hp += 2;
 		    }
@@ -1266,8 +1267,7 @@ static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) {
 	    }
 	    item[hval].pid     = hash->item[ix].pid;
 	    item[hval].count   = hash->item[ix].count;
-	    item[hval].s_time  = hash->item[ix].s_time;
-	    item[hval].us_time = hash->item[ix].us_time;
+	    item[hval].time    = hash->item[ix].time;
 	}
     }
 
@@ -1315,8 +1315,7 @@ static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_da
     item = &(hash->item[hval]);
 
     item->pid     = sitem->pid;
-    item->s_time  = sitem->s_time;
-    item->us_time = sitem->us_time;
+    item->time    = sitem->time;
     item->count   = sitem->count;
     hash->used++;
 
@@ -1330,41 +1329,7 @@ static void bp_hash_delete(bp_time_hash_t *hash) {
     hash->item = NULL;
 }
 
-static void bp_time_diff(bp_data_time_item_t *item, /* out */
-	process_breakpoint_time_t *pbt,             /* in  */
-	Uint ms, Uint s, Uint us) {
-    int ds,dus;
-#ifdef DEBUG
-    int dms;
-
-
-    dms = ms - pbt->ms;
-#endif
-    ds  = s  - pbt->s;
-    dus = us - pbt->us;
-
-    /* get_sys_now may return zero difftime,
-     * this is ok.
-     */
-
-#ifdef DEBUG
-    ASSERT(dms >= 0 || ds >= 0 || dus >= 0);
-#endif
-
-    if (dus < 0) {
-	dus += 1000000;
-	ds  -= 1;
-    }
-    if (ds < 0) {
-	ds += 1000000;
-    }
-
-    item->s_time  = ds;
-    item->us_time = dus;
-}
-
 void erts_schedule_time_break(Process *p, Uint schedule) {
-    Uint ms, s, us;
     process_breakpoint_time_t *pbt = NULL;
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
@@ -1387,8 +1352,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 
 	    pbdt = get_time_break(pbt->pc);
 	    if (pbdt) {
-		get_sys_now(&ms,&s,&us);
-		bp_time_diff(&sitem, pbt, ms, s, us);
+		sitem.time = get_mtime(p) - pbt->time;
 		sitem.pid   = p->common.id;
 		sitem.count = 0;
 
@@ -1410,10 +1374,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 	     * timestamp it and remove the previous
 	     * timestamp in the psd.
 	     */
-	    get_sys_now(&ms,&s,&us);
-	    pbt->ms = ms;
-	    pbt->s  = s;
-	    pbt->us = us;
+	    pbt->time = get_mtime(p);
 	    break;
 	default :
 	    ASSERT(0);
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 97d0539ac7..2b89d6fc71 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -29,8 +29,7 @@
 typedef struct {
     Eterm pid;
     Sint  count;
-    Uint  s_time;
-    Uint  us_time;
+    ErtsMonotonicTime time;
 } bp_data_time_item_t;
 
 typedef struct {
@@ -46,9 +45,7 @@ typedef struct bp_data_time {     /* Call time */
 } BpDataTime;
 
 typedef struct {
-    Uint ms;
-    Uint s;
-    Uint us;
+    ErtsMonotonicTime time;
     BeamInstr *pc;
 } process_breakpoint_time_t; /* used within psd */
 
-- 
cgit v1.2.3


From 172d3bf7b28b28f3ac6ecd2348f1d8cd8db7ff7a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 29 Dec 2015 15:26:00 +0100
Subject: Fix asynchronous BIF timer cancellation message reply

---
 erts/emulator/beam/erl_hl_timer.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 6853278828..734057a12c 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -2018,7 +2018,7 @@ bif_timer_access_request(void *vreq)
 static int
 try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 			    Process *c_p, Uint32 sid,
-			    Eterm tref, Uint32 *trefn,
+			    Uint32 *trefn,
 			    int async, int cancel,
 			    int info, Eterm *resp)
 {
@@ -2078,13 +2078,13 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
     }
     else {
 	ErtsMessage *mp;
-	Eterm tag, res, msg;
+	Eterm tag, res, msg, tref;
 	Uint hsz;
 	Eterm *hp;
 	ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
 	ErlOffHeap *ohp;
 
-	hsz = 4;
+	hsz = 4 + REF_THING_SIZE;
 	if (time_left > (Sint64) MAX_SMALL)
 	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
 
@@ -2095,6 +2095,13 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 	else
 	    tag = am_read_timer;
 
+	write_ref_thing(hp,
+			trefn[0],
+			trefn[1],
+			trefn[2]);
+	tref = make_internal_ref(hp);
+	hp += REF_THING_SIZE;
+
 	if (time_left < 0)
 	    res = am_false;
 	else if (time_left <= (Sint64) MAX_SMALL)
@@ -2145,8 +2152,8 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
 				     info);
 	ERTS_BIF_PREP_RET(ret, res);
     }
-    else if (try_access_sched_remote_btm(esdp, c_p, sid,
-					 tref, trefn,
+    else if (try_access_sched_remote_btm(esdp, c_p,
+					 sid, trefn,
 					 async, cancel,
 					 info, &res)) {
 	ERTS_BIF_PREP_RET(ret, res);
-- 
cgit v1.2.3


From 9293a250e02e63d295a841786f03c3e3a3f30e97 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 29 Dec 2015 15:45:20 +0100
Subject: Fix HL timer hard debug implementation

---
 erts/emulator/beam/erl_hl_timer.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 51a0d68247..fb6d249145 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1055,6 +1055,8 @@ create_hl_timer(ErtsSchedulerData *esdp,
     erts_aint32_t refc;
     Uint32 roflgs;
 
+    ERTS_HLT_HDBG_CHK_SRV(srv);
+
     check_canceled_queue(esdp, srv);
 
     ERTS_HLT_ASSERT((esdp->no & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
@@ -1179,8 +1181,6 @@ create_hl_timer(ErtsSchedulerData *esdp,
     erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
     erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE);
 
-    ERTS_HLT_HDBG_CHK_SRV(srv);
-
     if (!srv->next_timeout
 	|| tmr->timeout < srv->next_timeout->timeout) {
 	if (srv->next_timeout)
@@ -3099,7 +3099,8 @@ tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg)
 				& ~ERTS_HLT_PFLGS_MASK);
 	ERTS_HLT_ASSERT(tmr == prnt);
     }
-    ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr);
+    if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)
+	ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr);
     if (tmr->time.tree.same_time) {
 	ErtsHdbgHLT st_hdbg;
 	st_hdbg.srv = hdbg->srv;
-- 
cgit v1.2.3


From 37f58ad6bff2bf2bac4f3f20c2684e8cee66af03 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 15 Dec 2015 09:40:30 +0100
Subject: Light weight statistics of run queue lengths

- statistics(total_run_queue_lengths)
- statistics(run_queue_lengths)
- statistics(total_active_tasks)
- statistics(active_tasks)

Conflicts:
	erts/emulator/beam/erl_process.c
---
 erts/emulator/beam/atom.names     |  4 +++
 erts/emulator/beam/erl_bif_info.c | 37 ++++++++++++++++++++--
 erts/emulator/beam/erl_process.c  | 65 ++++++++++++++++++++++++++-------------
 erts/emulator/beam/erl_process.h  | 48 +++++++++++++++++++----------
 4 files changed, 114 insertions(+), 40 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 190e7817dc..fb3368eae2 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -71,6 +71,7 @@ atom absoluteURI
 atom ac
 atom accessor
 atom active
+atom active_tasks
 atom all
 atom all_but_first
 atom all_names
@@ -512,6 +513,7 @@ atom return_from
 atom return_to
 atom return_trace
 atom run_queue
+atom run_queue_lengths
 atom runnable
 atom runnable_ports
 atom runnable_procs
@@ -579,7 +581,9 @@ atom timeout_value
 atom Times='*'
 atom timestamp
 atom total
+atom total_active_tasks
 atom total_heap_size
+atom total_run_queue_lengths
 atom tpkt
 atom trace trace_ts traced 
 atom trace_control_word
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b44382cde8..414ff6711a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3234,6 +3234,39 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	if (is_non_value(res))
 	    BIF_RET(am_undefined);
 	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
+    } else if (BIF_ARG_1 == am_total_active_tasks
+	       || BIF_ARG_1 == am_total_run_queue_lengths) {
+	Uint no = erts_run_queues_len(NULL, 0, BIF_ARG_1 == am_total_active_tasks);
+	if (IS_USMALL(0, no))
+	    res = make_small(no);
+	else {
+	    Eterm *hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
+	    res = uint_to_big(no, hp);
+	}
+	BIF_RET(res);
+    } else if (BIF_ARG_1 == am_active_tasks
+	       || BIF_ARG_1 == am_run_queue_lengths) {
+	Eterm res, *hp, **hpp;
+	Uint sz, *szp;
+	int no_qs = erts_no_run_queues;
+	Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2);
+	(void) erts_run_queues_len(qszs, 0, BIF_ARG_1 == am_active_tasks);
+	sz = 0;
+	szp = &sz;
+	hpp = NULL;
+	while (1) {
+	    int i;
+	    for (i = 0; i < no_qs; i++)
+		qszs[no_qs+i] = erts_bld_uint(hpp, szp, qszs[i]);
+	    res = erts_bld_list(hpp, szp, no_qs, &qszs[no_qs]);
+	    if (hpp) {
+		erts_free(ERTS_ALC_T_TMP, qszs);
+		BIF_RET(res);
+	    }
+	    hp = HAlloc(BIF_P, sz);
+	    szp = NULL;
+	    hpp = &hp;
+	}
     } else if (BIF_ARG_1 == am_context_switches) {
 	Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P);
 	hp = HAlloc(BIF_P, 3);
@@ -3282,7 +3315,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = TUPLE2(hp, b1, b2);
 	BIF_RET(res);
     } else if (BIF_ARG_1 ==  am_run_queue) {
-	res = erts_run_queues_len(NULL);
+	res = erts_run_queues_len(NULL, 1, 0);
 	BIF_RET(make_small(res));
     } else if (BIF_ARG_1 == am_wall_clock) {
 	UWord w1, w2;
@@ -3302,7 +3335,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	Uint sz, *szp;
 	int no_qs = erts_no_run_queues;
 	Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2);
-	(void) erts_run_queues_len(qszs);
+	(void) erts_run_queues_len(qszs, 0, 0);
 	sz = 0;
 	szp = &sz;
 	hpp = NULL;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d583118e7b..b47aae0f74 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -3149,7 +3149,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    }
 
 #ifndef ERTS_SMP
-	    if (rq->len != 0 || rq->misc.start)
+	    if (erts_smp_atomic32_read_dirty(&rq->len) != 0 || rq->misc.start)
 		goto sys_woken;
 #else
 	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
@@ -3248,7 +3248,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	}
 
 #ifndef ERTS_SMP
-	if (rq->len == 0 && !rq->misc.start)
+	if (erts_smp_atomic32_read_dirty(&rq->len) == 0 && !rq->misc.start)
 	    goto sys_aux_work;
     sys_woken:
 #else
@@ -4965,7 +4965,7 @@ erts_fprintf(stderr, "--------------------------------\n");
 	rq->out_of_work_count = 0;
 	(void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
 
-	rq->max_len = rq->len;
+	rq->max_len = erts_smp_atomic32_read_dirty(&rq->len);
 	for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
 	    ErtsRunQueueInfo *rqi;
 	    rqi = (pix == ERTS_PORT_PRIO_LEVEL
@@ -5123,7 +5123,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 {
     int wo_reds = rq->wakeup_other_reds;
     if (wo_reds) {
-	int left_len = rq->len - 1;
+	int left_len = erts_smp_atomic32_read_dirty(&rq->len) - 1;
 	if (left_len < 1) {
 	    int wo_reduce = wo_reds << wakeup_other.dec_shift;
 	    wo_reduce &= wakeup_other.dec_mask;
@@ -5196,7 +5196,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 {
     int wo_reds = rq->wakeup_other_reds;
     if (wo_reds) {
-	erts_aint32_t len = rq->len;
+	erts_aint32_t len = erts_smp_atomic32_read_dirty(&rq->len);
 	if (len < 2) {
 	    rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds;
 	    if (rq->wakeup_other < 0)
@@ -5292,7 +5292,7 @@ runq_supervisor(void *unused)
 	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 	    if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) {
 		erts_smp_runq_lock(rq);
-		if (rq->len != 0)
+		if (erts_smp_atomic32_read_dirty(&rq->len) != 0)
 		    wake_scheduler_on_empty_runq(rq); /* forced wakeup... */
 		erts_smp_runq_unlock(rq);
 	    }
@@ -5642,7 +5642,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	}
 	rq->out_of_work_count = 0;
 	rq->max_len = 0;
-	rq->len = 0;
+	erts_smp_atomic32_set_nob(&rq->len, 0);
 	rq->wakeup_other = 0;
 	rq->wakeup_other_reds = 0;
 	rq->halt_in_progress = 0;
@@ -7939,6 +7939,9 @@ sched_thread_func(void *vesdp)
 
     erts_sched_init_time_sup(esdp);
 
+    (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue,
+				  ERTS_RUNQ_FLG_EXEC);
+
 #ifdef ERTS_SMP
     tse = erts_tse_fetch();
     erts_tse_prepare_timed(tse);
@@ -8947,24 +8950,39 @@ resume_process_1(BIF_ALIST_1)
 }
 
 Uint
-erts_run_queues_len(Uint *qlen)
+erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched)
 {
     int i = 0;
     Uint len = 0;
-    ERTS_ATOMIC_FOREACH_RUNQ(rq,
-    {
-	Sint pqlen = 0;
-	int pix;
-	for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++)
-	    pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len);
+    if (atomic_queues_read)
+	ERTS_ATOMIC_FOREACH_RUNQ(rq,
+	 {
+	     Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len);
+	     ASSERT(rq_len >= 0);
+	     if (incl_active_sched
+		 && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) {
+		 rq_len++;
+	     }
+	     if (qlen)
+		 qlen[i++] = rq_len;
+	     len += (Uint) rq_len;
+	 }
+	    );
+    else {
+	for (i = 0; i < erts_no_run_queues; i++) {
+	    ErtsRunQueue *rq = ERTS_RUNQ_IX(i);
+	    Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len);
+	    ASSERT(rq_len >= 0);
+	     if (incl_active_sched
+		 && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) {
+		 rq_len++;
+	     }
+	    if (qlen)
+		qlen[i] = rq_len;
+	    len += (Uint) rq_len;
+	}
 
-	if (pqlen < 0)
-	    pqlen = 0;
- 	if (qlen)
-	    qlen[i++] = pqlen;
-	len += pqlen;
     }
-	);
     return len;
 }
 
@@ -9391,8 +9409,10 @@ Process *schedule(Process *p, int calls)
 
 	if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
 	    if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
+		(void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
 		suspend_scheduler(esdp);
-		flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+		flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
+		flags |= ERTS_RUNQ_FLG_EXEC;
 	    }
 	    if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) {
 		flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND);
@@ -9483,7 +9503,10 @@ Process *schedule(Process *p, int calls)
 	    }
 #endif
 
+	    (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_EXEC);
 	    scheduler_wait(&fcalls, esdp, rq);
+	    flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
+	    flags |= ERTS_RUNQ_FLG_EXEC;
 
 #ifdef ERTS_SMP
 	    non_empty_runq(rq);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 10c6fa4a67..fd7d4183f4 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -170,8 +170,10 @@ extern int erts_sched_thread_suggested_stack_size;
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
 #define ERTS_RUNQ_FLG_PROTECTED \
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
+#define ERTS_RUNQ_FLG_EXEC \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))
 
-#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 7)
+#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)
 
 #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
   (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
@@ -215,6 +217,9 @@ extern int erts_sched_thread_suggested_stack_size;
 #define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
     ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
 					      (erts_aint32_t) (FLGS)))
+#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
+    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
+					     (erts_aint32_t) (FLGS)))
 #define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
     ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
 					       (erts_aint32_t) (MSK),	\
@@ -222,6 +227,9 @@ extern int erts_sched_thread_suggested_stack_size;
 #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
     ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
 					       (erts_aint32_t) ~(FLGS)))
+#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
+    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
+					      (erts_aint32_t) ~(FLGS)))
 #define ERTS_RUNQ_FLGS_GET(RQ)						\
     ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
 #define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
@@ -467,7 +475,7 @@ struct ErtsRunQueue_ {
     int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
     int out_of_work_count;
     erts_aint32_t max_len;
-    erts_aint32_t len;
+    erts_smp_atomic32_t len;
     int wakeup_other;
     int wakeup_other_reds;
     int halt_in_progress;
@@ -728,7 +736,19 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
-    len = erts_smp_atomic32_read_nob(&rqi->len);
+    len = erts_smp_atomic32_read_dirty(&rq->len);
+
+#ifdef ERTS_SMP
+    if (len == 0)
+	erts_non_empty_runq(rq);
+#endif
+    len++;
+    if (rq->max_len < len)
+	rq->max_len = len;
+    ASSERT(len > 0);
+    erts_smp_atomic32_set_nob(&rq->len, len);
+
+    len = erts_smp_atomic32_read_dirty(&rqi->len);
     ASSERT(len >= 0);
     if (len == 0) {
 	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
@@ -741,15 +761,6 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
 	rqi->max_len = len;
 
     erts_smp_atomic32_set_relb(&rqi->len, len);
-
-#ifdef ERTS_SMP
-    if (rq->len == 0)
-	erts_non_empty_runq(rq);
-#endif
-    rq->len++;
-    if (rq->max_len < rq->len)
-	rq->max_len = len;
-    ASSERT(rq->len > 0);
 }
 
 ERTS_GLB_INLINE void
@@ -759,7 +770,12 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
-    len = erts_smp_atomic32_read_nob(&rqi->len);
+    len = erts_smp_atomic32_read_dirty(&rq->len);
+    len--;
+    ASSERT(len >= 0);
+    erts_smp_atomic32_set_nob(&rq->len, len);
+
+    len = erts_smp_atomic32_read_dirty(&rqi->len);
     len--;
     ASSERT(len >= 0);
     if (len == 0) {
@@ -770,8 +786,6 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
     }
     erts_smp_atomic32_set_relb(&rqi->len, len);
 
-    rq->len--;
-    ASSERT(rq->len >= 0);
 }
 
 ERTS_GLB_INLINE void
@@ -781,7 +795,7 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
-    len = erts_smp_atomic32_read_nob(&rqi->len);
+    len = erts_smp_atomic32_read_dirty(&rqi->len);
     ASSERT(rqi->max_len >= len);
     rqi->max_len = len;
 }
@@ -1678,7 +1692,7 @@ void erts_sched_notify_check_cpu_bind(void);
 Uint erts_active_schedulers(void);
 void erts_init_process(int, int, int);
 Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
-Uint erts_run_queues_len(Uint *);
+Uint erts_run_queues_len(Uint *, int, int);
 void erts_add_to_runq(Process *);
 Eterm erts_bound_schedulers_term(Process *c_p);
 Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
-- 
cgit v1.2.3


From b18d251931a43099cc18527bcf6898fff1b144f5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Jan 2016 14:56:28 +0100
Subject: erts: Refactor ERL_NIF_INIT macro

---
 erts/emulator/beam/erl_nif.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 1a21048ec9..6f1b7a7571 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -243,20 +243,20 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks;
-#  ifdef STATIC_ERLANG_NIF
-#    define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* MODNAME ## _nif_init(TWinDynNifCallbacks* callbacks)
-#  else
-#    define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
-#  endif
+#  define ERL_NIF_INIT_ARGS TWinDynNifCallbacks* callbacks
 #  define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
+#  define ERL_NIF_INIT_EXPORT __declspec(dllexport)
 #else 
 #  define ERL_NIF_INIT_GLOB
+#  define ERL_NIF_INIT_ARGS void
 #  define ERL_NIF_INIT_BODY
-#  ifdef STATIC_ERLANG_NIF
-#    define ERL_NIF_INIT_DECL(MODNAME)  ErlNifEntry* MODNAME ## _nif_init(void)
-#  else
-#    define ERL_NIF_INIT_DECL(MODNAME)  ErlNifEntry* nif_init(void)
-#  endif
+#  define ERL_NIF_INIT_EXPORT
+#endif
+
+#ifdef STATIC_ERLANG_NIF
+#  define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _nif_init(ERL_NIF_INIT_ARGS)
+#else
+#  define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS)
 #endif
 
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-- 
cgit v1.2.3


From 52fc89120c2f2237ec5191e72d844a6dca150040 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Jan 2016 16:41:21 +0100
Subject: erts: Cleanup erl_driver.h for windows

The comment is misleading and
no need to "export" static windows drivers.

DRIVER_INIT for dynamic windows drivers
is defined in erl_win_dyn_driver.h
---
 erts/emulator/beam/erl_driver.h | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index bda4d5d1c6..268bda8b58 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -382,18 +382,11 @@ typedef struct erl_drv_entry {
 #  define ERLANG_DRIVER_NAME(NAME) driver_init
 #endif
 
-/* For windows dynamic drivers */
 #ifndef ERL_DRIVER_TYPES_ONLY
 
-#if defined(__WIN32__)
-#  define DRIVER_INIT(DRIVER_NAME) \
-  __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void);	\
-    __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
-#else 
 #  define DRIVER_INIT(DRIVER_NAME) \
     ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \
     ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
-#endif
 
 #define ERL_DRV_BUSY_MSGQ_DISABLED	(~((ErlDrvSizeT) 0))
 #define ERL_DRV_BUSY_MSGQ_READ_ONLY	((ErlDrvSizeT) 0)
-- 
cgit v1.2.3


From 1117dd3651be3cf763abf6fedcd4e06a6444fa04 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Jan 2016 15:22:52 +0100
Subject: erts: Allow -fvisibility=hidden for NIFs and drivers

as is strongly recommended by gcc man page.

We use __attribute__ ((visibility("default"))) to make sure
the init functions are properly exported.
---
 erts/emulator/beam/erl_driver.h | 14 +++++++++++---
 erts/emulator/beam/erl_nif.h    |  8 +++++++-
 2 files changed, 18 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 268bda8b58..0636171ad1 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -378,15 +378,23 @@ typedef struct erl_drv_entry {
 
 #ifdef STATIC_ERLANG_DRIVER
 #  define ERLANG_DRIVER_NAME(NAME) NAME ## _driver_init
+#  define ERL_DRIVER_EXPORT
 #else
 #  define ERLANG_DRIVER_NAME(NAME) driver_init
+#  if defined(__GNUC__) && __GNUC__ >= 4
+#    define ERL_DRIVER_EXPORT __attribute__ ((visibility("default")))
+#  elif defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550)
+#    define ERL_DRIVER_EXPORT __global
+#  else
+#    define ERL_DRIVER_EXPORT
+#  endif
 #endif
 
 #ifndef ERL_DRIVER_TYPES_ONLY
 
-#  define DRIVER_INIT(DRIVER_NAME) \
-    ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \
-    ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
+#define DRIVER_INIT(DRIVER_NAME) \
+    ERL_DRIVER_EXPORT ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \
+    ERL_DRIVER_EXPORT ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
 
 #define ERL_DRV_BUSY_MSGQ_DISABLED	(~((ErlDrvSizeT) 0))
 #define ERL_DRV_BUSY_MSGQ_READ_ONLY	((ErlDrvSizeT) 0)
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 6f1b7a7571..40c2ad6f08 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -250,7 +250,13 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  define ERL_NIF_INIT_GLOB
 #  define ERL_NIF_INIT_ARGS void
 #  define ERL_NIF_INIT_BODY
-#  define ERL_NIF_INIT_EXPORT
+#  if defined(__GNUC__) && __GNUC__ >= 4
+#    define ERL_NIF_INIT_EXPORT __attribute__ ((visibility("default")))
+#  elif defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550)
+#    define ERL_NIF_INIT_EXPORT __global
+#  else
+#    define ERL_NIF_INIT_EXPORT
+#  endif
 #endif
 
 #ifdef STATIC_ERLANG_NIF
-- 
cgit v1.2.3


From ebe42ec76748546f464076d8cf5d1238a56baf91 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Dec 2015 16:37:10 +0100
Subject: erts: Introduce erts_code_purger

as a system process with preloaded code.
---
 erts/emulator/beam/erl_init.c | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 58ef09662c..42aca726bf 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -439,6 +439,29 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
     return res;
 }
 
+static Eterm
+erl_system_process_otp(Eterm parent_pid, char* modname)
+{
+    Eterm start_mod;
+    Process* parent;
+    ErlSpawnOpts so;
+    Eterm res;
+
+    start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1);
+    if (erts_find_function(start_mod, am_start, 0,
+			   erts_active_code_ix()) == NULL) {
+	erl_exit(5, "No function %s:start/0\n", modname);
+    }
+
+    parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN);
+
+    so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC;
+    res = erl_create_process(parent, start_mod, am_start, NIL, &so);
+    erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_MAIN);
+    return res;
+}
+
+
 Eterm
 erts_preloaded(Process* p)
 {
@@ -1234,6 +1257,7 @@ erl_start(int argc, char **argv)
     ErtsTimeWarpMode time_warp_mode;
     int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT;
     ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL;
+    Eterm otp_ring0_pid;
 
     set_default_time_adj(&time_correction,
 			 &time_warp_mode);
@@ -2183,8 +2207,10 @@ erl_start(int argc, char **argv)
 
     erts_initialized = 1;
 
-    (void) erl_first_process_otp("otp_ring0", NULL, 0,
-				 boot_argc, boot_argv);
+    otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0,
+					  boot_argc, boot_argv);
+
+    (void) erl_system_process_otp(otp_ring0_pid, "erts_code_purger");
 
 #ifdef ERTS_SMP
     erts_start_schedulers();
-- 
cgit v1.2.3


From fa44f865c3fc6253cf4691cf94839c303a3ee40f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Dec 2015 20:32:39 +0100
Subject: erts: Make erlang:purge_module/1 safe

Problem: erlang:purge_module/1 is not safe in the sense
that very bad things may happen if the code to be purged
is still referred to by live processes.

Introduce erts_internal:purge_module which is the same as the old
erlang:purge_module BIF (except it returns false if no such old module).

Implement erlang:purge_module in Erlang and let it invoke
erts_code_purger for safe purging where all clogging processes
first are killed.
---
 erts/emulator/beam/beam_bif_load.c | 14 ++++++++++----
 erts/emulator/beam/bif.tab         |  2 +-
 2 files changed, 11 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 6bb70cc5a7..014ee35fd0 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1094,7 +1094,12 @@ static void copy_literals_commit(void* null) {
 #endif /* ERTS_SMP */
 
 
-BIF_RETTYPE purge_module_1(BIF_ALIST_1)
+/* Do the actualy module purging and return:
+ * true for success
+ * false if no such old module
+ * BADARG if not an atom
+ */
+BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1)
 {
     ErtsCodeIndex code_ix;
     BeamInstr* code;
@@ -1108,7 +1113,8 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
     }
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
-	ERTS_BIF_YIELD1(bif_export[BIF_purge_module_1], BIF_P, BIF_ARG_1);
+	ERTS_BIF_YIELD1(bif_export[BIF_erts_internal_purge_module_1],
+                        BIF_P, BIF_ARG_1);
     }
 
     code_ix = erts_active_code_ix();
@@ -1118,7 +1124,7 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
      */
 
     if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) {
-	ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+        ERTS_BIF_PREP_RET(ret, am_false);
     }
     else {
 	erts_rwlock_old_code(code_ix);
@@ -1127,7 +1133,7 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
 	 * Any code to purge?
 	 */
 	if (!modp->old.code_hdr) {
-	    ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+            ERTS_BIF_PREP_RET(ret, am_false);
 	}
 	else {
 	    /*
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 0aee8681c6..3b95ec508c 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -125,7 +125,6 @@ bif erlang:process_flag/3
 bif erlang:process_info/1
 bif erlang:process_info/2
 bif erlang:processes/0
-bif erlang:purge_module/1
 bif erlang:put/2
 bif erlang:register/2
 bif erlang:registered/0
@@ -643,6 +642,7 @@ bif erts_debug:map_info/1
 #
 
 bif erlang:copy_literals/2
+bif erts_internal:purge_module/1
 bif binary:split/2
 bif binary:split/3
 bif erts_debug:size_shared/1
-- 
cgit v1.2.3


From dc54c2a27c41930a18e0c7f2b97eda6cd4a0b1c1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Dec 2015 19:11:48 +0100
Subject: erts: Move copy_literals/2 from erlang to erts_internal

as it's not a public interface.
---
 erts/emulator/beam/beam_bif_load.c | 5 +++--
 erts/emulator/beam/bif.tab         | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 014ee35fd0..39a5bff04c 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1031,7 +1031,7 @@ copy_literals_t erts_clrange = {NULL, 0};
  */
 
 
-BIF_RETTYPE copy_literals_2(BIF_ALIST_2)
+BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2)
 {
     Module* modp;
     ErtsCodeIndex code_ix;
@@ -1042,7 +1042,8 @@ BIF_RETTYPE copy_literals_2(BIF_ALIST_2)
     }
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
-	ERTS_BIF_YIELD2(bif_export[BIF_copy_literals_2], BIF_P, BIF_ARG_1, BIF_ARG_2);
+	ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_copy_literals_2],
+                        BIF_P, BIF_ARG_1, BIF_ARG_2);
     }
 
     code_ix = erts_active_code_ix();
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 3b95ec508c..1b8ae8cef5 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -641,7 +641,7 @@ bif erts_debug:map_info/1
 # New in 19.0
 #
 
-bif erlang:copy_literals/2
+bif erts_internal:copy_literals/2
 bif erts_internal:purge_module/1
 bif binary:split/2
 bif binary:split/3
-- 
cgit v1.2.3


From 79efde2d8503e5055ef9e8afa5d8d63208710b1f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Dec 2015 20:03:32 +0100
Subject: erts: Make copy_literals more fail safe

* Same process must do enable-disable.
* System process will force it and never get 'aborted'
---
 erts/emulator/beam/beam_bif_load.c | 28 +++++++++++++++++-----------
 erts/emulator/beam/global.h        |  1 +
 2 files changed, 18 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 39a5bff04c..892b2d16c2 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1013,7 +1013,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 static void copy_literals_commit(void*);
 #endif
 
-copy_literals_t erts_clrange = {NULL, 0};
+copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE};
 
 /* copy literals
  *
@@ -1033,7 +1033,6 @@ copy_literals_t erts_clrange = {NULL, 0};
 
 BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2)
 {
-    Module* modp;
     ErtsCodeIndex code_ix;
     Eterm res = am_true;
 
@@ -1048,21 +1047,28 @@ BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2)
 
     code_ix = erts_active_code_ix();
 
-    if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL || !modp->old.code_hdr) {
-        res = am_false;
-        goto done;
-    }
-
     if (BIF_ARG_2 == am_true) {
-        if (erts_clrange.ptr != NULL) {
+        Module* modp = erts_get_module(BIF_ARG_1, code_ix);
+        if (!modp || !modp->old.code_hdr) {
+            res = am_false;
+            goto done;
+        }
+        if (erts_clrange.ptr != NULL
+            && !(BIF_P->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) {
             res = am_aborted;
             goto done;
-       }
-        erts_clrange.ptr = (Eterm*) modp->old.code_hdr->literals_start;
-        erts_clrange.sz  = (Eterm*) modp->old.code_hdr->literals_end - erts_clrange.ptr;
+        }
+        erts_clrange.ptr = modp->old.code_hdr->literals_start;
+        erts_clrange.sz  = modp->old.code_hdr->literals_end - erts_clrange.ptr;
+        erts_clrange.pid = BIF_P->common.id;
     } else if (BIF_ARG_2 == am_false) {
+        if (erts_clrange.pid != BIF_P->common.id) {
+            res = am_false;
+            goto done;
+        }
         erts_clrange.ptr = NULL;
         erts_clrange.sz  = 0;
+        erts_clrange.pid = THE_NON_VALUE;
     }
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 0bf5988244..bbf684d49d 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -990,6 +990,7 @@ Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *red
 typedef struct {
     Eterm *ptr;
     Uint   sz;
+    Eterm pid;
 } copy_literals_t;
 
 extern copy_literals_t erts_clrange;
-- 
cgit v1.2.3


From 4c763443365591e170308a1c5f11a4586734ca4e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 12 Jan 2016 16:57:59 +0100
Subject: erts: Optimize erlang:check_process_code

by ignoring literals.

erts_internal:check_process_code will be called again anyway
(with option {copy_literals, true}) before the module is actually purged.
No need to check literals twice.
---
 erts/emulator/beam/beam_bif_load.c | 61 +++++++++++++-------------------------
 erts/emulator/beam/erl_process.c   |  4 +--
 erts/emulator/beam/global.h        |  5 +++-
 3 files changed, 26 insertions(+), 44 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 892b2d16c2..a000935388 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -38,7 +38,7 @@
 #include "erl_thr_progress.h"
 
 static void set_default_trace_pattern(Eterm module);
-static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp);
+static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp);
 static void delete_code(Module* modp);
 static void decrement_refc(BeamCodeHeader*);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
@@ -426,7 +426,7 @@ check_old_code_1(BIF_ALIST_1)
 }
 
 Eterm
-erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp)
+erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp)
 {
     Module* modp;
     Eterm res;
@@ -441,7 +441,8 @@ erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp)
     if (!modp)
 	return am_false;
     erts_rlock_old_code(code_ix);
-    res = modp->old.code_hdr ? check_process_code(c_p, modp, allow_gc, redsp) : am_false;
+    res = (!modp->old.code_hdr ? am_false :
+           check_process_code(c_p, modp, flags, redsp));
     erts_runlock_old_code(code_ix);
 
     return res;
@@ -450,49 +451,21 @@ erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp)
 BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2)
 {
     int reds = 0;
+    Uint flags;
     Eterm res;
-    Eterm olist = BIF_ARG_2;
-    int allow_gc = 1;
 
     if (is_not_atom(BIF_ARG_1))
 	goto badarg;
 
-    while (is_list(olist)) {
-	Eterm *lp = list_val(olist);
-	Eterm opt = CAR(lp);
-	if (is_tuple(opt)) {
-	    Eterm* tp = tuple_val(opt);
-	    switch (arityval(tp[0])) {
-	    case 2:
-		switch (tp[1]) {
-		case am_allow_gc:
-		    switch (tp[2]) {
-		    case am_false:
-			allow_gc = 0;
-			break;
-		    case am_true:
-			allow_gc = 1;
-			break;
-		    default:
-			goto badarg;
-		    }
-		    break;
-		default:
-		    goto badarg;
-		}
-		break;
-	    default:
-		goto badarg;
-	    }
-	}
-	else
-	    goto badarg;
-	olist = CDR(lp);
+    if (is_not_small(BIF_ARG_2))
+        goto badarg;
+
+    flags = unsigned_val(BIF_ARG_2);
+    if (flags & ~ERTS_CPC_ALL) {
+        goto badarg;
     }
-    if (is_not_nil(olist))
-	goto badarg;
 
-    res = erts_check_process_code(BIF_P, BIF_ARG_1, allow_gc, &reds);
+    res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds);
 
     ASSERT(is_value(res));
 
@@ -739,7 +712,7 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size)
 
 
 static Eterm
-check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
+check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
 {
     BeamInstr* start;
     char* literals;
@@ -852,6 +825,12 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	/* Check heap, stack etc... */
 	if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size))
 	    goto try_gc;
+        if (!(flags & ERTS_CPC_COPY_LITERALS)) {
+            /* Process ok. May contain old literals but we will be called
+             * again before module is purged.
+             */
+            return am_false;
+        }
 	if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) {
 	    rp->freason = EXC_NULL;
 	    rp->fvalue = NIL;
@@ -919,7 +898,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	if ((done_gc & need_gc) == need_gc)
 	    return am_true;
 
-	if (!allow_gc)
+	if (!(flags & ERTS_CPC_ALLOW_GC))
 	    return am_aborted;
 
 	need_gc &= ~done_gc;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9495436ba6..d36d866b2f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10055,7 +10055,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	case ERTS_PSTT_CPC:
 	    st_res = erts_check_process_code(c_p,
 					     st->arg[0],
-					     st->arg[1] == am_true,
+                                             unsigned_val(st->arg[1]),
 					     &reds);
 	    if (is_non_value(st_res)) {
 		/* Needed gc, but gc was disabled */
@@ -10219,7 +10219,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3)
     case am_check_process_code:
 	if (is_not_atom(st->arg[0]))
 	    goto badarg;
-	if (st->arg[1] != am_true && st->arg[1] != am_false)
+	if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL))
 	    goto badarg;
 	noproc_res = am_false;
 	st->type = ERTS_PSTT_CPC;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bbf684d49d..3f5925765d 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -985,7 +985,10 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg);
 Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 
 /* beam_bif_load.c */
-Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp);
+#define ERTS_CPC_ALLOW_GC      (1 << 0)
+#define ERTS_CPC_COPY_LITERALS (1 << 1)
+#define ERTS_CPC_ALL           (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS)
+Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp);
 
 typedef struct {
     Eterm *ptr;
-- 
cgit v1.2.3


From 52738d8bee488c262cf514ce4df49e87dfa47bc3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 Jan 2016 18:47:30 +0100
Subject: erts: Fix race between receive timeout and exit signal

Must re-read 'state' after seizing proc locks as other thread may
have set EXITING.
---
 erts/emulator/beam/erl_process.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7b3d12ce09..d767e1bb5a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9674,6 +9674,8 @@ Process *schedule(Process *p, int calls)
 
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 
+	state = erts_smp_atomic32_read_nob(&p->state);
+
 	if (erts_sched_stat.enabled) {
 	    int prio;
 	    UWord old = ERTS_PROC_SCHED_ID(p,
-- 
cgit v1.2.3


From 693db9ef9c5a67b36215f21c32f91a986fc5630d Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Mon, 18 Jan 2016 14:04:24 -0500
Subject: Fix dirty scheduler check in handle_aux_work

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 96d9a2f8b4..dd8bc9a698 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2239,7 +2239,7 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     erts_aint32_t aux_work = orig_aux_work;
     erts_aint32_t ignore = 0;
 
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
 #ifdef ERTS_SMP
     haw_thr_prgr_current_reset(awdp);
 #endif
-- 
cgit v1.2.3


From 858c6f7fa44f7b2dc363b359198d6522dd60e914 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 5 Jan 2016 16:55:04 +0100
Subject: Introduce time warp safe trace timestamp formats

New timestamp options for trace, sequential trace, and
system profile:
- monotonic_timestamp
- strict_monotonic_timestamp
---
 erts/emulator/beam/atom.names      |   2 +
 erts/emulator/beam/erl_bif_trace.c |  41 ++-
 erts/emulator/beam/erl_process.c   |   1 +
 erts/emulator/beam/erl_process.h   |  87 +++--
 erts/emulator/beam/erl_trace.c     | 700 ++++++++++++++++++++++---------------
 erts/emulator/beam/erl_trace.h     |  32 +-
 6 files changed, 545 insertions(+), 318 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index fb3368eae2..07f6492948 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -370,6 +370,7 @@ atom monitor
 atom monitor_nodes
 atom monitors
 atom monotonic
+atom monotonic_timestamp
 atom more
 atom multi_scheduling
 atom multiline
@@ -559,6 +560,7 @@ atom static
 atom stderr_to_stdout
 atom stop
 atom stream
+atom strict_monotonic_timestamp
 atom sunrm
 atom suspend
 atom suspended
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 03f51132b1..08807d72c9 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -431,6 +431,9 @@ Uint
 erts_trace_flag2bit(Eterm flag) 
 {
     switch (flag) {
+    case am_timestamp: return F_NOW_TS;
+    case am_strict_monotonic_timestamp: return F_STRICT_MON_TS;
+    case am_monotonic_timestamp: return F_MON_TS;
     case am_all: return TRACEE_FLAGS;
     case am_send: return F_TRACE_SEND;
     case am_receive: return F_TRACE_RECEIVE;
@@ -439,7 +442,6 @@ erts_trace_flag2bit(Eterm flag)
     case am_set_on_first_spawn: return F_TRACE_SOS1;
     case am_set_on_link: return F_TRACE_SOL;
     case am_set_on_first_link: return F_TRACE_SOL1;
-    case am_timestamp: return F_TIMESTAMP;
     case am_running: return F_TRACE_SCHED;
     case am_exiting: return F_TRACE_SCHED_EXIT;
     case am_garbage_collection: return F_TRACE_GC;
@@ -592,7 +594,7 @@ Eterm trace_3(BIF_ALIST_3)
 	    ERTS_TRACE_FLAGS(tracee_port) |= mask;
 	else
 	    ERTS_TRACE_FLAGS(tracee_port) &= ~mask;
-	
+
 	if (!ERTS_TRACE_FLAGS(tracee_port))
 	    ERTS_TRACER_PROC(tracee_port) = NIL;
 	else if (tracer != NIL)
@@ -978,7 +980,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
     }
 
     if (key == am_flags) {
-	int num_flags = 19;	/* MAXIMUM number of flags. */
+	int num_flags = 21;	/* MAXIMUM number of flags. */
 	Uint needed = 3+2*num_flags;
 	Eterm flag_list = NIL;
 	Eterm* limit;
@@ -996,6 +998,9 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 #endif
         hp = HAlloc(p, needed);
 	limit = hp+needed;
+	FLAG(F_NOW_TS, am_timestamp);
+	FLAG(F_STRICT_MON_TS, am_strict_monotonic_timestamp);
+	FLAG(F_MON_TS, am_monotonic_timestamp);
 	FLAG(F_TRACE_SEND, am_send);
 	FLAG(F_TRACE_RECEIVE, am_receive);
 	FLAG(F_TRACE_SOS, am_set_on_spawn);
@@ -1007,7 +1012,6 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 	FLAG(F_TRACE_SCHED, am_running);
 	FLAG(F_TRACE_SCHED_EXIT, am_exiting);
 	FLAG(F_TRACE_GC, am_garbage_collection);
-	FLAG(F_TIMESTAMP, am_timestamp);
 	FLAG(F_TRACE_ARITY_ONLY, am_arity);
 	FLAG(F_TRACE_RETURN_TO, am_return_to);
 	FLAG(F_TRACE_SILENT, am_silent);
@@ -1798,7 +1802,11 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2,
     } else if (arg1 == am_print) {
 	current_flag = SEQ_TRACE_PRINT; 
     } else if (arg1 == am_timestamp) {
-	current_flag = SEQ_TRACE_TIMESTAMP; 
+	current_flag = SEQ_TRACE_NOW_TS; 
+    } else if (arg1 == am_strict_monotonic_timestamp) {
+	current_flag = SEQ_TRACE_STRICT_MON_TS; 
+    } else if (arg1 == am_monotonic_timestamp) {
+	current_flag = SEQ_TRACE_MON_TS; 
     }
     else
 	current_flag = 0;
@@ -1909,7 +1917,9 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item)
 #endif
 	) {
 	if ((item == am_send) || (item == am_receive) || 
-	    (item == am_print) || (item == am_timestamp)) {
+	    (item == am_print) || (item == am_timestamp)
+	    || (item == am_monotonic_timestamp)
+	    || (item == am_strict_monotonic_timestamp)) {
 	    hp = HAlloc(p,3);
 	    res = TUPLE2(hp, item, am_false);
 	    BIF_RET(res);
@@ -1927,7 +1937,11 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item)
     } else if (item == am_print) {
 	current_flag = SEQ_TRACE_PRINT; 
     } else if (item == am_timestamp) {
-	current_flag = SEQ_TRACE_TIMESTAMP; 
+	current_flag = SEQ_TRACE_NOW_TS; 
+    } else if (item == am_strict_monotonic_timestamp) {
+	current_flag = SEQ_TRACE_STRICT_MON_TS; 
+    } else if (item == am_monotonic_timestamp) {
+	current_flag = SEQ_TRACE_MON_TS; 
     } else {
 	current_flag = 0;
     }
@@ -2237,6 +2251,7 @@ static Eterm system_profile_get(Process *p) {
 	if (erts_system_profile_flags.exclusive) {
 	    res = CONS(hp, am_exclusive, res); hp += 2;
 	}
+
     	return TUPLE2(hp, system_profile, res);
     }
 }
@@ -2255,6 +2270,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
     int system_blocked = 0;
     Process *profiler_p = NULL;
     Port *profiler_port = NULL;
+    int ts;
 
     if (profiler == am_undefined || list == NIL) {
 	prev = system_profile_get(p);
@@ -2286,7 +2302,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 	    goto error;
 	}
 
-	for (scheduler = 0, runnable_ports = 0, runnable_procs = 0, exclusive = 0;
+	for (ts = ERTS_TRACE_FLG_NOW_TIMESTAMP, scheduler = 0,
+		 runnable_ports = 0, runnable_procs = 0, exclusive = 0;
 	    is_list(list);
 	    list = CDR(list_val(list))) {
 	    
@@ -2299,6 +2316,12 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 		exclusive = !0;
 	    } else if (t == am_scheduler) {
 		scheduler = !0;
+	    } else if (t == am_timestamp) {
+		ts = ERTS_TRACE_FLG_NOW_TIMESTAMP;
+	    } else if (t == am_strict_monotonic_timestamp) {
+		ts = ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP;
+	    } else if (t == am_monotonic_timestamp) {
+		ts = ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP;
 	    } else goto error;
 	}	
 	if (is_not_nil(list)) goto error;
@@ -2311,7 +2334,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 	erts_system_profile_flags.runnable_ports = !!runnable_ports;
 	erts_system_profile_flags.runnable_procs = !!runnable_procs;
 	erts_system_profile_flags.exclusive = !!exclusive;
-
+	erts_system_profile_ts_type = ts;
 	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 	
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b47aae0f74..c1d2e8db8e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -353,6 +353,7 @@ struct erts_system_monitor_flags_t erts_system_monitor_flags;
 /* system performance monitor */
 Eterm erts_system_profile;
 struct erts_system_profile_flags_t erts_system_profile_flags;
+int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP;
 
 #if ERTS_MAX_PROCESSES > 0x7fffffff
 #error "Need to store process_count in another type"
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index fd7d4183f4..6512710f48 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -60,6 +60,9 @@ typedef struct process Process;
 #include "erl_mseg.h"
 #include "erl_async.h"
 #include "erl_gc.h"
+#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
+#include "erl_trace.h"
+#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS
 
 #ifdef HIPE
 #include "hipe_process.h"
@@ -1292,6 +1295,7 @@ struct erts_system_profile_flags_t {
     unsigned int exclusive : 1;
 };
 extern struct erts_system_profile_flags_t erts_system_profile_flags;
+extern int erts_system_profile_ts_type;
 
 /* process flags */
 #define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
@@ -1307,61 +1311,90 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC */
 
+#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0
+
+#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))
+
 /* process trace_flags */
-#define F_SENSITIVE          (1 << 0)
-#define F_TRACE_SEND         (1 << 1)   
-#define F_TRACE_RECEIVE      (1 << 2)
-#define F_TRACE_SOS          (1 << 3) /* Set on spawn       */
-#define F_TRACE_SOS1         (1 << 4) /* Set on first spawn */
-#define F_TRACE_SOL          (1 << 5) /* Set on link        */
-#define F_TRACE_SOL1         (1 << 6) /* Set on first link  */
-#define F_TRACE_CALLS        (1 << 7)
-#define F_TIMESTAMP          (1 << 8)
-#define F_TRACE_PROCS        (1 << 9)
-#define F_TRACE_FIRST_CHILD  (1 << 10)
-#define F_TRACE_SCHED        (1 << 11)
-#define F_TRACE_GC           (1 << 12)
-#define F_TRACE_ARITY_ONLY   (1 << 13)
-#define F_TRACE_RETURN_TO    (1 << 14) /* Return_to trace when breakpoint tracing */
-#define F_TRACE_SILENT       (1 << 15) /* No call trace msg suppress */
-#define F_TRACER             (1 << 16) /* May be (has been) tracer */
-#define F_EXCEPTION_TRACE    (1 << 17) /* May have exception trace on stack */
+
+#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
+			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
+#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
+			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
+#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
+			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
+#define F_SENSITIVE          F_TRACE_FLAG(0)
+#define F_TRACE_SEND         F_TRACE_FLAG(1)
+#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
+#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
+#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
+#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
+#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
+#define F_TRACE_CALLS        F_TRACE_FLAG(7)
+#define F_TRACE_PROCS        F_TRACE_FLAG(8)
+#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
+#define F_TRACE_SCHED        F_TRACE_FLAG(10)
+#define F_TRACE_GC           F_TRACE_FLAG(11)
+#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
+#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
+#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
+#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
+#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */
 
 /* port trace flags, currently the same as process trace flags */
-#define F_TRACE_SCHED_PORTS  (1 << 18) /* Trace of port scheduling */
-#define F_TRACE_SCHED_PROCS  (1 << 19) /* With virtual scheduling */
-#define F_TRACE_PORTS	     (1 << 20) /* Ports equivalent to F_TRACE_PROCS */
-#define F_TRACE_SCHED_NO     (1 << 21) /* Trace with scheduler id */
-#define F_TRACE_SCHED_EXIT   (1 << 22)
+#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
+#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
+#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
+#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
+#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)
 
-#define F_NUM_FLAGS          23
+#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
 #ifdef DEBUG
 #  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
 #else
 #  define F_INITIAL_TRACE_FLAGS 0
 #endif
 
+/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
+#define F_TIMESTAMP_MASK \
+    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
+
 #define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
 		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
 		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
-		     | F_TRACE_SCHED | F_TIMESTAMP | F_TRACE_GC \
+		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
 		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                      | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
 		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
 		     | F_TRACE_SCHED_EXIT)
 
 #define ERTS_TRACEE_MODIFIER_FLAGS \
-  (F_TRACE_SILENT | F_TIMESTAMP | F_TRACE_SCHED_NO)
+  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
 #define ERTS_PORT_TRACEE_FLAGS \
   (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
 #define ERTS_PROC_TRACEE_FLAGS \
   ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)
 
+#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))
+
 /* Sequential trace flags */
+
+/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
+#define SEQ_TRACE_TIMESTAMP_MASK \
+    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
+
 #define SEQ_TRACE_SEND     (1 << 0)
 #define SEQ_TRACE_RECEIVE  (1 << 1)
 #define SEQ_TRACE_PRINT    (1 << 2)
-#define SEQ_TRACE_TIMESTAMP (1 << 3)
+
+#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3
+
+#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
+			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
+#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
+				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
+#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
+			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
 
 #ifdef USE_VM_PROBES
 #define DT_UTAG_PERMANENT (1 << 0)
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index e1b03a057f..2243639099 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -38,6 +38,7 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "erl_thr_progress.h"
+#include "erl_bif_unique.h"
 
 #if 0
 #define DEBUG_PRINTOUTS
@@ -77,6 +78,263 @@ enum ErtsSysMsgType {
     SYS_MSG_TYPE_SYSPROF
 };
 
+#define ERTS_TRACE_TS_NOW_MAX_SIZE				\
+    4
+#define ERTS_TRACE_TS_MONOTONIC_MAX_SIZE			\
+    ERTS_MAX_SINT64_HEAP_SIZE
+#define ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE			\
+    (3 + ERTS_MAX_SINT64_HEAP_SIZE				\
+     + ERTS_MAX_UINT64_HEAP_SIZE)
+
+#define ERTS_TRACE_PATCH_TS_MAX_SIZE				\
+    (1 + ((ERTS_TRACE_TS_NOW_MAX_SIZE				\
+	   > ERTS_TRACE_TS_MONOTONIC_MAX_SIZE)			\
+	  ? ((ERTS_TRACE_TS_NOW_MAX_SIZE			\
+	      > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE)	\
+	     ? ERTS_TRACE_TS_NOW_MAX_SIZE			\
+	     : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE)		\
+	  : ((ERTS_TRACE_TS_MONOTONIC_MAX_SIZE			\
+	      > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE)	\
+	     ? ERTS_TRACE_TS_MONOTONIC_MAX_SIZE			\
+	     : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE)))
+
+#define TFLGS_TS_TYPE(p) ERTS_TFLGS2TSTYPE(ERTS_TRACE_FLAGS((p)))
+
+/*
+ * FUTURE CHANGES:
+ *
+ * The timestamp functionality has intentionally been
+ * split in two parts for future use even though it
+ * is not used like this today. take_timestamp() takes
+ * the timestamp and calculate heap need for it (which
+ * is not constant). write_timestamp() writes the
+ * timestamp to the allocated heap. That is, one typically
+ * want to take the timestamp before allocating the heap
+ * and then write it to the heap.
+ *
+ * The trace output functionality now use patch_ts_size(),
+ * write_ts(), and patch_ts(). write_ts() both takes the
+ * timestamp and writes it. Since we don't know the
+ * heap need when allocating the heap area we need to
+ * over allocate (maximum size from patch_ts_size()) and
+ * then potentially (often) shrink the heap area after the
+ * timestamp has been written. The only reason it is
+ * currently done this way is because we do not want to
+ * make major changes of the trace behavior in a patch.
+ * This is planned to be changed in next major release.
+ */
+
+typedef struct {
+    int ts_type_flag;
+    union {
+	struct {
+	    Uint ms;
+	    Uint s;
+	    Uint us;
+	} now;
+	struct {
+	    ErtsMonotonicTime time;
+	    Sint64 raw_unique;
+	} monotonic;
+    } u;
+} ErtsTraceTimeStamp;
+
+static ERTS_INLINE Uint
+take_timestamp(ErtsTraceTimeStamp *tsp, int ts_type)
+{
+    int ts_type_flag = ts_type & -ts_type; /* least significant flag */
+
+    ASSERT(ts_type_flag == ERTS_TRACE_FLG_NOW_TIMESTAMP
+	   || ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP
+	   || ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP
+	   || ts_type_flag == 0);
+
+    tsp->ts_type_flag = ts_type_flag;
+    switch (ts_type_flag) {
+    case 0:
+	return (Uint) 0;
+    case ERTS_TRACE_FLG_NOW_TIMESTAMP:
+#ifdef HAVE_ERTS_NOW_CPU
+	if (erts_cpu_timestamp)
+	    erts_get_now_cpu(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us);
+	else
+#endif
+	    get_now(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us);
+	return (Uint) 4;
+    case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP:
+    case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: {
+	Uint hsz = 0;
+	ErtsMonotonicTime mtime = erts_get_monotonic_time(NULL);
+	mtime += ERTS_MONOTONIC_OFFSET_NATIVE;
+	hsz = (IS_SSMALL(mtime) ?
+	       (Uint) 0
+	       : ERTS_SINT64_HEAP_SIZE((Sint64) mtime));
+	tsp->u.monotonic.time = mtime;
+	if (ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP) {
+	    Sint64 raw_unique;
+	    hsz += 3; /* 2-tuple */
+	    raw_unique = erts_raw_get_unique_monotonic_integer();
+	    tsp->u.monotonic.raw_unique = raw_unique;
+	    hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique);
+	}
+	return hsz;
+    }
+    default:
+	ERTS_INTERNAL_ERROR("invalid timestamp type");
+	return 0;
+    }
+}
+
+static ERTS_INLINE Eterm
+write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp)
+{
+    int ts_type_flag = tsp->ts_type_flag;
+    Eterm res;
+
+    switch (ts_type_flag) {
+    case 0:
+	return NIL;
+    case ERTS_TRACE_FLG_NOW_TIMESTAMP:
+	res = TUPLE3(*hpp,
+		     make_small(tsp->u.now.ms),
+		     make_small(tsp->u.now.s),
+		     make_small(tsp->u.now.us));
+	*hpp += 4;
+	return res;
+    case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP:
+    case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: {
+	Sint64 mtime, raw;
+	Eterm unique, emtime;
+
+	mtime = (Sint64) tsp->u.monotonic.time;
+	emtime = (IS_SSMALL(mtime)
+		  ? make_small((Sint64) mtime)
+		  : erts_sint64_to_big((Sint64) mtime, hpp));
+
+	if (ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP)
+	    return emtime;
+
+	raw = tsp->u.monotonic.raw_unique;
+	unique = erts_raw_make_unique_monotonic_integer_value(hpp,
+							      raw);
+	res = TUPLE2(*hpp, emtime, unique);
+	*hpp += 3;
+	return res;
+    }
+    default:
+	ERTS_INTERNAL_ERROR("invalid timestamp type");
+	return THE_NON_VALUE;
+    }
+}
+
+#define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p))
+
+static ERTS_INLINE Uint
+patch_ts_size(int ts_type)
+{
+    int ts_type_flag = ts_type & -ts_type; /* least significant flag */
+    switch (ts_type_flag) {
+    case 0:
+	return 0;
+    case ERTS_TRACE_FLG_NOW_TIMESTAMP:
+	return 1 + ERTS_TRACE_TS_NOW_MAX_SIZE;
+    case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP:
+	return 1 + ERTS_TRACE_TS_MONOTONIC_MAX_SIZE;
+    case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP:
+	return 1 + ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE;
+    default:
+	ERTS_INTERNAL_ERROR("invalid timestamp type");
+	return 0;
+    }
+}
+
+/*
+ * Write a timestamp. The timestamp MUST be the last
+ * thing built on the heap. This since write_ts() might
+ * adjust the size of the used area.
+ */
+static Eterm
+write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer)
+{
+    ErtsTraceTimeStamp ts;
+    Sint shrink;
+    Eterm res, *ts_hp = hp;
+    Uint hsz;
+
+    ASSERT(ts_type);
+
+    hsz = take_timestamp(&ts, ts_type);
+
+    res = write_timestamp(&ts, &ts_hp);
+
+    ASSERT(ts_hp == hp + hsz);
+
+    switch (ts.ts_type_flag) {
+    case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP:
+	shrink = ERTS_TRACE_TS_MONOTONIC_MAX_SIZE;
+	break;
+    case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP:
+	shrink = ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE;
+	break;
+    default:
+	return res;
+    }
+
+    shrink -= hsz;
+
+    ASSERT(shrink >= 0);
+
+    if (shrink) {
+	if (bp)
+	    bp->used_size -= shrink;
+#ifndef ERTS_SMP
+	else if (tracer) {
+	    Eterm *endp = ts_hp + shrink;
+	    HRelease(tracer, endp, ts_hp);
+	}
+#endif
+    }
+
+    return res;
+}
+
+/*
+ * Patch a timestamp into a tuple. The tuple MUST be the last thing
+ * built on the heap before the call, and the timestamp MUST be
+ * the last thing after the call. This since patch_ts() might adjust
+ * the size of the used area.
+ */
+
+#define PATCH_TS__(Type, Tuple, Hp, Bp, Tracer)			\
+    do {							\
+	int ts_type__ = (Type);					\
+	if (ts_type__)						\
+	    patch_ts(ts_type__, (Tuple), (Hp), (Bp), (Tracer));	\
+    } while (0)
+
+#ifdef ERTS_SMP
+#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \
+    PATCH_TS__((Type), (Tuple), (Hp), (Bp), NULL)
+#else
+#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \
+    PATCH_TS__((Type), (Tuple), (Hp), (Bp), (Tracer))
+#endif
+
+static ERTS_INLINE void
+patch_ts(int ts_type, Eterm tuple, Eterm* hp, ErlHeapFragment *bp, Process *tracer)
+{
+    Eterm *tptr = tuple_val(tuple);
+    int arity = arityval(*tptr);
+
+    ASSERT(ts_type);
+    ASSERT((tptr+arity+1) == hp);
+
+    tptr[0] = make_arityval(arity+1);
+    tptr[1] = am_trace_ts;
+
+    *hp = write_ts(ts_type, hp+1, bp, tracer);
+}
+
 #ifdef ERTS_SMP
 static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type,
 				     Eterm from,
@@ -365,23 +623,6 @@ erts_get_system_profile(void) {
     return profile;
 }
 
-
-#ifdef HAVE_ERTS_NOW_CPU
-#  define GET_NOW(m, s, u) \
-do { \
-    if (erts_cpu_timestamp) \
-	erts_get_now_cpu(m, s, u); \
-    else \
-	get_now(m, s, u); \
-} while (0)
-#else
-#  define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0)
-#endif
-
-
-
-static Eterm* patch_ts(Eterm tuple4, Eterm* hp);
-
 #ifdef ERTS_SMP
 static void
 do_send_to_port(Eterm to,
@@ -436,11 +677,11 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
 /* Send        {trace_ts, Pid, out, 0, Timestamp}
  * followed by {trace_ts, Pid, in, 0, NewTimestamp}
  *
- * 'NewTimestamp' is fetched from GET_NOW() through patch_ts().
+ * 'NewTimestamp' through patch_ts().
  */
 static void 
-do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
-#define LOCAL_HEAP_SIZE (4+5+5)
+do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp, int ts_type) {
+#define LOCAL_HEAP_SIZE (5+5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
     DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     Eterm message;
     Eterm *hp;
@@ -462,9 +703,11 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
 		    SYS_MSG_TYPE_UNDEFINED,
 		    message);
 
-    message = TUPLE4(hp, am_trace_ts, pid, am_in, mfarity);
-    hp += 5;
-    hp = patch_ts(message, hp);
+
+    message = TUPLE5(hp, am_trace_ts, pid, am_in, mfarity,
+		     NIL /* Will be overwritten by timestamp */);
+    hp += 6;
+    hp[-1] = write_ts(ts_type, hp, NULL, NULL);
 
     do_send_to_port(trace_port->common.id,
 		    trace_port,
@@ -481,7 +724,7 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
  * It is assumed that 'message' is not an 'out' message.
  *
  * 'c_p' is the currently executing process, "tracee" is the traced process
- * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP), 
+ * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP_MASK), 
  * 'message' must contain a timestamp.
  */
 static void
@@ -489,8 +732,9 @@ send_to_port(Process *c_p, Eterm message,
 	     Eterm *tracer_pid, Uint *tracee_flags) {
     Port* trace_port;
 #ifndef ERTS_SMP
-#define LOCAL_HEAP_SIZE (4)
-    Eterm ts, *hp;
+    int ts_type;
+#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE
+    Eterm ts;
     DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 #endif
 
@@ -519,7 +763,7 @@ send_to_port(Process *c_p, Eterm message,
      */
 
     if ( c_p == NULL || 
-	 (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) { 
+	 (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { 
 #endif
 	do_send_to_port(*tracer_pid,
 			trace_port,
@@ -538,22 +782,12 @@ send_to_port(Process *c_p, Eterm message,
      */
     UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
-    if (*tracee_flags & F_TIMESTAMP) {
-	ASSERT(is_tuple(message));
-	hp = tuple_val(message);
-	ts = hp[arityval(hp[0])];
-    } else {
-	/* A fake schedule might be needed,
-	 * but this message does not contain a timestamp.
-	 * Create a dummy trace message with timestamp to be
-	 * passed to do_send_schedfix_to_port().
-	 */
-	Uint ms,s,us;
-	GET_NOW(&ms, &s, &us);
-	hp = local_heap;
-	ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us));
-	hp += 4;
-    }
+    /* A fake schedule might be needed.
+     * Create a dummy trace message with timestamp to be
+     * passed to do_send_schedfix_to_port().
+     */
+    ts_type = TFLGS_TS_TYPE(c_p);
+    ts = write_ts(ts_type, local_heap, NULL, NULL);
 
     trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY;
     do_send_to_port(*tracer_pid,
@@ -572,7 +806,7 @@ send_to_port(Process *c_p, Eterm message,
 	 * just after writning the real trace message, and now gets scheduled
 	 * in again.
 	 */
-	do_send_schedfix_to_port(trace_port, c_p->common.id, ts);
+	do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type);
     }
 
     erts_port_release(trace_port);
@@ -641,20 +875,19 @@ profile_send(Eterm from, Eterm message) {
 
 /* A fake schedule out/in message pair will be sent,
  * if the driver so requests.
- * If (timestamp == NIL), one is fetched from GET_NOW().
  *
  * 'c_p' is the currently executing process, may be NULL.
  */
 static void
 seq_trace_send_to_port(Process *c_p,
 		       Eterm seq_tracer,
-		       Eterm message,
-		       Eterm timestamp)
+		       Eterm message)
 {
     Port* trace_port;
 #ifndef ERTS_SMP
-    Eterm ts, *hp;
-#define LOCAL_HEAP_SIZE (4)
+    int ts_type;
+    Eterm ts;
+#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE
     DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #endif
@@ -679,7 +912,7 @@ seq_trace_send_to_port(Process *c_p,
     }
 
     if (c_p == NULL
-	|| (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) {
+	|| (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) {
 #endif
 	do_send_to_port(seq_tracer,
 			trace_port,
@@ -696,20 +929,12 @@ seq_trace_send_to_port(Process *c_p,
      * with 'running' and 'timestamp'.
      */
 
-    if (timestamp != NIL) {
-	ts = timestamp;
-    } else {
-	/* A fake schedule might be needed,
-	 * but this message does not contain a timestamp.
-	 * Create a dummy trace message with timestamp to be
-	 * passed to do_send_schedfix_to_port().
-	 */
-	Uint ms,s,us;
-	GET_NOW(&ms, &s, &us);
-	hp = local_heap;
-	ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us));
-	hp += 4;
-    }
+    /* A fake schedule might be needed.
+     * Create a dummy trace message with timestamp to be
+     * passed to do_send_schedfix_to_port().
+     */
+    ts_type = TFLGS_TS_TYPE(c_p);
+    ts = write_ts(ts_type, local_heap, NULL, NULL);
 
     trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY;
     do_send_to_port(seq_tracer,
@@ -728,7 +953,7 @@ seq_trace_send_to_port(Process *c_p,
 	 * just after writing the real trace message, and now gets scheduled
 	 * in again.
 	 */
-	do_send_schedfix_to_port(trace_port, c_p->common.id, ts);
+	do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type);
     }
 
     erts_port_release(trace_port);
@@ -738,32 +963,6 @@ seq_trace_send_to_port(Process *c_p,
 #endif
 }
 
-#define TS_HEAP_WORDS 5
-#define TS_SIZE(p) ((ERTS_TRACE_FLAGS((p)) & F_TIMESTAMP) \
-		    ? TS_HEAP_WORDS \
-		    : 0)
-
-/*
- * Patch a timestamp into a tuple.  The tuple must be the last thing
- * built on the heap.
- *
- * Returns the new hp pointer.
-*/
-static Eterm*
-patch_ts(Eterm tuple, Eterm* hp)
-{
-    Uint ms, s, us;
-    Eterm* ptr = tuple_val(tuple);
-    int arity = arityval(*ptr);
-
-    ASSERT((ptr+arity+1) == hp);
-    ptr[0] = make_arityval(arity+1);
-    ptr[1] = am_trace_ts;
-    GET_NOW(&ms, &s, &us);
-    *hp = TUPLE3(hp+1, make_small(ms), make_small(s), make_small(us));
-    return hp+5;
-}
-
 static ERTS_INLINE void
 send_to_tracer(Process *tracee,
 	       ERTS_TRACER_REF_TYPE tracer_ref,
@@ -776,13 +975,13 @@ send_to_tracer(Process *tracee,
 
     erts_smp_mtx_lock(&smq_mtx);
 
-    if (ERTS_TRACE_FLAGS(tracee) & F_TIMESTAMP)
-	*hpp = patch_ts(msg, *hpp);
-
-    if (is_internal_pid(ERTS_TRACER_PROC(tracee)))
+    if (is_internal_pid(ERTS_TRACER_PROC(tracee))) {
+	PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, bp, tracer_ref);
 	ERTS_ENQ_TRACE_MSG(tracee->common.id, tracer_ref, msg, bp);
+    }
     else {
 	ASSERT(is_internal_port(ERTS_TRACER_PROC(tracee)));
+	PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, NULL, NULL);
 	send_to_port(no_fake_sched ? NULL : tracee,
 		     msg,
 		     &ERTS_TRACER_PROC(tracee),
@@ -796,7 +995,7 @@ send_to_tracer(Process *tracee,
 static void
 trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 {
-#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS)
+#define LOCAL_HEAP_SIZE (5+4+1+ERTS_TRACE_PATCH_TS_MAX_SIZE)
     DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
     Eterm tmp, mess, *hp;
     ErlHeapFragment *bp = NULL;
@@ -852,7 +1051,7 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 	    size += 4;
 	if (sched_no)
 	    size += 1;
-	size += TS_SIZE(p);
+	size += PATCH_TS_SIZE(p);
 	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
     }
 
@@ -926,7 +1125,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
     }
 
     if (is_internal_port(ERTS_TRACER_PROC(p))) {
-#define LOCAL_HEAP_SIZE (11)
+#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
@@ -934,9 +1133,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
 	mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to);
 	hp += 6;
 	erts_smp_mtx_lock(&smq_mtx);
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
 	send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
@@ -955,7 +1152,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
 
 	sz_msg = size_object(msg);
 	sz_to  = size_object(to);
-	need = sz_msg + sz_to + 6 + TS_SIZE(p);
+	need = sz_msg + sz_to + 6 + PATCH_TS_SIZE(p);
 	
 	hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
 
@@ -972,10 +1169,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    patch_ts(mess, hp);
-	}
-
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
 	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
     }
@@ -992,7 +1186,7 @@ trace_receive(Process *rp, Eterm msg)
     Eterm* hp;
 
     if (is_internal_port(ERTS_TRACER_PROC(rp))) {
-#define LOCAL_HEAP_SIZE (10)
+#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
@@ -1000,9 +1194,7 @@ trace_receive(Process *rp, Eterm msg)
 	mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg);
 	hp += 5;
 	erts_smp_mtx_lock(&smq_mtx);
-	if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, NULL, NULL);
 	send_to_port(rp, mess, &ERTS_TRACER_PROC(rp), &ERTS_TRACE_FLAGS(rp));
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
@@ -1021,7 +1213,7 @@ trace_receive(Process *rp, Eterm msg)
 
 	sz_msg = size_object(msg);
 
-	hsz = sz_msg + 5 + TS_SIZE(rp);
+	hsz = sz_msg + 5 + PATCH_TS_SIZE(rp);
 
 	hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref);
 
@@ -1031,10 +1223,7 @@ trace_receive(Process *rp, Eterm msg)
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) {
-	    patch_ts(mess, hp);
-	}
-
+	PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, bp, tracer_ref);
 	ERTS_ENQ_TRACE_MSG(rp->common.id, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
     }
@@ -1084,6 +1273,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
     Eterm type_atom;
     int sz_exit;
     Eterm seq_tracer;
+    int ts_type;
 
     seq_tracer = erts_get_system_seq_tracer();
 
@@ -1111,8 +1301,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	return;			/* no need to send anything */
     }
 
+    ts_type = ERTS_SEQTFLGS2TSTYPE(unsigned_val(SEQ_TRACE_T_FLAGS(token)));
+
     if (is_internal_port(seq_tracer)) {
-#define LOCAL_HEAP_SIZE (64)
+#define LOCAL_HEAP_SIZE (60 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
@@ -1128,17 +1320,17 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token),
 		      receiver, msg);
 	hp += 6;
+
 	erts_smp_mtx_lock(&smq_mtx);
-	if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) == 0) {
+	if (!ts_type) {
 	    mess = TUPLE3(hp, am_seq_trace, label, mess);
-	    seq_trace_send_to_port(NULL, seq_tracer, mess, NIL);
+	    seq_trace_send_to_port(NULL, seq_tracer, mess);
 	} else {
-	    Uint ms,s,us,ts;
-	    GET_NOW(&ms, &s, &us);
-	    ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us));
-	    hp += 4;
-	    mess = TUPLE4(hp, am_seq_trace, label, mess, ts);
-	    seq_trace_send_to_port(process, seq_tracer, mess, ts);
+	    mess = TUPLE4(hp, am_seq_trace, label, mess,
+			  NIL /* Will be overwritten by timestamp */);
+	    hp += 5;
+	    hp[-1] = write_ts(ts_type, hp, NULL, NULL);
+	    seq_trace_send_to_port(process, seq_tracer, mess);
 	}
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
@@ -1173,8 +1365,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	sz_lastcnt_serial = 3; /* TUPLE2 */
 	sz_msg = size_object(msg);
 
-	sz_ts = ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) ? 
-		 5 : 0);
+	sz_ts = patch_ts_size(ts_type);
 	if (exitfrom != NIL) {
 	    sz_exit = 4; /* create {'EXIT',exitfrom,msg} */
 	    sz_exitfrom = size_object(exitfrom);
@@ -1218,14 +1409,20 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (sz_ts) {/* timestamp should be included */
-	    Uint ms,s,us,ts;
-	    GET_NOW(&ms, &s, &us);
-	    ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us));
-	    hp += 4;
-	    mess = TUPLE4(hp, am_seq_trace, label, mess, ts);
-	} else {
+	if (!ts_type)
 	    mess = TUPLE3(hp, am_seq_trace, label, mess);
+	else {
+	    mess = TUPLE4(hp, am_seq_trace, label, mess,
+			  NIL /* Will be overwritten by timestamp */);
+	    hp += 5;
+	    /* Write timestamp in element 6 of the 'msg' tuple */
+	    hp[-1] = write_ts(ts_type, hp, bp,
+#ifndef ERTS_SMP
+			      tracer
+#else
+			      NULL
+#endif
+		);
 	}
 
 #ifdef ERTS_SMP
@@ -1244,7 +1441,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 void 
 erts_trace_return_to(Process *p, BeamInstr *pc)
 {
-#define LOCAL_HEAP_SIZE (4+5+5)
+#define LOCAL_HEAP_SIZE (4+5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
     Eterm* hp;
     Eterm mfa;
     Eterm mess;
@@ -1269,9 +1466,7 @@ erts_trace_return_to(Process *p, BeamInstr *pc)
 
     erts_smp_mtx_lock(&smq_mtx);
 
-    if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	hp = patch_ts(mess, hp);
-    }
+    PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
 
     if (is_internal_port(ERTS_TRACER_PROC(p))) {
 	send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
@@ -1318,6 +1513,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
     Eterm mod, name;
     int arity;
     Uint meta_flags, *tracee_flags;
+    int ts_type;
 #ifdef ERTS_SMP
     Eterm tracee;
 #endif
@@ -1353,7 +1549,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
 	 *     meta trace =>
 	 *       use fixed flag set instead of process flags
 	 */	    
-	meta_flags = F_TRACE_CALLS | F_TIMESTAMP;
+	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
 #ifdef ERTS_SMP
 	tracee = NIL;
@@ -1367,8 +1563,10 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
     name = fi[1];
     arity = fi[2];
     
+    ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags);
+
     if (is_internal_port(*tracer_pid)) {
-#define LOCAL_HEAP_SIZE (4+6+5)
+#define LOCAL_HEAP_SIZE (4+6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 	hp = local_heap;
@@ -1377,9 +1575,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
 	mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval);
 	hp += 6;
 	erts_smp_mtx_lock(&smq_mtx);
-	if (*tracee_flags & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(ts_type, mess, hp, NULL, NULL);
 	send_to_port(p, mess, tracer_pid, tracee_flags);
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
@@ -1390,24 +1586,15 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
 	ERTS_TRACER_REF_TYPE tracer_ref;
 	unsigned size;
 	unsigned retval_size;
-#ifdef DEBUG
-	Eterm* limit;
-#endif
 
 	ASSERT(is_internal_pid(*tracer_pid));
 
 	ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags);
-	
+
 	retval_size = size_object(retval);
-	size = 6 + 4 + retval_size;
-	if (*tracee_flags & F_TIMESTAMP) {
-	    size += 1+4;
-	}
+	size = 6 + 4 + retval_size + patch_ts_size(ts_type);
 
 	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
-#ifdef DEBUG
-	limit = hp + size;
-#endif
 
 	/*
 	 * Build the trace tuple and put it into receive queue of the tracer process.
@@ -1421,11 +1608,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (*tracee_flags & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
-
-	ASSERT(hp == limit);
+	PATCH_TS(ts_type, mess, hp, bp, tracer_ref);
 
 	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
@@ -1448,6 +1631,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
     Eterm cv;
     Eterm mess;
     Uint meta_flags, *tracee_flags;
+    int ts_type;
 #ifdef ERTS_SMP
     Eterm tracee;
 #endif
@@ -1486,15 +1670,17 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
 	 *     meta trace =>
 	 *       use fixed flag set instead of process flags
 	 */	    
-	meta_flags = F_TRACE_CALLS | F_TIMESTAMP;
+	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
 #ifdef ERTS_SMP
 	tracee = NIL;
 #endif
     }
     
+    ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags);
+
     if (is_internal_port(*tracer_pid)) {
-#define LOCAL_HEAP_SIZE (4+3+6+5)
+#define LOCAL_HEAP_SIZE (4+3+6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
@@ -1507,10 +1693,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
 	hp += 6;
 	ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE);
 	erts_smp_mtx_lock(&smq_mtx);
-	if (*tracee_flags & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp); /* hp += 5 */
-	    ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE);
-	}
+	PATCH_TS(ts_type, mess, hp, NULL, NULL);
 	send_to_port(p, mess, tracer_pid, tracee_flags);
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
@@ -1521,24 +1704,15 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
 	ERTS_TRACER_REF_TYPE tracer_ref;
 	unsigned size;
 	unsigned value_size;
-#ifdef DEBUG
-	Eterm* limit;
-#endif
 
 	ASSERT(is_internal_pid(*tracer_pid));
 
 	ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags);
 	
 	value_size = size_object(value);
-	size = 6 + 4 + 3 + value_size;
-	if (*tracee_flags & F_TIMESTAMP) {
-	    size += 1+4;
-	}
+	size = 6 + 4 + 3 + value_size + patch_ts_size(ts_type);
 
 	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
-#ifdef DEBUG
-	limit = hp + size;
-#endif
 
 	/*
 	 * Build the trace tuple and put it into receive queue of the tracer process.
@@ -1555,11 +1729,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (*tracee_flags & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
-
-	ASSERT(hp == limit);
+	PATCH_TS(ts_type, mess, hp, bp, tracer_ref);
 
 	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
@@ -1592,6 +1762,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
     Eterm pam_result = am_true;
     Eterm mess;
     Uint meta_flags, *tracee_flags;
+    int ts_type;
 #ifdef ERTS_SMP
     Eterm tracee;
 #endif
@@ -1633,7 +1804,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	    /* No trace messages for sensitive processes. */
 	    return 0;
 	}
-	meta_flags = F_TRACE_CALLS | F_TIMESTAMP;
+	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
 #ifdef ERTS_SMP
 	tracee = NIL;
@@ -1676,12 +1847,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
     }
     args = transformed_args;
 
+    ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags);
+
     if (is_internal_port(*tracer_pid)) {
 #if HEAP_ON_C_STACK
-	Eterm local_heap[64+MAX_ARG];
+	Eterm local_heap[60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG];
 #else
 	Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM,
-				       sizeof(Eterm)*(64+MAX_ARG));
+				       sizeof(Eterm)*(60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG));
 #endif
 	hp = local_heap;
 
@@ -1796,9 +1969,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	    *hp++ = pam_result;
 	}
 	erts_smp_mtx_lock(&smq_mtx);
-	if (*tracee_flags & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(ts_type, mess, hp, NULL, NULL);
 	send_to_port(p, mess, tracer_pid, tracee_flags);
 	erts_smp_mtx_unlock(&smq_mtx);
 	erts_match_set_release_result(p);
@@ -1820,9 +1991,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	unsigned sizes[MAX_ARG];
 	unsigned pam_result_size = 0;
 	int invalid_tracer;
-#ifdef DEBUG
-	Eterm* limit;
-#endif
 
 	ASSERT(is_internal_pid(*tracer_pid));
 	
@@ -1915,10 +2083,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 		size += sizes[i];
 	    }
 	}
-	if (*tracee_flags & F_TIMESTAMP) {
-	    size += 1 + 4;
-	    /* One element in trace tuple + timestamp tuple. */
-	}
+	size += patch_ts_size(ts_type);
 	if (pam_result != am_true) {
 	    pam_result_size = size_object(pam_result);
 	    size += 1 + pam_result_size;
@@ -1926,9 +2091,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	}
 	
 	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
-#ifdef DEBUG
-	limit = hp + size;
-#endif
 
 	/*
 	 * Build the the {M,F,A} tuple in the message buffer. 
@@ -1971,11 +2133,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (*tracee_flags & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
-
-	ASSERT(hp == limit);
+	PATCH_TS(ts_type, mess, hp, bp, tracer_ref);
 	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
 	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
@@ -2010,9 +2168,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
 	mess = TUPLE4(hp, am_trace, t_p->common.id, what, data);
 	hp += 5;
 	erts_smp_mtx_lock(&smq_mtx);
-	if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL);
 	send_to_port(
 #ifndef ERTS_SMP
 	    /* No fake schedule out and in again after an exit */
@@ -2042,7 +2198,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
 
 	sz_data = size_object(data);
 
-	need = sz_data + 5 + TS_SIZE(t_p);
+	need = sz_data + 5 + PATCH_TS_SIZE(t_p);
 
 	hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
 
@@ -2052,9 +2208,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref);
 
 	ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
@@ -2088,9 +2242,7 @@ trace_proc_spawn(Process *p, Eterm pid,
 	mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, pid, mfa);
 	hp += 6;
 	erts_smp_mtx_lock(&smq_mtx);
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
 	send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
@@ -2111,7 +2263,7 @@ trace_proc_spawn(Process *p, Eterm pid,
 
 	sz_args = size_object(args);
 	sz_pid = size_object(pid);
-	need = sz_args + 4 + 6 + TS_SIZE(p);
+	need = sz_args + 4 + 6 + PATCH_TS_SIZE(p);
 
 	hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
 
@@ -2124,9 +2276,7 @@ trace_proc_spawn(Process *p, Eterm pid,
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
 
 	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
@@ -2208,11 +2358,8 @@ trace_gc(Process *p, Eterm what)
 #define LOCAL_HEAP_SIZE						\
     (sizeof(values)/sizeof(*values)) *				\
 	(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) +	\
-	5/*4-tuple */ + TS_HEAP_WORDS
+	5/*4-tuple */ + ERTS_TRACE_PATCH_TS_MAX_SIZE
     DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
-#ifdef DEBUG
-    Eterm* limit;
-#endif
 
     ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm));
 
@@ -2227,7 +2374,7 @@ trace_gc(Process *p, Eterm what)
 					    sizeof(values)/sizeof(*values),
 					    tags,
 					    values);
-	size += 5/*4-tuple*/ + TS_SIZE(p);
+	size += 5/*4-tuple*/ + PATCH_TS_SIZE(p);
 #endif
     } else {
 	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
@@ -2242,15 +2389,12 @@ trace_gc(Process *p, Eterm what)
 					    sizeof(values)/sizeof(*values),
 					    tags,
 					    values);
-	size += 5/*4-tuple*/ + TS_SIZE(p);
+	size += 5/*4-tuple*/ + PATCH_TS_SIZE(p);
 
 	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
     }
 
-#ifdef DEBUG
-    limit = hp + size;
     ASSERT(size <= LOCAL_HEAP_SIZE);
-#endif
 
     msg = erts_bld_atom_uword_2tup_list(&hp,
 				       NULL,
@@ -2263,14 +2407,14 @@ trace_gc(Process *p, Eterm what)
 
     erts_smp_mtx_lock(&smq_mtx);
 
-    if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	hp = patch_ts(msg, hp);
-    }
-    ASSERT(hp == limit);
-    if (is_internal_port(ERTS_TRACER_PROC(p)))
+    if (is_internal_port(ERTS_TRACER_PROC(p))) {
+	PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, NULL, NULL);
 	send_to_port(p, msg, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
-    else
+    }
+    else {
+	PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, bp, tracer_ref);
 	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, msg, bp);
+    }
     erts_smp_mtx_unlock(&smq_mtx);
     UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
 #undef LOCAL_HEAP_SIZE
@@ -2574,19 +2718,18 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
 
 void
 profile_scheduler(Eterm scheduler_id, Eterm state) {
-    Eterm *hp, msg, timestamp;
-    Uint Ms, s, us;
+    Eterm *hp, msg;
+    ErlHeapFragment *bp = NULL;
 
 #ifndef ERTS_SMP
-#define LOCAL_HEAP_SIZE (4 + 7)
+#define LOCAL_HEAP_SIZE (7 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
     DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
     hp = local_heap;
 #else
-    ErlHeapFragment *bp;
     Uint hsz;
 
-    hsz = 4 + 7;
+    hsz = 7 + patch_ts_size(erts_system_profile_ts_type)-1;
 	
     bp = new_message_buffer(hsz);
     hp = bp->mem;
@@ -2606,10 +2749,13 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
 	break;
     }
 
-    GET_NOW(&Ms, &s, &us);
-    timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
-    msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state,
-		 make_small(active_sched), timestamp); hp += 7;
+    msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id,
+		 state, make_small(active_sched),
+		 NIL /* Will be overwritten by timestamp */);
+    hp += 7;
+
+    /* Write timestamp in element 6 of the 'msg' tuple */
+    hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
 #ifndef ERTS_SMP
     profile_send(NIL, msg);
@@ -2680,7 +2826,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
     Eterm* hp;
 
     if (is_internal_port(ERTS_TRACER_PROC(p))) {
-#define LOCAL_HEAP_SIZE (5+6)
+#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
@@ -2689,9 +2835,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
 	mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name);
 	hp += 6;
 	erts_smp_mtx_lock(&smq_mtx);
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
 	/* No fake schedule */
 	send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
@@ -2705,7 +2849,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
 	
 	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
 
-	sz_data = 6 + TS_SIZE(p);
+	sz_data = 6 + PATCH_TS_SIZE(p);
 
 	ERTS_GET_TRACER_REF(tracer_ref,
 			    ERTS_TRACER_PROC(p),
@@ -2718,9 +2862,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
 
 	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
@@ -2744,7 +2886,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 		       || erts_thr_progress_is_blocking());
 
     if (is_internal_port(ERTS_TRACER_PROC(t_p))) {
-#define LOCAL_HEAP_SIZE (5+5)
+#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
@@ -2752,9 +2894,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 	mess = TUPLE4(hp, am_trace, t_p->common.id, what, data);
 	hp += 5;
 	erts_smp_mtx_lock(&smq_mtx);
-	if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL);
 	/* No fake schedule */
 	send_to_port(NULL,mess,&ERTS_TRACER_PROC(t_p),&ERTS_TRACE_FLAGS(t_p));
 	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
@@ -2768,7 +2908,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 
 	ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p)));
 
-	sz_data = 5 + TS_SIZE(t_p);
+	sz_data = 5 + PATCH_TS_SIZE(t_p);
 
 	ERTS_GET_TRACER_REF(tracer_ref,
 			    ERTS_TRACER_PROC(t_p),
@@ -2781,9 +2921,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref);
 
 	ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
@@ -2811,7 +2949,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
     Eterm sched_id = am_undefined;
     
     if (is_internal_port(ERTS_TRACER_PROC(p))) {
-#define LOCAL_HEAP_SIZE (5+6)
+#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
 	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
@@ -2834,9 +2972,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
 	hp += ws;
 	
 	erts_smp_mtx_lock(&smq_mtx);
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
+
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
 	
 	/* No fake scheduling */
 	send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
@@ -2856,7 +2993,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
 			    ERTS_TRACER_PROC(p),
 			    ERTS_TRACE_FLAGS(p));
 
-	hp = ERTS_ALLOC_SYSMSG_HEAP(ws+TS_SIZE(p), &bp, &off_heap, tracer_ref);
+	hp = ERTS_ALLOC_SYSMSG_HEAP(ws+PATCH_TS_SIZE(p), &bp, &off_heap, tracer_ref);
 
 	if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
 #ifdef ERTS_SMP
@@ -2874,10 +3011,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
 
 	erts_smp_mtx_lock(&smq_mtx);
 
-	if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) {
-	    hp = patch_ts(mess, hp);
-	}
-
+	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
 	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
     }
@@ -2887,13 +3021,12 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
 
 void
 profile_runnable_port(Port *p, Eterm status) {
-    Uint Ms, s, us;
-    Eterm *hp, msg, timestamp;
-
+    Eterm *hp, msg;
+    ErlHeapFragment *bp = NULL;
     Eterm count = make_small(0);
 
 #ifndef ERTS_SMP
-#define LOCAL_HEAP_SIZE (4 + 6)
+#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
 
     DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
@@ -2901,10 +3034,9 @@ profile_runnable_port(Port *p, Eterm status) {
     hp = local_heap;
     
 #else
-    ErlHeapFragment *bp;
     Uint hsz;
 
-    hsz = 4 + 6;
+    hsz = 6 + patch_ts_size(erts_system_profile_ts_type)-1;
 
     bp = new_message_buffer(hsz);
     hp = bp->mem;
@@ -2912,9 +3044,12 @@ profile_runnable_port(Port *p, Eterm status) {
 
     erts_smp_mtx_lock(&smq_mtx);
 
-    GET_NOW(&Ms, &s, &us);
-    timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
-    msg = TUPLE5(hp, am_profile, p->common.id, status, count, timestamp); hp += 6;
+    msg = TUPLE5(hp, am_profile, p->common.id, status, count,
+		 NIL /* Will be overwritten by timestamp */);
+    hp += 6;
+
+    /* Write timestamp in element 5 of the 'msg' tuple */
+    hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
 #ifndef ERTS_SMP
     profile_send(p->common.id, msg);
@@ -2929,20 +3064,19 @@ profile_runnable_port(Port *p, Eterm status) {
 /* Process profiling */
 void 
 profile_runnable_proc(Process *p, Eterm status){
-    Uint Ms, s, us;
-    Eterm *hp, msg, timestamp;
+    Eterm *hp, msg;
     Eterm where = am_undefined;
+    ErlHeapFragment *bp = NULL;
 
 #ifndef ERTS_SMP
-#define LOCAL_HEAP_SIZE (4 + 6 + 4)
+#define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
 
     DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
     hp = local_heap;
 #else
-    ErlHeapFragment *bp;
-    Uint hsz = 4 + 6 + 4;
+    Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1;
 #endif
 	
     if (!p->current) {
@@ -2951,7 +3085,7 @@ profile_runnable_proc(Process *p, Eterm status){
 
 #ifdef ERTS_SMP
     if (!p->current) {
-	hsz = 4 + 6;
+	hsz -= 4;
     }
 
     bp = new_message_buffer(hsz);
@@ -2966,9 +3100,13 @@ profile_runnable_proc(Process *p, Eterm status){
 	
     erts_smp_mtx_lock(&smq_mtx);
 
-    GET_NOW(&Ms, &s, &us);
-    timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
-    msg = TUPLE5(hp, am_profile, p->common.id, status, where, timestamp); hp += 6;
+    msg = TUPLE5(hp, am_profile, p->common.id, status, where,
+		 NIL /* Will be overwritten by timestamp */);
+    hp += 6;
+
+    /* Write timestamp in element 5 of the 'msg' tuple */
+    hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
+
 #ifndef ERTS_SMP
     profile_send(p->common.id, msg);
     UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 7405490f76..a0058264d7 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -18,8 +18,38 @@
  * %CopyrightEnd%
  */
 
+#ifndef ERL_TRACE_H__FLAGS__
+#define ERL_TRACE_H__FLAGS__
+/*
+ * NOTE! The bits used for these flags matter. The flag with
+ * the least significant bit will take precedence!
+ *
+ * The "now timestamp" has highest precedence due to
+ * compatibility reasons.
+ */
+#define ERTS_TRACE_FLG_NOW_TIMESTAMP			(1 << 0)
+#define ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP	(1 << 1)
+#define ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP		(1 << 2)
+
+/*
+ * The bits used effects trace flags (of processes and ports)
+ * as well as sequential trace flags. If changed make sure
+ * these arn't messed up...
+ */
+#define ERTS_TRACE_TS_TYPE_BITS 			3
+#define ERTS_TRACE_TS_TYPE_MASK					\
+    ((1 << ERTS_TRACE_TS_TYPE_BITS) - 1)
+
+#define ERTS_TFLGS2TSTYPE(TFLGS)				\
+    ((int) (((TFLGS) >> ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) 	\
+	    & ERTS_TRACE_TS_TYPE_MASK))
+#define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS)				\
+    ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)	\
+	    & ERTS_TRACE_TS_TYPE_MASK))
+
+#endif /* ERL_TRACE_H__FLAGS__ */
 
-#ifndef ERL_TRACE_H__
+#if !defined(ERL_TRACE_H__) && !defined(ERTS_ONLY_INCLUDE_TRACE_FLAGS)
 #define ERL_TRACE_H__
 
 struct binary;
-- 
cgit v1.2.3


From dc8e62f33ad0ca7a772ec74d67b6d3e157f639b4 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 13 Jan 2016 18:33:42 +0100
Subject: Introduce time warp safe replacement for safe_fixed option

The new time warp safe option is safe_fixed_monotonic_time which
gives erlang:monotonic_time().

The safe_fixed option was also slightly changed. It now gives
erlang:timestamp() instead of erlang:now(). This has however
not been documented, so it is considered a compatible change.

The above effects both ets, and dets.

This commit also include the bugfix OTP-13239 for
dets:info(Tab, safe_fixed). The timestamp in the result returned
by dets:info(Tab, safe_fixed) was unintentionally broken as a
result of the time API rewrites in OTP 18.0.
---
 erts/emulator/beam/erl_db.c       | 46 ++++++++++++++++++++++++++++----------
 erts/emulator/beam/erl_db_util.h  |  5 ++++-
 erts/emulator/beam/erl_time_sup.c | 47 ++++++++++++++++++++++++++++++---------
 erts/emulator/beam/sys.h          |  4 ++++
 4 files changed, 78 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 878ee32b47..645f9e3c28 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1376,7 +1376,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	    status |= DB_ORDERED_SET;
 	    status &= ~(DB_SET | DB_BAG | DB_DUPLICATE_BAG);
 	}
-	/*TT*/
 	else if (is_tuple(val)) {
 	    Eterm *tp = tuple_val(val);
 	    if (arityval(tp[0]) == 2) {
@@ -3466,10 +3465,10 @@ static void fix_table_locked(Process* p, DbTable* tb)
 #endif
     erts_refc_inc(&tb->common.ref,1);
     fix = tb->common.fixations;
-    if (fix == NULL) { 
-	get_now(&(tb->common.megasec),
-		&(tb->common.sec), 
-		&(tb->common.microsec));
+    if (fix == NULL) {
+	tb->common.time.monotonic
+	    = erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(p));
+	tb->common.time.offset = erts_get_time_offset();
     }
     else {
 	for (; fix != NULL; fix = fix->next) {
@@ -3731,6 +3730,7 @@ static int free_table_cont(Process *p,
 static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 {
     Eterm ret = THE_NON_VALUE;
+    int use_monotonic;
 
     if (What == am_size) {
 	ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems));
@@ -3788,7 +3788,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    ret = am_true;
 	else
 	    ret = am_false;
-    } else if (What == am_atom_put("safe_fixed",10)) {
+    } else if ((use_monotonic
+		= ERTS_IS_ATOM_STR("safe_fixed_monotonic_time",
+				   What))
+	       || ERTS_IS_ATOM_STR("safe_fixed", What)) {
 #ifdef ERTS_SMP
 	erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
@@ -3797,7 +3800,19 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    Eterm *hp;
 	    Eterm tpl, lst;
 	    DbFixation *fix;
-	    need = 7;
+	    Sint64 mtime;
+	    
+	    need = 3;
+	    if (use_monotonic) {
+		mtime = (Sint64) tb->common.time.monotonic;
+		mtime += ERTS_MONOTONIC_OFFSET_NATIVE;
+		if (!IS_SSMALL(mtime))
+		    need += ERTS_SINT64_HEAP_SIZE(mtime);
+	    }
+	    else {
+		mtime = 0;
+		need += 4;
+	    }
 	    for (fix = tb->common.fixations; fix != NULL; fix = fix->next) {
 		need += 5;
 	    }
@@ -3809,11 +3824,18 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 		lst = CONS(hp,tpl,lst);
 		hp += 2;
 	    }
-	    tpl = TUPLE3(hp,
-			 make_small(tb->common.megasec),
-			 make_small(tb->common.sec),
-			 make_small(tb->common.microsec));
-	    hp += 4;
+	    if (use_monotonic)
+		tpl = (IS_SSMALL(mtime)
+		       ? make_small(mtime)
+		       : erts_sint64_to_big(mtime, &hp));
+	    else {
+		Uint ms, s, us;
+		erts_make_timestamp_value(&ms, &s, &us,
+					  tb->common.time.monotonic,
+					  tb->common.time.offset);
+		tpl = TUPLE3(hp, make_small(ms), make_small(s), make_small(us));
+		hp += 4;
+	    }
 	    ret = TUPLE2(hp, tpl, lst);
 	} else {
 	    ret = am_false;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 1ccdc0305b..0903a40460 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -229,7 +229,10 @@ typedef struct db_table_common {
     DbTableMethod* meth;      /* table methods */
     erts_smp_atomic_t nitems; /* Total number of items in table */
     erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */
-    Uint megasec,sec,microsec; /* Last fixation time */
+    struct {                  /* Last fixation time */
+	ErtsMonotonicTime monotonic;
+	ErtsMonotonicTime offset;
+    } time;
     DbFixation* fixations;    /* List of processes who have done safe_fixtable,
                                  "local" fixations not included. */ 
     /* All 32-bit fields */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 7327e0b48c..2c87b8e81c 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1747,6 +1747,39 @@ erts_get_monotonic_time(ErtsSchedulerData *esdp)
     return mtime;    
 }
 
+ErtsMonotonicTime
+erts_get_time_offset(void)
+{
+    return get_time_offset();
+}
+
+static ERTS_INLINE void
+make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
+		     ErtsMonotonicTime mtime, ErtsMonotonicTime offset)
+{
+    ErtsMonotonicTime stime, as;
+    Uint ms;
+
+    stime = ERTS_MONOTONIC_TO_USEC(mtime + offset);
+
+    as = stime / ERTS_MONOTONIC_TIME_MEGA;
+    *megasec = ms = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA);
+    *sec = (Uint) (as - (((ErtsMonotonicTime) ms)
+			 * ERTS_MONOTONIC_TIME_MEGA));
+    *microsec = (Uint) (stime - as*ERTS_MONOTONIC_TIME_MEGA);
+
+    ASSERT(((ErtsMonotonicTime) ms)*ERTS_MONOTONIC_TIME_TERA
+	   + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA
+	   + *microsec == stime);
+}
+
+void
+erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
+			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset)
+{
+    make_timestamp_value(megasec, sec, microsec, mtime, offset);
+}
+
 void
 get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
 {
@@ -2220,22 +2253,14 @@ BIF_RETTYPE time_offset_1(BIF_ALIST_1)
 BIF_RETTYPE timestamp_0(BIF_ALIST_0)
 {
     Eterm *hp, res;
-    ErtsMonotonicTime stime, mtime, all_sec, offset;
+    ErtsMonotonicTime mtime, offset;
     Uint mega_sec, sec, micro_sec;
 
     mtime = time_sup.r.o.get_time();
     offset = get_time_offset();
     update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
-    stime = ERTS_MONOTONIC_TO_USEC(mtime + offset);
-    all_sec = stime / ERTS_MONOTONIC_TIME_MEGA;
-    mega_sec = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA);
-    sec = (Uint) (all_sec - (((ErtsMonotonicTime) mega_sec)
-			     * ERTS_MONOTONIC_TIME_MEGA));
-    micro_sec = (Uint) (stime - all_sec*ERTS_MONOTONIC_TIME_MEGA);
-
-    ASSERT(((ErtsMonotonicTime) mega_sec)*ERTS_MONOTONIC_TIME_TERA
-	   + ((ErtsMonotonicTime) sec)*ERTS_MONOTONIC_TIME_MEGA
-	   + micro_sec == stime);
+
+    make_timestamp_value(&mega_sec, &sec, &micro_sec, mtime, offset);
 
     /*
      * Mega seconds is the only value that potentially
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index ec94e3a596..03e30573de 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -819,6 +819,10 @@ int local_to_univ(Sint *year, Sint *month, Sint *day,
 void get_now(Uint*, Uint*, Uint*);
 struct ErtsSchedulerData_;
 ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
+ErtsMonotonicTime erts_get_time_offset(void);
+void
+erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
+			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset);
 void get_sys_now(Uint*, Uint*, Uint*);
 void set_break_quit(void (*)(void), void (*)(void));
 
-- 
cgit v1.2.3


From 3f33428db9aea0d767295322c4e882a5c6bbf7db Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 19 Jan 2016 17:05:55 +0100
Subject: Introduce time management in native APIs

---
 erts/emulator/beam/erl_driver.h        |  92 ++++++---------------
 erts/emulator/beam/erl_drv_nif.h       |  84 +++++++++++++++++++
 erts/emulator/beam/erl_nif.c           |  20 +++++
 erts/emulator/beam/erl_nif.h           |  62 ++++----------
 erts/emulator/beam/erl_nif_api_funcs.h |   6 ++
 erts/emulator/beam/erl_time.h          |   4 +
 erts/emulator/beam/erl_time_sup.c      | 145 +++++++++++++++++++++++++++++++++
 erts/emulator/beam/io.c                |  23 ++++++
 8 files changed, 327 insertions(+), 109 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index dbb4d719c1..7ab58e336b 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -37,47 +37,6 @@
 #  endif
 #endif
 
-#ifdef SIZEOF_CHAR
-#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
-#  undef SIZEOF_CHAR
-#endif
-#ifdef SIZEOF_SHORT
-#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
-#  undef SIZEOF_SHORT
-#endif
-#ifdef SIZEOF_INT
-#  define SIZEOF_INT_SAVED__ SIZEOF_INT
-#  undef SIZEOF_INT
-#endif
-#ifdef SIZEOF_LONG
-#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
-#  undef SIZEOF_LONG
-#endif
-#ifdef SIZEOF_LONG_LONG
-#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
-#  undef SIZEOF_LONG_LONG
-#endif
-#ifdef HALFWORD_HEAP_EMULATOR
-#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
-#  undef HALFWORD_HEAP_EMULATOR
-#endif
-#include "erl_int_sizes_config.h"
-#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR
-#  error SIZEOF_CHAR mismatch
-#endif
-#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT
-#  error SIZEOF_SHORT mismatch
-#endif
-#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT
-#  error SIZEOF_INT mismatch
-#endif
-#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG
-#  error SIZEOF_LONG mismatch
-#endif
-#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG
-#  error SIZEOF_LONG_LONG mismatch
-#endif
-
 /* This is OK to override by the NIF/driver implementor */
 #if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR)
 #define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__
@@ -134,7 +93,7 @@ typedef struct {
 
 #define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
 #define ERL_DRV_EXTENDED_MAJOR_VERSION	3
-#define ERL_DRV_EXTENDED_MINOR_VERSION	2
+#define ERL_DRV_EXTENDED_MINOR_VERSION	3
 
 /*
  * The emulator will refuse to load a driver with a major version
@@ -176,28 +135,12 @@ typedef struct {
 /*
  * Integer types
  */
-#if  defined(__WIN32__) && (SIZEOF_VOID_P == 8)
-typedef unsigned __int64 ErlDrvTermData;
-typedef unsigned __int64 ErlDrvUInt;
-typedef signed __int64 ErlDrvSInt;
-#else
-typedef unsigned long ErlDrvTermData;
-typedef unsigned long ErlDrvUInt;
-typedef signed long ErlDrvSInt;
-#endif
 
-#if defined(__WIN32__)
-typedef unsigned __int64 ErlDrvUInt64;
-typedef __int64 ErlDrvSInt64;
-#elif SIZEOF_LONG == 8
-typedef unsigned long ErlDrvUInt64;
-typedef long ErlDrvSInt64;
-#elif SIZEOF_LONG_LONG == 8
-typedef unsigned long long ErlDrvUInt64;
-typedef long long ErlDrvSInt64;
-#else
-#error No 64-bit integer type
-#endif
+typedef ErlNapiUInt64 ErlDrvUInt64;
+typedef ErlNapiSInt64 ErlDrvSInt64;
+typedef ErlNapiUInt ErlDrvUInt;
+typedef ErlNapiSInt ErlDrvSInt;
+typedef ErlNapiUInt ErlDrvTermData;
 
 #if defined(__WIN32__) || defined(_WIN32)
 typedef ErlDrvUInt ErlDrvSizeT;
@@ -250,6 +193,17 @@ typedef struct {
     unsigned long microsecs;
 } ErlDrvNowData;
 
+typedef ErlDrvSInt64 ErlDrvTime;
+
+#define ERL_DRV_TIME_ERROR ((ErlDrvSInt64) ERTS_NAPI_TIME_ERROR__)
+
+typedef enum {
+    ERL_DRV_SEC = ERTS_NAPI_SEC__,
+    ERL_DRV_MSEC = ERTS_NAPI_MSEC__,
+    ERL_DRV_USEC = ERTS_NAPI_USEC__,
+    ERL_DRV_NSEC = ERTS_NAPI_NSEC__
+} ErlDrvTimeUnit;
+
 /*
  * Error codes that can be return from driver.
  */
@@ -685,8 +639,16 @@ EXTERN long driver_async(ErlDrvPort ix,
 EXTERN int driver_lock_driver(ErlDrvPort ix);
 
 /* Get the current 'now' timestamp (analogue to erlang:now()) */
-EXTERN int driver_get_now(ErlDrvNowData *now);
-
+EXTERN int driver_get_now(ErlDrvNowData *now) ERL_DRV_DEPRECATED_FUNC;
+
+/* Erlang Monotonic Time */
+EXTERN ErlDrvTime erl_drv_monotonic_time(ErlDrvTimeUnit time_unit);
+/* Time offset between Erlang Monotonic Time and Erlang System Time */
+EXTERN ErlDrvTime erl_drv_time_offset(ErlDrvTimeUnit time_unit);
+/* Time unit conversion */
+EXTERN ErlDrvTime erl_drv_convert_time_unit(ErlDrvTime val,
+					    ErlDrvTimeUnit from,
+					    ErlDrvTimeUnit to);
 
 /* These were removed from the ANSI version, now they're back. */
 
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index e2385f63f4..f6b946ae82 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -50,6 +50,90 @@ typedef enum {
 } ErlDrvDirtyJobFlags;
 #endif
 
+#ifdef SIZEOF_CHAR
+#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
+#  undef SIZEOF_CHAR
+#endif
+#ifdef SIZEOF_SHORT
+#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
+#  undef SIZEOF_SHORT
+#endif
+#ifdef SIZEOF_INT
+#  define SIZEOF_INT_SAVED__ SIZEOF_INT
+#  undef SIZEOF_INT
+#endif
+#ifdef SIZEOF_LONG
+#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
+#  undef SIZEOF_LONG
+#endif
+#ifdef SIZEOF_LONG_LONG
+#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
+#  undef SIZEOF_LONG_LONG
+#endif
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
+#  undef HALFWORD_HEAP_EMULATOR
+#endif
+#include "erl_int_sizes_config.h"
+#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR
+#  error SIZEOF_CHAR mismatch
+#endif
+#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT
+#  error SIZEOF_SHORT mismatch
+#endif
+#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT
+#  error SIZEOF_INT mismatch
+#endif
+#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG
+#  error SIZEOF_LONG mismatch
+#endif
+#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG
+#  error SIZEOF_LONG_LONG mismatch
+#endif
+
+#if !defined(__GNUC__) && (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+typedef unsigned __int64 ErlNapiUInt64;
+typedef signed __int64 ErlNapiSInt64;
+#define ERL_NAPI_SINT64_MAX__ 9223372036854775807i64
+#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1i64)
+#elif SIZEOF_LONG == 8
+typedef unsigned long ErlNapiUInt64;
+typedef signed long ErlNapiSInt64;
+#define ERL_NAPI_SINT64_MAX__ 9223372036854775807L
+#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1L)
+#elif SIZEOF_LONG_LONG == 8
+typedef unsigned long long ErlNapiUInt64;
+typedef signed long long ErlNapiSInt64;
+#define ERL_NAPI_SINT64_MAX__ 9223372036854775807LL
+#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1LL)
+#else
+#  error No 64-bit integer type
+#endif
+
+#if SIZEOF_VOID_P == 8
+typedef ErlNapiUInt64 ErlNapiUInt;
+typedef ErlNapiSInt64 ErlNapiSInt;
+#elif SIZEOF_VOID_P == 4
+#  if SIZEOF_LONG == SIZEOF_VOID_P
+typedef unsigned long ErlNapiUInt;
+typedef signed long ErlNapiSInt;
+#  elif SIZEOF_INT == SIZEOF_VOID_P
+typedef unsigned int ErlNapiUInt;
+typedef signed int ErlNapiSInt;
+#  else
+#    error No 32-bit integer type
+#  endif
+#else
+#  error Not support arch
+#endif
+
+#define ERTS_NAPI_TIME_ERROR__ ERL_NAPI_SINT64_MIN__
+
+#define ERTS_NAPI_SEC__		0
+#define ERTS_NAPI_MSEC__	1
+#define ERTS_NAPI_USEC__	2
+#define ERTS_NAPI_NSEC__	3
+
 #endif  /* __ERL_DRV_NIF_H__ */
 
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d7a2076d85..3141b05e2b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1175,6 +1175,26 @@ void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); }
 int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); }
 int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); }
 
+ErlNifTime enif_monotonic_time(ErlNifTimeUnit time_unit)
+{
+    return (ErlNifTime) erts_napi_monotonic_time((int) time_unit);
+}
+
+ErlNifTime enif_time_offset(ErlNifTimeUnit time_unit)
+{
+    return (ErlNifTime) erts_napi_time_offset((int) time_unit);
+}
+
+ErlNifTime
+enif_convert_time_unit(ErlNifTime val,
+		       ErlNifTimeUnit from,
+		       ErlNifTimeUnit to)
+{
+    return (ErlNifTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val,
+						    (int) from,
+						    (int) to);
+}
+
 int enif_fprintf(void* filep, const char* format, ...) 
 { 
     int ret;
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 5e39343e9b..75070ad901 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -49,9 +49,10 @@
 **           add ErlNifFunc flags
 ** 2.8: 18.0 add enif_has_pending_exception
 ** 2.9: 18.2 enif_getenv
+** 2.10: Time API
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 9
+#define ERL_NIF_MINOR_VERSION 10
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
@@ -67,63 +68,36 @@
 
 #include <stdlib.h>
 
-#ifdef SIZEOF_CHAR
-#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
-#  undef SIZEOF_CHAR
-#endif
-#ifdef SIZEOF_SHORT
-#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
-#  undef SIZEOF_SHORT
-#endif
-#ifdef SIZEOF_INT
-#  define SIZEOF_INT_SAVED__ SIZEOF_INT
-#  undef SIZEOF_INT
-#endif
-#ifdef SIZEOF_LONG
-#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
-#  undef SIZEOF_LONG
-#endif
-#ifdef SIZEOF_LONG_LONG
-#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
-#  undef SIZEOF_LONG_LONG
-#endif
-#ifdef HALFWORD_HEAP_EMULATOR
-#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
-#  undef HALFWORD_HEAP_EMULATOR
-#endif
-#include "erl_int_sizes_config.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
-typedef unsigned __int64 ErlNifUInt64;
-typedef __int64 ErlNifSInt64;
-#elif SIZEOF_LONG == 8
-typedef unsigned long ErlNifUInt64;
-typedef long ErlNifSInt64;
-#elif SIZEOF_LONG_LONG == 8
-typedef unsigned long long ErlNifUInt64;
-typedef long long ErlNifSInt64;
-#else
-#error No 64-bit integer type
-#endif
+typedef ErlNapiUInt64 ErlNifUInt64;
+typedef ErlNapiSInt64 ErlNifSInt64;
+typedef ErlNapiUInt ErlNifUInt;
+typedef ErlNapiSInt ErlNifSInt;
 
 #ifdef HALFWORD_HEAP_EMULATOR
 #  define ERL_NIF_VM_VARIANT "beam.halfword" 
 typedef unsigned int ERL_NIF_TERM;
 #else
 #  define ERL_NIF_VM_VARIANT "beam.vanilla" 
-#  if SIZEOF_LONG == SIZEOF_VOID_P
-typedef unsigned long ERL_NIF_TERM;
-#  elif SIZEOF_LONG_LONG == SIZEOF_VOID_P
-typedef unsigned long long ERL_NIF_TERM;
-#  endif
+typedef ErlNifUInt ERL_NIF_TERM;
 #endif
 
 typedef ERL_NIF_TERM ERL_NIF_UINT;
 
+typedef ErlNifSInt64 ErlNifTime;
+
+#define ERL_NIF_TIME_ERROR ((ErlNifSInt64) ERTS_NAPI_TIME_ERROR__)
+
+typedef enum {
+    ERL_NIF_SEC = ERTS_NAPI_SEC__,
+    ERL_NIF_MSEC = ERTS_NAPI_MSEC__,
+    ERL_NIF_USEC = ERTS_NAPI_USEC__,
+    ERL_NIF_NSEC = ERTS_NAPI_NSEC__
+} ErlNifTimeUnit;
+
 struct enif_environment_t;
 typedef struct enif_environment_t ErlNifEnv;
 
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 08b9afc6af..1448a508a2 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -160,6 +160,9 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int
 ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason));
 ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* value_size));
+ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_monotonic_time, (ErlNifTimeUnit));
+ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_time_offset, (ErlNifTimeUnit));
+ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTimeUnit, ErlNifTimeUnit));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -312,6 +315,9 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception)
 #  define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception)
 #  define enif_getenv ERL_NIF_API_FUNC_MACRO(enif_getenv)
+#  define enif_monotonic_time ERL_NIF_API_FUNC_MACRO(enif_monotonic_time)
+#  define enif_time_offset ERL_NIF_API_FUNC_MACRO(enif_time_offset)
+#  define enif_convert_time_unit ERL_NIF_API_FUNC_MACRO(enif_convert_time_unit)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 43e543e035..93a0d556bf 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -133,6 +133,10 @@ typedef struct {
 
 extern ErtsTimeSupData erts_time_sup__;
 
+ErtsMonotonicTime erts_napi_monotonic_time(int time_unit);
+ErtsMonotonicTime erts_napi_time_offset(int time_unit);
+ErtsMonotonicTime erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to);
+
 ERTS_GLB_INLINE Uint64
 erts_time_unit_conversion(Uint64 value,
 			  Uint32 from_time_unit,
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 2c87b8e81c..5f12c7809a 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -33,6 +33,8 @@
 #include "global.h"
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
+#include "erl_driver.h"
+#include "erl_nif.h"
  
 static erts_smp_mtx_t erts_timeofday_mtx;
 static erts_smp_mtx_t erts_get_time_mtx;
@@ -57,6 +59,7 @@ static int time_sup_initialized = 0;
 #define ERTS_MONOTONIC_TIME_TERA \
     (ERTS_MONOTONIC_TIME_GIGA*ERTS_MONOTONIC_TIME_KILO)
 
+static void init_time_napi(void);
 static void
 schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset);
 
@@ -948,6 +951,8 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
     ErtsMonotonicTime abs_native_offset, native_offset;
 #endif
 
+    init_time_napi();
+
     erts_hl_timer_init();
 
     ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX);
@@ -2197,6 +2202,146 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto
     return ret;
 }
 
+
+/*
+ * Time Native API (drivers and NIFs)
+ */
+
+#define ERTS_NAPI_TIME_ERROR ((ErtsMonotonicTime) ERTS_NAPI_TIME_ERROR__)
+
+static void
+init_time_napi(void)
+{
+    /* Verify that time native api constants are as expected... */
+
+    ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlDrvTime));
+    ASSERT(ERL_DRV_TIME_ERROR == (ErlDrvTime) ERTS_NAPI_TIME_ERROR);
+    ASSERT(ERL_DRV_TIME_ERROR < (ErlDrvTime) 0);
+    ASSERT(ERTS_NAPI_SEC__ == (int) ERL_DRV_SEC);
+    ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_DRV_MSEC);
+    ASSERT(ERTS_NAPI_USEC__ == (int) ERL_DRV_USEC);
+    ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_DRV_NSEC);
+
+    ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlNifTime));
+    ASSERT(ERL_NIF_TIME_ERROR == (ErlNifTime) ERTS_NAPI_TIME_ERROR);
+    ASSERT(ERL_NIF_TIME_ERROR < (ErlNifTime) 0);
+    ASSERT(ERTS_NAPI_SEC__ == (int) ERL_NIF_SEC);
+    ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_NIF_MSEC);
+    ASSERT(ERTS_NAPI_USEC__ == (int) ERL_NIF_USEC);
+    ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_NIF_NSEC);
+}
+
+ErtsMonotonicTime
+erts_napi_monotonic_time(int time_unit)
+{
+    ErtsSchedulerData *esdp;
+    ErtsMonotonicTime mtime;
+
+    /* At least for now only allow schedulers to do this... */
+    esdp = erts_get_scheduler_data();
+    if (!esdp)
+	return ERTS_NAPI_TIME_ERROR;
+
+    mtime = time_sup.r.o.get_time();
+    update_last_mtime(esdp, mtime);
+
+    switch (time_unit) {
+    case ERTS_NAPI_SEC__:
+	mtime = ERTS_MONOTONIC_TO_SEC(mtime);
+	mtime += ERTS_MONOTONIC_OFFSET_SEC;
+	break;
+    case ERTS_NAPI_MSEC__:
+	mtime = ERTS_MONOTONIC_TO_MSEC(mtime);
+	mtime += ERTS_MONOTONIC_OFFSET_MSEC;
+	break;
+    case ERTS_NAPI_USEC__:
+	mtime = ERTS_MONOTONIC_TO_USEC(mtime);
+	mtime += ERTS_MONOTONIC_OFFSET_USEC;
+	break;
+    case ERTS_NAPI_NSEC__:
+	mtime = ERTS_MONOTONIC_TO_NSEC(mtime);
+	mtime += ERTS_MONOTONIC_OFFSET_NSEC;
+	break;
+    default:
+	return ERTS_NAPI_TIME_ERROR;
+    }
+
+    return mtime;
+}
+
+ErtsMonotonicTime
+erts_napi_time_offset(int time_unit)
+{
+    ErtsSchedulerData *esdp;
+    ErtsSystemTime offs;
+
+    /* At least for now only allow schedulers to do this... */
+    esdp = erts_get_scheduler_data();
+    if (!esdp)
+	return ERTS_NAPI_TIME_ERROR;
+
+    offs = get_time_offset();
+    switch (time_unit) {
+    case ERTS_NAPI_SEC__:
+	offs = ERTS_MONOTONIC_TO_SEC(offs);
+	offs -= ERTS_MONOTONIC_OFFSET_SEC;
+	break;
+    case ERTS_NAPI_MSEC__:
+	offs = ERTS_MONOTONIC_TO_MSEC(offs);
+	offs -= ERTS_MONOTONIC_OFFSET_MSEC;
+	break;
+    case ERTS_NAPI_USEC__:
+	offs = ERTS_MONOTONIC_TO_USEC(offs);
+	offs -= ERTS_MONOTONIC_OFFSET_USEC;
+	break;
+    case ERTS_NAPI_NSEC__:
+	offs = ERTS_MONOTONIC_TO_NSEC(offs);
+	offs -= ERTS_MONOTONIC_OFFSET_NSEC;
+	break;
+    default:
+	return ERTS_NAPI_TIME_ERROR;
+    }
+    return offs;
+}
+
+ErtsMonotonicTime
+erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to)
+{
+    ErtsMonotonicTime ffreq, tfreq, denom;
+    /*
+     * Convertion between time units using floor function.
+     *
+     * Note that this needs to work also for negative
+     * values. Ordinary integer division on a negative
+     * value will give ceiling...
+     */
+
+    switch ((int) from) {
+    case ERTS_NAPI_SEC__: ffreq = 1; break;
+    case ERTS_NAPI_MSEC__: ffreq = 1000; break;
+    case ERTS_NAPI_USEC__: ffreq = 1000*1000; break;
+    case ERTS_NAPI_NSEC__: ffreq = 1000*1000*1000; break;
+    default: return ERTS_NAPI_TIME_ERROR;
+    }
+
+    switch ((int) to) {
+    case ERTS_NAPI_SEC__: tfreq = 1; break;
+    case ERTS_NAPI_MSEC__: tfreq = 1000; break;
+    case ERTS_NAPI_USEC__: tfreq = 1000*1000; break;
+    case ERTS_NAPI_NSEC__: tfreq = 1000*1000*1000; break;
+    default: return ERTS_NAPI_TIME_ERROR;
+    }
+
+    if (tfreq >= ffreq)
+	return val * (tfreq / ffreq);
+
+    denom = ffreq / tfreq;
+    if (val >= 0)
+	return val / denom;
+
+    return (val - (denom - 1)) / denom;
+}
+
 /* Built in functions */
 
 BIF_RETTYPE monotonic_time_0(BIF_ALIST_0)
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c64c8802b9..2bd31ee97e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -50,6 +50,7 @@
 #include "erl_map.h"
 #include "erl_bif_unique.h"
 #include "erl_hl_timer.h"
+#include "erl_time.h"
 
 extern ErlDrvEntry fd_driver_entry;
 #ifndef __OSE__
@@ -6762,6 +6763,28 @@ driver_get_now(ErlDrvNowData *now_data)
     return 0;
 }
 
+ErlDrvTime
+erl_drv_monotonic_time(ErlDrvTimeUnit time_unit)
+{
+    return (ErlDrvTime) erts_napi_monotonic_time((int) time_unit);
+}
+
+ErlDrvTime
+erl_drv_time_offset(ErlDrvTimeUnit time_unit)
+{
+    return (ErlDrvTime) erts_napi_time_offset((int) time_unit);
+}
+
+ErlDrvTime
+erl_drv_convert_time_unit(ErlDrvTime val,
+			  ErlDrvTimeUnit from,
+			  ErlDrvTimeUnit to)
+{
+    return (ErlDrvTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val,
+						    (int) from,
+						    (int) to);
+}
+
 static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
 {
     RefThing *refp;
-- 
cgit v1.2.3


From 66a80a7ab735a22249ffbbb7c88eccebba906194 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 21 Jan 2016 19:53:59 +0100
Subject: erts: Add checks for thread safe allocation

Assert thread unsafe allocator is only created on non-smp
and only called by the main thread.

Removed test of unsafe allocator in custom thread.
---
 erts/emulator/beam/erl_alloc.c      | 12 ++++++++++--
 erts/emulator/beam/erl_alloc_util.c |  5 +++++
 erts/emulator/beam/erl_init.c       | 11 ++++++-----
 erts/emulator/beam/global.h         |  3 +++
 4 files changed, 24 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 5544712e8d..e7523ac989 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -968,6 +968,10 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
     else
 #endif
     {
+#ifdef ERTS_SMP
+        erl_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n",
+                 init->init.util.name_prefix);
+#else
 	af->alloc = erts_alcu_alloc;
 	if (init->init.util.fix_type_size)
 	    af->realloc = erts_realloc_fixed_size;
@@ -976,6 +980,7 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
 	else
 	    af->realloc = erts_alcu_realloc;
 	af->free = erts_alcu_free;
+#endif
     }
     af->extra	= NULL;
     ai->alloc_util	= 1;
@@ -3402,8 +3407,11 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 	    init.enable = 1;
 	    init.atype = GOODFIT;
 	    init.init.util.name_prefix = (char *) a1;
-	    init.init.util.ts = a2 ? 1 : 0;
-
+#ifdef ERTS_SMP
+	    init.init.util.ts = 1;
+#else
+            init.init.util.ts = a2 ? 1 : 0;
+#endif
 	    if ((char **) a3) {
 		char **argv = (char **) a3;
 		int i = 0;
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index eedfd1e13d..3230b6ef34 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -5337,6 +5337,11 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 {
     void *res;
+#ifdef ERTS_SMP
+    ASSERT(!"This is not thread safe");
+#elif defined(USE_THREADS)
+    ASSERT(erts_equal_tids(erts_main_thread, erts_thr_self()));
+#endif
     res = do_erts_alcu_alloc(type, extra, size);
     DEBUG_CHECK_ALIGNMENT(res);
     return res;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 42aca726bf..39957eb58b 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -152,7 +152,7 @@ volatile int erts_writing_erl_crash_dump = 0;
 int erts_initialized = 0;
 
 #if defined(USE_THREADS) && !defined(ERTS_SMP)
-static erts_tid_t main_thread;
+erts_tid_t erts_main_thread;
 #endif
 
 int erts_use_sender_punish;
@@ -744,6 +744,10 @@ early_init(int *argc, char **argv) /*
     char envbuf[21]; /* enough for any 64-bit integer */
     size_t envbufsz;
 
+#if defined(USE_THREADS) && !defined(ERTS_SMP)
+    erts_main_thread = erts_thr_self();
+#endif
+
     erts_save_emu_args(*argc, argv);
 
     erts_sched_compact_load = 1;
@@ -797,9 +801,6 @@ early_init(int *argc, char **argv) /*
 			       (erts_aint32_t) ((Uint16) -1));
 
     erts_pre_init_process();
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-    main_thread = erts_thr_self();
-#endif
 
     /*
      * We need to know the number of schedulers to use before we
@@ -2285,7 +2286,7 @@ system_cleanup(int flush_async)
     if (!flush_async
 	|| !erts_initialized
 #if defined(USE_THREADS) && !defined(ERTS_SMP)
-	|| !erts_equal_tids(main_thread, erts_thr_self())
+	|| !erts_equal_tids(erts_main_thread, erts_thr_self())
 #endif
 	)
 	return;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 3f5925765d..ad9fdfc878 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1168,6 +1168,9 @@ extern ErtsModifiedTimings erts_modified_timings[];
 extern int erts_no_line_info;
 extern Eterm erts_error_logger_warnings;
 extern int erts_initialized;
+#if defined(USE_THREADS) && !defined(ERTS_SMP)
+extern erts_tid_t erts_main_thread;
+#endif
 extern int erts_compat_rel;
 extern int erts_use_sender_punish;
 void erts_short_init(void);
-- 
cgit v1.2.3


From e9d6797e15e687828e5ef0d33fb790181d657779 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 21 Jan 2016 19:54:46 +0100
Subject: erts: Fix faulty assert for non-smp

---
 erts/emulator/beam/erl_message.c | 2 +-
 erts/emulator/beam/sys.h         | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index b3e74e3e6a..88efb2c59f 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1504,7 +1504,7 @@ erts_factory_message_create(ErtsHeapFactory* factory,
     }
 
     if (on_heap) {
-	ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN);
+	ERTS_SMP_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN);
 	ASSERT(ohp == &proc->off_heap);
 	factory->mode = FACTORY_HALLOC;
 	factory->p = proc;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 53f8313daa..37fcfd1c52 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -196,6 +196,12 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 #  define ASSERT(e) ((void) 1)
 #endif
 
+#ifdef ERTS_SMP
+#  define ERTS_SMP_ASSERT(e) ASSERT(e)
+#else
+#  define ERTS_SMP_ASSERT(e) ((void)1)
+#endif
+
 /* ERTS_UNDEF can be used to silence false warnings about
  * "variable may be used uninitialized" while keeping the variable
  * marked as undefined by valgrind.
-- 
cgit v1.2.3


From 0236a875929729eca1933cbb854267f584734b26 Mon Sep 17 00:00:00 2001
From: Dmytro Lytovchenko <dmytro.lytovchenko@gmail.com>
Date: Thu, 21 Jan 2016 17:26:56 +0100
Subject: Moved do_list_to_integer from bif.c to big.c

---
 erts/emulator/beam/bif.c | 157 -----------------------------------------------
 erts/emulator/beam/big.c | 153 +++++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/big.h |   8 +++
 3 files changed, 161 insertions(+), 157 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index bb9165cd79..d519216fbd 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2907,163 +2907,6 @@ BIF_RETTYPE integer_to_list_1(BIF_ALIST_1)
 /* convert a list of ascii ascii integer value to an integer */
 
 
-#define LTI_BAD_STRUCTURE 0
-#define LTI_NO_INTEGER 1
-#define LTI_SOME_INTEGER 2
-#define LTI_ALL_INTEGER 3
-
-static int do_list_to_integer(Process *p, Eterm orig_list, 
-			      Eterm *integer, Eterm *rest)
-{
-     Sint i = 0;
-     Uint ui = 0;
-     int skip = 0;
-     int neg = 0;
-     Sint n = 0;
-     int m;
-     int lg2;
-     Eterm res;
-     Eterm* hp;
-     Eterm *hp_end;
-     Eterm lst = orig_list;
-     Eterm tail = lst;
-     int error_res = LTI_BAD_STRUCTURE;
-
-     if (is_nil(lst)) {
-       error_res = LTI_NO_INTEGER;
-     error:
-	 *rest = tail;
-	 *integer = make_small(0);
-	 return error_res;
-     }       
-     if (is_not_list(lst))
-       goto error;
-
-     /* if first char is a '-' then it is a negative integer */
-     if (CAR(list_val(lst)) == make_small('-')) {
-	  neg = 1;
-	  skip = 1;
-	  lst = CDR(list_val(lst));
-	  if (is_not_list(lst)) {
-	      tail = lst;
-	      error_res = LTI_NO_INTEGER;
-	      goto error;
-	  }
-     } else if (CAR(list_val(lst)) == make_small('+')) {
-	 /* ignore plus */
-	 skip = 1;
-	 lst = CDR(list_val(lst));
-	 if (is_not_list(lst)) {
-	     tail = lst;
-	     error_res = LTI_NO_INTEGER;
-	     goto error;
-	 }
-     }
-
-     /* Calculate size and do type check */
-
-     while(1) {
-	 if (is_not_small(CAR(list_val(lst)))) {
-	     break;
-	 }
-	 if (unsigned_val(CAR(list_val(lst))) < '0' ||
-	     unsigned_val(CAR(list_val(lst))) > '9') {
-	     break;
-	 }
-	 ui = ui * 10;
-	 ui = ui + unsigned_val(CAR(list_val(lst))) - '0';
-	 n++;
-	 lst = CDR(list_val(lst));
-	 if (is_nil(lst)) {
-	     break;
-	 }
-	 if (is_not_list(lst)) {
-	     break;
-	 }
-     }
-
-     tail = lst;
-     if (!n) {
-	 error_res = LTI_NO_INTEGER;
-	 goto error;
-     } 
-
-	 
-      /* If n <= 8 then we know it's a small int 
-      ** since 2^27 = 134217728. If n > 8 then we must
-      ** construct a bignum and let that routine do the checking
-      */
-
-     if (n <= SMALL_DIGITS) {  /* It must be small */
-	 if (neg) i = -(Sint)ui;
-         else i = (Sint)ui;
-	 res = make_small(i);
-     } else {
-	 /* Convert from log10 to log2 by multiplying with 1/log10(2)=3.3219
-	    which we round up to (3 + 1/3) */
-	 lg2 = (n+1)*3 + (n+1)/3 + 1;
-	 m  = (lg2+D_EXP-1)/D_EXP; /* number of digits */
-	 m  = BIG_NEED_SIZE(m);    /* number of words + thing */
-
-	 hp = HAlloc(p, m);
-	 hp_end = hp + m;
-	 
-	 lst = orig_list;
-	 if (skip)
-	     lst = CDR(list_val(lst));
-	 
-	 /* load first digits (at least one digit) */
-	 if ((i = (n % D_DECIMAL_EXP)) == 0)
-	     i = D_DECIMAL_EXP;
-	 n -= i;
-	 m = 0;
-	 while(i--) {
-	     m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0');
-	     lst = CDR(list_val(lst));
-	 }
-	 res = small_to_big(m, hp);  /* load first digits */
-	 
-	 while(n) {
-	     i = D_DECIMAL_EXP;
-	     n -= D_DECIMAL_EXP;
-	     m = 0;
-	     while(i--) {
-		 m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0');
-		 lst = CDR(list_val(lst));
-	     }
-	     if (is_small(res))
-		 res = small_to_big(signed_val(res), hp);
-	     res = big_times_small(res, D_DECIMAL_BASE, hp);
-	     if (is_small(res))
-		 res = small_to_big(signed_val(res), hp);
-	     res = big_plus_small(res, m, hp);
-	 }
-
-	 if (neg) {
-	     if (is_small(res))
-		 res = make_small(-signed_val(res));
-	     else {
-		 Uint *big = big_val(res); /* point to thing */
-		 *big = bignum_header_neg(*big);
-	     }
-	 }
-
-	 if (is_not_small(res)) {
-	     res = big_plus_small(res, 0, hp); /* includes conversion to small */
-
-	     if (is_not_small(res)) {
-		 hp += (big_arity(res)+1);
-	     }
-	 }
-	 HRelease(p,hp_end,hp);
-     }
-     *integer = res;
-     *rest = tail;
-     if (tail != NIL) {
-	 return LTI_SOME_INTEGER;
-     }
-     return LTI_ALL_INTEGER;
-}
 BIF_RETTYPE string_to_integer_1(BIF_ALIST_1)
 {
      Eterm res;
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 02d37e24df..29e677d2e5 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2732,3 +2732,156 @@ bytebuf_to_integer_1_done:
     return res;
 
 }
+
+int do_list_to_integer(Process *p, Eterm orig_list,
+                       Eterm *integer, Eterm *rest)
+{
+     Sint i = 0;
+     Uint ui = 0;
+     int skip = 0;
+     int neg = 0;
+     Sint n = 0;
+     int m;
+     int lg2;
+     Eterm res;
+     Eterm* hp;
+     Eterm *hp_end;
+     Eterm lst = orig_list;
+     Eterm tail = lst;
+     int error_res = LTI_BAD_STRUCTURE;
+
+     if (is_nil(lst)) {
+       error_res = LTI_NO_INTEGER;
+     error:
+         *rest = tail;
+         *integer = make_small(0);
+         return error_res;
+     }
+     if (is_not_list(lst))
+       goto error;
+
+     /* if first char is a '-' then it is a negative integer */
+     if (CAR(list_val(lst)) == make_small('-')) {
+          neg = 1;
+          skip = 1;
+          lst = CDR(list_val(lst));
+          if (is_not_list(lst)) {
+              tail = lst;
+              error_res = LTI_NO_INTEGER;
+              goto error;
+          }
+     } else if (CAR(list_val(lst)) == make_small('+')) {
+         /* ignore plus */
+         skip = 1;
+         lst = CDR(list_val(lst));
+         if (is_not_list(lst)) {
+             tail = lst;
+             error_res = LTI_NO_INTEGER;
+             goto error;
+         }
+     }
+
+     /* Calculate size and do type check */
+
+     while(1) {
+         if (is_not_small(CAR(list_val(lst)))) {
+             break;
+         }
+         if (unsigned_val(CAR(list_val(lst))) < '0' ||
+             unsigned_val(CAR(list_val(lst))) > '9') {
+             break;
+         }
+         ui = ui * 10;
+         ui = ui + unsigned_val(CAR(list_val(lst))) - '0';
+         n++;
+         lst = CDR(list_val(lst));
+         if (is_nil(lst)) {
+             break;
+         }
+         if (is_not_list(lst)) {
+             break;
+         }
+     }
+
+     tail = lst;
+     if (!n) {
+         error_res = LTI_NO_INTEGER;
+         goto error;
+     }
+
+
+      /* If n <= 8 then we know it's a small int
+      ** since 2^27 = 134217728. If n > 8 then we must
+      ** construct a bignum and let that routine do the checking
+      */
+
+     if (n <= SMALL_DIGITS) {  /* It must be small */
+         if (neg) i = -(Sint)ui;
+         else i = (Sint)ui;
+         res = make_small(i);
+     } else {
+         /* Convert from log10 to log2 by multiplying with 1/log10(2)=3.3219
+            which we round up to (3 + 1/3) */
+         lg2 = (n+1)*3 + (n+1)/3 + 1;
+         m  = (lg2+D_EXP-1)/D_EXP; /* number of digits */
+         m  = BIG_NEED_SIZE(m);    /* number of words + thing */
+
+         hp = HAlloc(p, m);
+         hp_end = hp + m;
+
+         lst = orig_list;
+         if (skip)
+             lst = CDR(list_val(lst));
+
+         /* load first digits (at least one digit) */
+         if ((i = (n % D_DECIMAL_EXP)) == 0)
+             i = D_DECIMAL_EXP;
+         n -= i;
+         m = 0;
+         while(i--) {
+             m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0');
+             lst = CDR(list_val(lst));
+         }
+         res = small_to_big(m, hp);  /* load first digits */
+
+         while(n) {
+             i = D_DECIMAL_EXP;
+             n -= D_DECIMAL_EXP;
+             m = 0;
+             while(i--) {
+                 m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0');
+                 lst = CDR(list_val(lst));
+             }
+             if (is_small(res))
+                 res = small_to_big(signed_val(res), hp);
+             res = big_times_small(res, D_DECIMAL_BASE, hp);
+             if (is_small(res))
+                 res = small_to_big(signed_val(res), hp);
+             res = big_plus_small(res, m, hp);
+         }
+
+         if (neg) {
+             if (is_small(res))
+                 res = make_small(-signed_val(res));
+             else {
+                 Uint *big = big_val(res); /* point to thing */
+                 *big = bignum_header_neg(*big);
+             }
+         }
+
+         if (is_not_small(res)) {
+             res = big_plus_small(res, 0, hp); /* includes conversion to small */
+
+             if (is_not_small(res)) {
+                 hp += (big_arity(res)+1);
+             }
+         }
+         HRelease(p,hp_end,hp);
+     }
+     *integer = res;
+     *rest = tail;
+     if (tail != NIL) {
+         return LTI_SOME_INTEGER;
+     }
+     return LTI_ALL_INTEGER;
+}
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 94f9bce10e..85807d6eea 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -173,4 +173,12 @@ Eterm erts_sint64_to_big(Sint64, Eterm **);
 
 Eterm erts_chars_to_integer(Process *, char*, Uint, const int);
 
+#define LTI_BAD_STRUCTURE 0
+#define LTI_NO_INTEGER 1
+#define LTI_SOME_INTEGER 2
+#define LTI_ALL_INTEGER 3
+
+int do_list_to_integer(Process *p, Eterm orig_list,
+                       Eterm *integer, Eterm *rest);
+
 #endif
-- 
cgit v1.2.3


From cd6903be0740db0c0061533cfb46729b43016316 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 27 Jan 2016 15:13:00 +0100
Subject: erts: When erts_alloc fails, the emulator no longer aborts

---
 erts/emulator/beam/erl_alloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index c3f4fe5a63..2a97069ac2 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1923,7 +1923,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
 	va_start(argp, n);
 	size = va_arg(argp, Uint);
 	va_end(argp);
-	erl_exit(1,
+	erl_exit(-1,
 		 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
 		 allctr_str, op, size, t_str);
 	break;
-- 
cgit v1.2.3


From 042677624b1d7b3f4c99be4e1483180e7fe8b2c0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 28 Jan 2016 15:09:35 +0100
Subject: erts: Fix bug concerning line information for hipe modules

Line table was left uninitialized for hipe (stub) modules
causing process_info(OtherPid, current_location) to crash.
---
 erts/emulator/beam/beam_load.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index b70e5b9a2d..a6dce2d1d2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6249,6 +6249,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     code[MI_LITERALS_END] = 0;
     code[MI_LITERALS_OFF_HEAP] = 0;
     code[MI_ON_LOAD_FUNCTION_PTR] = 0;
+    code[MI_LINE_TABLE] = 0;
     code[MI_MD5_PTR] = 0;
     ci = MI_FUNCTIONS + n + 1;
 
-- 
cgit v1.2.3


From aa62ef506163c3b56d64aa0c77350775697b5999 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 29 Jan 2016 11:56:18 +0100
Subject: Remove faulty asserts

---
 erts/emulator/beam/erl_gc.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6839c24c25..06acece471 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -729,9 +729,7 @@ erts_garbage_collect_hibernate(Process* p)
      */
     erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     ErtsGcQuickSanityCheck(p);
-    ASSERT(p->mbuf == NULL);
     ASSERT(p->stop == p->hend);	/* Stack must be empty. */
-    ASSERT(!p->abandoned_heap);
 
     /*
      * Do it.
-- 
cgit v1.2.3


From d76ee58c07f32dfc0652844ec2b513af2105ffa1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Wed, 22 Oct 2014 17:15:31 +0200
Subject: erts: Add ERTS_WRITE_UNLIKELY

ERTS_WRITE_UNLIKELY can be used to place global variables in
a specific section where only data that is very rarely modified
sits. This is used to improve cache locality.
---
 erts/emulator/beam/erl_alloc.c   |  4 ++--
 erts/emulator/beam/erl_process.c | 24 ++++++++++++------------
 erts/emulator/beam/sys.h         | 11 +++++++++++
 3 files changed, 25 insertions(+), 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 5544712e8d..51b522c9e0 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -103,9 +103,9 @@ static Uint install_debug_functions(void);
 
 static int lock_all_physical_memory = 0;
 
-ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
+ErtsAllocatorFunctions_t ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]);
 ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
-ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
+ErtsAllocatorThrSpec_t ERTS_WRITE_UNLIKELY(erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]);
 
 #define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B))
 #define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B))
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7d194f9840..3902632ef8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -148,14 +148,14 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
-int erts_default_spo_flags = 0;
-int erts_eager_check_io = 1;
-int erts_sched_compact_load;
-int erts_sched_balance_util = 0;
-Uint erts_no_schedulers;
+int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = 0;
+int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
+int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
+int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
+Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
 #ifdef ERTS_DIRTY_SCHEDULERS
-Uint erts_no_dirty_cpu_schedulers;
-Uint erts_no_dirty_io_schedulers;
+Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
+Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
 #endif
 
 static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
@@ -295,7 +295,7 @@ do {							\
 erts_sched_stat_t erts_sched_stat;
 
 #ifdef USE_THREADS
-static erts_tsd_key_t sched_data_key;
+static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key);
 #endif
 
 static erts_smp_atomic32_t function_calls;
@@ -335,10 +335,10 @@ static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info;
 
 static Uint last_reductions;
 static Uint last_exact_reductions;
-Eterm erts_system_monitor;
-Eterm erts_system_monitor_long_gc;
-Uint erts_system_monitor_long_schedule;
-Eterm erts_system_monitor_large_heap;
+Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor);
+Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc);
+Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule);
+Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap);
 struct erts_system_monitor_flags_t erts_system_monitor_flags;
 
 /* system performance monitor */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 279e4db0af..e77b4b2dc8 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -136,6 +136,17 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #  define ERTS_LIKELY(BOOL)   (BOOL)
 #  define ERTS_UNLIKELY(BOOL) (BOOL)
 #endif
+
+#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
+#ifndef __llvm__
+#  define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("ERTS_LOW_WRITE") ))
+#else
+#  define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") ))
+#endif
+#else
+#  define ERTS_WRITE_UNLIKELY(X) X
+#endif
+
 #ifdef __GNUC__
 #  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
 #    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
-- 
cgit v1.2.3


From eea5f896780e07f7ca76685061d01e7be5a7abaa Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 11 Sep 2014 18:26:26 +0200
Subject: erts, kernel: Add os:perf_counter function

The perf_counter is a very very cheap and high resolution timer
that can be used to timestamp system events. It does not have
monoticity guarantees, but should on most OS's expose a monotonous
time.

A special instruction has been created for this counter to further
speed up fetching it.

OTP-12908
---
 erts/emulator/beam/beam_emu.c     | 32 +++++++++++++++++
 erts/emulator/beam/bif.tab        |  2 ++
 erts/emulator/beam/erl_time.h     |  2 ++
 erts/emulator/beam/erl_time_sup.c | 76 +++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/ops.tab        |  9 +++++
 5 files changed, 119 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4d7b00b032..a0979c7b0e 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4895,6 +4895,38 @@ do {						\
      }
  }
 
+ /* This is optimised as an instruction because
+    it has to be very very fast */
+ OpCase(i_perf_counter): {
+    BeamInstr* next;
+    ErtsSysHrTime ts;
+    PreFetch(0, next);
+
+    sys_perf_counter(&ts);
+
+    if (IS_SSMALL(ts)) {
+        r(0) = make_small((Sint)ts);
+    } else {
+        TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0);
+        r(0) = make_big(HTOP);
+#if defined(ARCH_32) || HALFWORD_HEAP
+        if (ts >= (((Uint64) 1) << 32)) {
+            *HTOP = make_pos_bignum_header(2);
+            BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff));
+            BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff));
+            HTOP += 3;
+        }
+        else
+#endif
+        {
+                *HTOP = make_pos_bignum_header(1);
+                BIG_DIGIT(HTOP, 0) = (Uint) ts;
+                HTOP += 2;
+        }
+    }
+    NextPF(0, next);
+ }
+
  OpCase(i_debug_breakpoint): {
      HEAVY_SWAPOUT;
      I = call_error_handler(c_p, I-3, reg, am_breakpoint);
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 1b8ae8cef5..4efc055aaf 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -170,6 +170,7 @@ bif erts_internal:term_type/1
 bif erts_internal:map_hashmap_children/1
 
 bif erts_internal:time_unit/0
+bif erts_internal:perf_counter_unit/0
 
 bif erts_internal:is_system_process/1
 
@@ -369,6 +370,7 @@ bif os:getpid/0
 bif os:timestamp/0
 bif os:system_time/0
 bif os:system_time/1
+bif os:perf_counter/0
 
 #
 # Bifs in the erl_ddll module (the module actually does not exist)
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 93a0d556bf..446adcf4af 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -142,6 +142,8 @@ erts_time_unit_conversion(Uint64 value,
 			  Uint32 from_time_unit,
 			  Uint32 to_time_unit);
 
+ErtsSysHrTime erts_perf_counter_unit(void);
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE Uint64
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 668dbb5daa..6509c6a805 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1749,7 +1749,7 @@ erts_get_monotonic_time(ErtsSchedulerData *esdp)
 {
     ErtsMonotonicTime mtime = time_sup.r.o.get_time();
     update_last_mtime(esdp, mtime);
-    return mtime;    
+    return mtime;
 }
 
 ErtsMonotonicTime
@@ -2435,9 +2435,81 @@ BIF_RETTYPE os_system_time_0(BIF_ALIST_0)
     BIF_RET(make_time_val(BIF_P, stime));
 }
 
-BIF_RETTYPE os_system_time_1(BIF_ALIST_0)
+BIF_RETTYPE os_system_time_1(BIF_ALIST_1)
 {
     ErtsSystemTime stime = erts_os_system_time();
     BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0));
 }
 
+BIF_RETTYPE
+os_perf_counter_0(BIF_ALIST_0)
+{
+    ErtsSysHrTime pcounter;
+    sys_perf_counter(&pcounter);
+    BIF_RET(make_time_val(BIF_P, pcounter));
+}
+
+BIF_RETTYPE erts_internal_perf_counter_unit_0(BIF_ALIST_0)
+{
+    BIF_RET(make_time_val(BIF_P, SYS_PERF_COUNTER_UNIT));
+}
+
+/* What resolution to spin to in micro seconds */
+#define RESOLUTION 100
+/* How many iterations to spin */
+#define ITERATIONS 1
+/* How many significant figures to round to */
+#define SIGFIGS 3
+
+static ErtsSysHrTime perf_counter_unit = 0;
+
+static ErtsSysHrTime erts_calculate_perf_counter_unit(void);
+static ErtsSysHrTime erts_calculate_perf_counter_unit() {
+    int i;
+    ErtsSysHrTime pre, post;
+    double value = 0;
+    double round_factor;
+#if defined(HAVE_GETHRTIME) && defined(GETHRTIME_WITH_CLOCK_GETTIME)
+    struct timespec basetime,comparetime;
+#define __GETTIME(arg) clock_gettime(CLOCK_MONOTONIC,arg)
+#define __GETUSEC(arg) (arg.tv_nsec / 1000)
+#else
+    SysTimeval basetime,comparetime;
+#define __GETTIME(arg) sys_gettimeofday(arg)
+#define __GETUSEC(arg) arg.tv_usec
+#endif
+
+    for (i = 0; i < ITERATIONS; i++) {
+        /* Make sure usec just flipped over at current resolution */
+        __GETTIME(&basetime);
+        do {
+            __GETTIME(&comparetime);
+        } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
+
+        sys_perf_counter(&pre);
+
+        __GETTIME(&basetime);
+        do {
+            __GETTIME(&comparetime);
+        } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
+
+        sys_perf_counter(&post);
+
+        value += post - pre;
+    }
+    /* After this value is ticks per us */
+    value /= (RESOLUTION*ITERATIONS);
+
+    /* We round to 3 significant figures */
+    round_factor = pow(10.0, SIGFIGS - ceil(log10(value)));
+    value = ((ErtsSysHrTime)(value * round_factor + 0.5)) / round_factor;
+
+    /* convert to ticks per second */
+    return 1000000 * value;
+}
+
+ErtsSysHrTime erts_perf_counter_unit() {
+    if (perf_counter_unit == 0)
+        perf_counter_unit = erts_calculate_perf_counter_unit();
+    return perf_counter_unit;
+}
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 081c4108a0..9e53b4bfcc 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -988,6 +988,13 @@ move Discarded x==0 | move Something x==0 => move Something x=0
 
 %endif
 
+call_ext u==0 u$func:os:perf_counter/0 => \
+    i_perf_counter
+call_ext_last u==0 u$func:os:perf_counter/0 D => \
+    i_perf_counter | deallocate_return D
+call_ext_only u==0 u$func:os:perf_counter/0 => \
+    i_perf_counter | return
+
 #
 # The general case for BIFs that have no special instructions.
 # A BIF used in the tail must be followed by a return instruction.
@@ -1027,6 +1034,8 @@ i_apply_fun_only
 
 i_hibernate
 
+i_perf_counter
+
 call_bif e
 
 #
-- 
cgit v1.2.3


From 664ed2a6fd2b324bb6b56db3d3eca853cfda8f61 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Fri, 12 Sep 2014 16:38:00 +0200
Subject: erts: Add microstate accounting

Microstate accounting is a way to track which state the
different threads within ERTS are in. The main usage area
is to pin point performance bottlenecks by checking which
states the threads are in and then from there figuring out
why and where to optimize.

Since checking whether microstate accounting is on or off is
relatively expensive if done in a short loop only a few of the
states are enabled by default and more states can be enabled
through configure.

I've done some benchmarking and the overhead with it turned off
is not noticible and with it on it is a fraction of a percent.
If you enable the extra states, depending on the benchmark,
the ovehead when turned off is about 1% and when turned on
somewhere inbetween 5-15%.

OTP-12345
---
 erts/emulator/beam/atom.names       |   4 +
 erts/emulator/beam/beam_emu.c       |  43 +++-
 erts/emulator/beam/bif.c            |  48 +++-
 erts/emulator/beam/erl_alloc.h      |  18 +-
 erts/emulator/beam/erl_alloc.types  |   1 +
 erts/emulator/beam/erl_async.c      |  34 ++-
 erts/emulator/beam/erl_bif_info.c   |  12 +
 erts/emulator/beam/erl_gc.c         |   7 +
 erts/emulator/beam/erl_init.c       |   3 +
 erts/emulator/beam/erl_lock_check.c |   2 +
 erts/emulator/beam/erl_msacc.c      | 486 ++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_msacc.h      | 409 ++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_port_task.c  |   3 +
 erts/emulator/beam/erl_process.c    |  48 +++-
 erts/emulator/beam/erl_threads.h    |  34 ++-
 erts/emulator/beam/global.h         |   2 +
 erts/emulator/beam/io.c             |  32 +++
 erts/emulator/beam/time.c           |  20 +-
 18 files changed, 1182 insertions(+), 24 deletions(-)
 create mode 100644 erts/emulator/beam/erl_msacc.c
 create mode 100644 erts/emulator/beam/erl_msacc.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 1629a92aa9..6fb08ee896 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -102,6 +102,7 @@ atom asynchronous
 atom atom
 atom atom_used
 atom attributes
+atom await_microstate_accounting_modifications
 atom await_port_send_result
 atom await_proc_exit
 atom await_result
@@ -172,6 +173,7 @@ atom const
 atom context_switches
 atom control
 atom copy
+atom counters
 atom cpu
 atom cpu_timestamp
 atom cr
@@ -269,6 +271,7 @@ atom get_tcw
 atom getenv
 atom gather_gc_info_result
 atom gather_io_bytes
+atom gather_microstate_accounting_result
 atom gather_sched_wall_time_result
 atom getting_linked
 atom getting_unlinked
@@ -360,6 +363,7 @@ atom merge_trap
 atom meta
 atom meta_match_spec
 atom micro_seconds
+atom microstate_accounting
 atom milli_seconds
 atom min_heap_size
 atom min_bin_vheap_size
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a0979c7b0e..14b81a0a6e 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -113,6 +113,9 @@ do {                                     \
 #define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #endif
 
+#define GET_BIF_MODULE(p)  ((Eterm) (((Export *) p)->code[0]))
+#define GET_BIF_FUNCTION(p)  ((Eterm) (((Export *) p)->code[1]))
+#define GET_BIF_ARITY(p)  ((Eterm) (((Export *) p)->code[2]))
 #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
 #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
@@ -1249,6 +1252,8 @@ void process_main(void)
     Uint64 start_time = 0;          /* Monitor long schedule */
     BeamInstr* start_time_i = NULL;
 
+    ERTS_MSACC_DECLARE_CACHE_X(); /* a cached value of the tsd pointer for msacc */
+
     ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
 
 
@@ -1302,6 +1307,8 @@ void process_main(void)
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
+    ERTS_MSACC_UPDATE_CACHE_X();
+
     if (erts_system_monitor_long_schedule != 0) {
 	start_time = erts_timestamp_millis();
 	start_time_i = c_p->i;
@@ -2741,11 +2748,21 @@ do {						\
      */
  OpCase(call_bif_e):
     {
-	Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
+        Eterm (*bf)(Process*, Eterm*, BeamInstr*);
 	Eterm result;
 	BeamInstr *next;
 	ErlHeapFragment *live_hf_end;
 
+        if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
+            if (GET_BIF_MODULE(Arg(0)) == am_ets) {
+                ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
+            } else {
+                ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
+            }
+        }
+
+	bf = GET_BIF_ADDRESS(Arg(0));
+
 	PRE_BIF_SWAPOUT(c_p);
 	c_p->fcalls = FCALLS - 1;
 	if (FCALLS <= 0) {
@@ -2768,6 +2785,12 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	HTOP = HEAP_TOP(c_p);
 	FCALLS = c_p->fcalls;
+        /* We have to update the cache if we are enabled in order
+           to make sure no book keeping is done after we disabled
+           msacc. We don't always do this as it is quite expensive. */
+        if (ERTS_MSACC_IS_ENABLED_CACHED_X())
+            ERTS_MSACC_UPDATE_CACHE_X();
+	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
 	if (is_value(result)) {
 	    r(0) = result;
 	    CHECK_TERM(r(0));
@@ -3452,6 +3475,8 @@ do {						\
 	    BifFunction vbf;
 	    ErlHeapFragment *live_hf_end;
 
+	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
+
 	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
 	    SWAPOUT;
@@ -3476,6 +3501,8 @@ do {						\
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 
+	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+
 	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    goto apply_bif_or_nif_epilogue;
 	 
@@ -3490,6 +3517,13 @@ do {						\
 	     * code[3]: &&apply_bif
 	     * code[4]: Function pointer to BIF function
 	     */
+            if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
+                if ((Eterm)I[-3] == am_ets) {
+                    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
+                } else {
+                    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
+                }
+            }
 
 	    c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
 	    c_p->i = I;		/* In case we apply check_process_code/2. */
@@ -3516,7 +3550,12 @@ do {						\
 		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		PROCESS_MAIN_CHK_LOCKS(c_p);
 	    }
-
+            /* We have to update the cache if we are enabled in order
+               to make sure no book keeping is done after we disabled
+               msacc. We don't always do this as it is quite expensive. */
+            if (ERTS_MSACC_IS_ENABLED_CACHED_X())
+                ERTS_MSACC_UPDATE_CACHE_X();
+	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
 	    DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 
 	apply_bif_or_nif_epilogue:
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index bb9165cd79..7e8eb44680 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -44,6 +44,7 @@
 #include "erl_ptab.h"
 #include "erl_bits.h"
 #include "erl_bif_unique.h"
+#include "erl_msacc.h"
 
 Export *erts_await_result;
 static Export* flush_monitor_messages_trap = NULL;
@@ -54,6 +55,9 @@ Export* erts_format_cpu_topology_trap = NULL;
 static Export dsend_continue_trap_export;
 Export *erts_convert_time_unit_trap = NULL;
 
+static Export *await_msacc_mod_trap = NULL;
+static erts_smp_atomic32_t msacc;
+
 static Export *await_sched_wall_time_mod_trap;
 static erts_smp_atomic32_t sched_wall_time;
 
@@ -2143,7 +2147,11 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     int connect = !0;
     Eterm l = opts;
     Sint result;
+
     DeclareTypedTmpHeap(ErtsSendContext, ctx, BIF_P);
+
+    ERTS_MSACC_PUSH_STATE_M_X();
+
     UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P);
 
     ctx->suspend = !0;
@@ -2172,7 +2180,10 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     ref = NIL;
 #endif
 
+    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_SEND);
     result = do_send(p, to, msg, &ref, ctx);
+    ERTS_MSACC_POP_STATE_M_X();
+
     if (result > 0) {
 	ERTS_VBUMP_REDS(p, result);
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
@@ -2288,8 +2299,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
     Eterm ref;
     Sint result;
     DeclareTypedTmpHeap(ErtsSendContext, ctx, p);
+    ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_SEND);
     UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p);
-
 #ifdef DEBUG
     ref = NIL;
 #endif
@@ -2300,7 +2311,9 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
     ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT;
 
     result = do_send(p, to, msg, &ref, ctx);
-    
+
+    ERTS_MSACC_POP_STATE_M_X();
+
     if (result > 0) {
 	ERTS_VBUMP_REDS(p, result);
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
@@ -4586,6 +4599,31 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	default:
 	    ERTS_INTERNAL_ERROR("Unknown state");
 	}
+#ifdef ERTS_ENABLE_MSACC
+    } else if (BIF_ARG_1 == am_microstate_accounting) {
+      Eterm threads;
+      if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
+        erts_aint32_t new = BIF_ARG_2 == am_true ? ERTS_MSACC_ENABLE : ERTS_MSACC_DISABLE;
+	erts_aint32_t old = erts_smp_atomic32_xchg_nob(&msacc, new);
+	Eterm ref = erts_msacc_request(BIF_P, new, &threads);
+        if (is_non_value(ref))
+            BIF_RET(old ? am_true : am_false);
+	BIF_TRAP3(await_msacc_mod_trap,
+		  BIF_P,
+		  ref,
+		  old ? am_true : am_false,
+		  threads);
+      } else if (BIF_ARG_2 == am_reset) {
+	Eterm ref = erts_msacc_request(BIF_P, ERTS_MSACC_RESET, &threads);
+	erts_aint32_t old = erts_smp_atomic32_read_nob(&msacc);
+	ASSERT(is_value(ref));
+	BIF_TRAP3(await_msacc_mod_trap,
+		  BIF_P,
+		  ref,
+		  old ? am_true : am_false,
+		  threads);
+      }
+#endif
     } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) {
 	int what;
 	if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2))
@@ -4915,8 +4953,12 @@ void erts_init_bif(void)
     await_port_send_result_trap
 	= erts_export_put(am_erts_internal, am_await_port_send_result, 3);
     await_sched_wall_time_mod_trap
-	= erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2);
+        = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2);
+    await_msacc_mod_trap
+	= erts_export_put(am_erts_internal, am_await_microstate_accounting_modifications, 3);
+
     erts_smp_atomic32_init_nob(&sched_wall_time, 0);
+    erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED());
 }
 
 #ifdef HARDDEBUG
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 14e80960f5..71e4713624 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -233,12 +233,14 @@ ERTS_ALC_INLINE
 void *erts_alloc(ErtsAlcType_t type, Uint size)
 {
     void *res;
+    ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
     res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
 	ERTS_ALC_T2N(type),
 	erts_allctrs[ERTS_ALC_T2A(type)].extra,
 	size);
     if (!res)
 	erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
+    ERTS_MSACC_POP_STATE_X();
     return res;
 }
 
@@ -246,6 +248,7 @@ ERTS_ALC_INLINE
 void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size)
 {
     void *res;
+    ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
     res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)(
 	ERTS_ALC_T2N(type),
 	erts_allctrs[ERTS_ALC_T2A(type)].extra,
@@ -253,37 +256,48 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size)
 	size);
     if (!res)
 	erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size);
+    ERTS_MSACC_POP_STATE_X();
     return res;
 }
 
 ERTS_ALC_INLINE
 void erts_free(ErtsAlcType_t type, void *ptr)
 {
+    ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
     (*erts_allctrs[ERTS_ALC_T2A(type)].free)(
 	ERTS_ALC_T2N(type),
 	erts_allctrs[ERTS_ALC_T2A(type)].extra,
 	ptr);
+    ERTS_MSACC_POP_STATE_X();
 }
 
 
 ERTS_ALC_INLINE
 void *erts_alloc_fnf(ErtsAlcType_t type, Uint size)
 {
-    return (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
+    void *res;
+    ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
+    res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
 	ERTS_ALC_T2N(type),
 	erts_allctrs[ERTS_ALC_T2A(type)].extra,
 	size);
+    ERTS_MSACC_POP_STATE_X();
+    return res;
 }
 
 
 ERTS_ALC_INLINE
 void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size)
 {
-    return (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)(
+    void *res;
+    ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
+    res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)(
 	ERTS_ALC_T2N(type),
 	erts_allctrs[ERTS_ALC_T2A(type)].extra,
 	ptr,
 	size);
+    ERTS_MSACC_POP_STATE_X();
+    return res;
 }
 
 ERTS_ALC_INLINE
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index e0bc71c88a..6b7eff1428 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -361,6 +361,7 @@ type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
+type    MSACC           LONG_LIVED      SYSTEM          microstate_accounting
 
 #
 # Types used by system specific code
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index be0bc0cfec..cdeeb5281b 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -400,13 +400,19 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a)
 				     ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
 #endif
     if (!p) {
-	if (a->async_free)
+	if (a->async_free) {
+            ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT);
 	    a->async_free(a->async_data);
+            ERTS_MSACC_POP_STATE();
+        }
     }
     else {
 	if (async_ready(p, a->async_data)) {
-	    if (a->async_free)
+	    if (a->async_free) {
+                ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT);
 		a->async_free(a->async_data);
+                ERTS_MSACC_POP_STATE();
+            }
 	}
 #if ERTS_USE_ASYNC_READY_Q
 	erts_port_release(p);
@@ -460,6 +466,8 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq)
 {
     ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
     erts_tse_t *tse = erts_tse_fetch();
+    ERTS_DECLARE_DUMMY(Uint no);
+
 #ifdef ERTS_SMP
     ErtsThrPrgrCallbacks callbacks;
 
@@ -483,10 +491,12 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq)
 
     /* Inform main thread that we are done initializing... */
     erts_mtx_lock(&async->init.data.mtx);
-    async->init.data.no_initialized++;
+    no = async->init.data.no_initialized++;
     erts_cnd_signal(&async->init.data.cnd);
     erts_mtx_unlock(&async->init.data.mtx);
 
+    erts_msacc_init_thread("async", no, 0);
+
     return tse;
 }
 
@@ -494,6 +504,7 @@ static void *async_main(void* arg)
 {
     ErtsAsyncQ *aq = (ErtsAsyncQ *) arg;
     erts_tse_t *tse = async_thread_init(aq);
+    ERTS_MSACC_DECLARE_CACHE();
 
     while (1) {
 	ErtsThrQPrepEnQ_t *prep_enq;
@@ -501,11 +512,14 @@ static void *async_main(void* arg)
 	if (is_nil(a->port))
 	    break; /* Time to die */
 
+        ERTS_MSACC_UPDATE_CACHE();
+
 #if ERTS_ASYNC_PRINT_JOB
 	erts_fprintf(stderr, "<- %ld\n", a->async_id);
 #endif
-
+        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT);
 	a->async_invoke(a->async_data);
+        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
 
 	async_reply(a, prep_enq);
     }
@@ -628,10 +642,13 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
     unsigned int qix;
 #if ERTS_USE_ASYNC_READY_Q
     Uint sched_id;
+    ERTS_MSACC_PUSH_STATE();
 
     sched_id = erts_get_scheduler_id();
     if (!sched_id)
 	sched_id = 1;
+#else
+    ERTS_MSACC_PUSH_STATE();
 #endif
 
     prt = erts_drvport2port(ix);
@@ -684,12 +701,17 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 	return id;
     }
 #endif
-
+    
+    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT);
     (*a->async_invoke)(a->async_data);
+    ERTS_MSACC_POP_STATE();
 
     if (async_ready(prt, a->async_data)) {
-	if (a->async_free != NULL)
+	if (a->async_free != NULL) {
+            ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT);
 	    (*a->async_free)(a->async_data);
+            ERTS_MSACC_POP_STATE();
+        }
     }
     erts_free(ERTS_ALC_T_ASYNC, (void *) a);
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 06e7dc8661..017339e1f6 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -64,6 +64,7 @@ static Export* alloc_sizes_trap = NULL;
 static Export* gather_io_bytes_trap = NULL;
 
 static Export *gather_sched_wall_time_res_trap;
+static Export *gather_msacc_res_trap;
 static Export *gather_gc_info_res_trap;
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
@@ -3260,6 +3261,14 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	    szp = NULL;
 	    hpp = &hp;
 	}
+#ifdef ERTS_ENABLE_MSACC
+    } else if (BIF_ARG_1 == am_microstate_accounting) {
+        Eterm threads;
+        res = erts_msacc_request(BIF_P, ERTS_MSACC_GATHER, &threads);
+	if (is_non_value(res))
+	    BIF_RET(am_undefined);
+	BIF_TRAP2(gather_msacc_res_trap, BIF_P, res, threads);
+#endif
     } else if (BIF_ARG_1 == am_context_switches) {
 	Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P);
 	hp = HAlloc(BIF_P, 3);
@@ -4396,6 +4405,9 @@ erts_bif_info_init(void)
 	= erts_export_put(am_erlang, am_gather_gc_info_result, 1);
     gather_io_bytes_trap
 	= erts_export_put(am_erts_internal, am_gather_io_bytes, 2);
+    gather_msacc_res_trap
+	= erts_export_put(am_erts_internal, am_gather_microstate_accounting_result, 2);
+
     process_info_init();
     os_info_init();
 }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 06acece471..21b03ae8bd 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -576,6 +576,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     int reds;
     ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
     ErtsSchedulerData *esdp;
+    ERTS_MSACC_PUSH_STATE_M();
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
@@ -588,6 +589,8 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     else if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
 	live_hf_end = p->live_hf_end;
 
+    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC);
+
     esdp = erts_get_scheduler_data();
 
     if (IS_TRACED_FL(p, F_TRACE_GC)) {
@@ -624,9 +627,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     }
     else {
     do_major_collection:
+        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL);
 	DTRACE2(gc_major_start, pidbuf, need);
 	reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
 	DTRACE2(gc_major_end, pidbuf, reclaimed_now);
+        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC);
     }
 
     reset_active_writer(p);
@@ -669,6 +674,8 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     FLAGS(p) &= ~F_FORCE_GC;
     p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
 
+    ERTS_MSACC_POP_STATE_M();
+
 #ifdef CHECK_FOR_HOLES
     /*
      * We intentionally do not rescan the areas copied by the GC.
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 42aca726bf..520b504fcb 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -394,6 +394,7 @@ erl_init(int ncpu,
 #endif
     packet_parser_init();
     erl_nif_init();
+    erts_msacc_init();
 }
 
 static Eterm
@@ -1193,6 +1194,7 @@ early_init(int *argc, char **argv) /*
 	erts_thr_late_init(&elid);
     }
 #endif
+    erts_msacc_early_init();
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_late_init();
@@ -2220,6 +2222,7 @@ erl_start(int argc, char **argv)
 #else
     {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+        erts_msacc_init_thread("scheduler", 1, 1);
 	erts_thr_set_main_status(1, 1);
 #if ERTS_USE_ASYNC_READY_Q
 	esdp->aux_work_data.async_ready.queue
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index f7b4bd8041..598bf84c0b 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -137,6 +137,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"mmap_init_atoms",			NULL			},
     {	"drv_tsd",				NULL			},
     {	"async_enq_mtx",			NULL			},
+    {   "msacc_list_mutex",                     NULL                    },
+    {   "msacc_unmanaged_mutex",                NULL                    },
 #ifdef ERTS_SMP
     {	"atom_tab",				NULL			},
     {	"misc_op_list_pre_alloc_lock",		"address"		},
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
new file mode 100644
index 0000000000..bf1c06dea7
--- /dev/null
+++ b/erts/emulator/beam/erl_msacc.c
@@ -0,0 +1,486 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2014-2015. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Microstate accounting.
+ *
+ *              We keep track of the different states that the
+ *              Erlang VM threads are in, in order to provide
+ *              performance/debugging statistics. There is a
+ *              small overhead in enabling this, but in the big
+ *              scheme of things it should be negligible.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#define ERTS_MSACC_STATE_STRINGS 1
+
+#include "sys.h"
+#include "global.h"
+#include "erl_threads.h"
+#include "erl_bif_unique.h"
+#include "erl_map.h"
+#include "erl_msacc.h"
+
+#if ERTS_ENABLE_MSACC
+
+static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp);
+static void erts_msacc_reset(ErtsMsAcc *msacc);
+static ErtsMsAcc* get_msacc(void);
+
+#ifdef USE_THREADS
+erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key);
+#else
+ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL;
+#endif
+int ERTS_WRITE_UNLIKELY(erts_msacc_enabled);
+
+static Eterm *erts_msacc_state_atoms = NULL;
+static erts_rwmtx_t msacc_mutex;
+static ErtsMsAcc *msacc_managed = NULL;
+#ifdef USE_THREADS
+static ErtsMsAcc *msacc_unmanaged = NULL;
+static Uint msacc_unmanaged_count = 0;
+#endif
+
+/* we have to split initiation as atoms are not inited in early init */
+void erts_msacc_early_init(void) {
+#ifndef ERTS_MSACC_ALWAYS_ON
+    erts_msacc_enabled = 0;
+#endif
+    erts_rwmtx_init(&msacc_mutex,"msacc_list_mutex");
+#ifdef USE_THREADS
+    erts_tsd_key_create(&erts_msacc_key,"erts_msacc_key");
+#else
+    erts_msacc = NULL;
+#endif
+}
+
+void erts_msacc_init(void) {
+    int i;
+    erts_msacc_state_atoms = erts_alloc(ERTS_ALC_T_MSACC,
+                                        sizeof(Eterm)*ERTS_MSACC_STATE_COUNT);
+    for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
+        erts_msacc_state_atoms[i] = am_atom_put(erts_msacc_states[i],
+                                                strlen(erts_msacc_states[i]));
+    }
+}
+
+void erts_msacc_init_thread(char *type, int id, int managed) {
+    ErtsMsAcc *msacc;
+
+    msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc));
+
+    msacc->type = strdup(type);
+    msacc->id = make_small(id);
+    msacc->unmanaged = !managed;
+    msacc->tid = erts_thr_self();
+    msacc->perf_counter = 0;
+
+#ifdef USE_THREADS
+    erts_rwmtx_rwlock(&msacc_mutex);
+    if (!managed) {
+        erts_mtx_init(&msacc->mtx,"msacc_unmanaged_mutex");
+        msacc->next = msacc_unmanaged;
+        msacc_unmanaged = msacc;
+        msacc_unmanaged_count++;
+        ERTS_MSACC_TSD_SET(msacc);
+    } else {
+        msacc->next = msacc_managed;
+        msacc_managed = msacc;
+    }
+    erts_rwmtx_rwunlock(&msacc_mutex);
+#else
+    msacc_managed = msacc;
+#endif
+
+    erts_msacc_reset(msacc);
+
+#ifdef ERTS_MSACC_ALWAYS_ON
+    ERTS_MSACC_TSD_SET(msacc);
+    sys_perf_counter(&msacc->perf_counter);
+    msacc->state = ERTS_MSACC_STATE_OTHER;
+#endif
+}
+
+/*
+ * Creates a structure looking like this
+ * #{ type => scheduler, id => 1, counters => #{ State1 => Counter1 ... StateN => CounterN}}
+ */
+static
+Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp) {
+    int i;
+    Eterm *hp;
+    Eterm key, state_key, state_map;
+    Eterm res = THE_NON_VALUE;
+    flatmap_t *map;
+
+    if (szp) {
+      *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(3);
+      *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(ERTS_MSACC_STATE_COUNT);
+      for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
+          (void)erts_bld_sint64(NULL,szp,(Sint64)msacc->perf_counters[i]);
+#ifdef ERTS_MSACC_STATE_COUNTERS
+          (void)erts_bld_uint64(NULL,szp,msacc->state_counters[i]);
+          *szp += 3; /* tuple to put state+perf counter in */
+#endif
+      }
+    }
+
+    if (hpp) {
+        Eterm counters[ERTS_MSACC_STATE_COUNT];
+        hp = *hpp;
+        for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
+            Eterm counter = erts_bld_sint64(&hp,NULL,(Sint64)msacc->perf_counters[i]);
+#ifdef ERTS_MSACC_STATE_COUNTERS
+            Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->state_counters[i]);
+            counters[i] = TUPLE2(hp,counter,counter__);
+            hp += 3;
+#else
+            counters[i] = counter;
+#endif
+        }
+
+      key = TUPLE3(hp,am_counters,am_id,am_type);
+      hp += 4;
+
+      state_key = make_tuple(hp);
+      hp[0] = make_arityval(ERTS_MSACC_STATE_COUNT);
+      
+      for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++)
+        hp[1+i] = erts_msacc_state_atoms[i];
+      hp += 1 + ERTS_MSACC_STATE_COUNT;
+
+      map = (flatmap_t*)hp;
+      hp += MAP_HEADER_FLATMAP_SZ;
+      map->thing_word = MAP_HEADER_FLATMAP;
+      map->size = ERTS_MSACC_STATE_COUNT;
+      map->keys = state_key;
+      for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++)
+            hp[i] = counters[i];
+      hp += ERTS_MSACC_STATE_COUNT;
+      state_map = make_flatmap(map);
+
+      map = (flatmap_t*)hp;
+      hp += MAP_HEADER_FLATMAP_SZ;
+      map->thing_word = MAP_HEADER_FLATMAP;
+      map->size = 3;
+      map->keys = key;
+      hp[0] = state_map;
+      hp[1] = msacc->id;
+      hp[2] = am_atom_put(msacc->type,strlen(msacc->type));
+      hp += 3;
+
+      *hpp = hp;
+      res = make_flatmap(map);
+    }
+
+    return res;
+}
+
+typedef struct {
+    int action;
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    Uint req_sched;
+    erts_smp_atomic32_t refc;
+} ErtsMSAccReq;
+
+static ErtsMsAcc* get_msacc(void) {
+    ErtsMsAcc *msacc;
+    erts_rwmtx_rlock(&msacc_mutex);
+    msacc = msacc_managed;
+    while (!erts_equal_tids(msacc->tid,erts_thr_self())) {
+        msacc = msacc->next;
+        ASSERT(msacc != NULL);
+    }
+    erts_rwmtx_runlock(&msacc_mutex);
+    return msacc;
+}
+
+static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    Process *rp = msaccrp->proc;
+    ErtsMessage *msgp = NULL;
+    Eterm **hpp, *hp;
+    Eterm ref_copy = NIL, msg;
+    Uint sz, *szp;
+    ErlOffHeap *ohp = NULL;
+    ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no
+                              ? ERTS_PROC_LOCK_MAIN : 0);
+
+    sz = 0;
+    hpp = NULL;
+    szp = &sz;
+
+    if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
+
+    while (1) {
+	if (hpp)
+            ref_copy = STORE_NC(hpp, ohp, msaccrp->ref);
+	else
+            *szp += REF_THING_SIZE;
+
+	if (msaccrp->action != ERTS_MSACC_GATHER)
+            msg = ref_copy;
+	else {
+            msg = erts_msacc_gather_stats(msacc, hpp, szp);
+            msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+	}
+	if (hpp)
+            break;
+
+	msgp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+        hpp = &hp;
+	szp = NULL;
+    }
+
+    if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
+
+    erts_queue_message(rp, &rp_locks, msgp, msg, NIL);
+
+    if (esdp && msaccrp->req_sched == esdp->no)
+	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+
+    if (rp_locks)
+	erts_smp_proc_unlock(rp, rp_locks);
+
+}
+
+static void
+reply_msacc(void *vmsaccrp)
+{
+    ErtsMsAcc *msacc = ERTS_MSACC_TSD_GET();
+    ErtsMSAccReq *msaccrp = (ErtsMSAccReq *) vmsaccrp;
+
+    ASSERT(!msacc || !msacc->unmanaged);
+
+    if (msaccrp->action == ERTS_MSACC_ENABLE && !msacc) {
+        msacc = get_msacc();
+
+        sys_perf_counter(&msacc->perf_counter);
+
+        msacc->state = ERTS_MSACC_STATE_OTHER;
+
+        ERTS_MSACC_TSD_SET(msacc);
+
+    } else if (msaccrp->action == ERTS_MSACC_DISABLE && msacc) {
+        ERTS_MSACC_TSD_SET(NULL);
+    } else if (msaccrp->action == ERTS_MSACC_RESET) {
+        msacc = msacc ? msacc : get_msacc();
+        erts_msacc_reset(msacc);
+    } else if (msaccrp->action == ERTS_MSACC_GATHER && !msacc) {
+        msacc = get_msacc();
+    }
+
+    ASSERT(!msacc || !msacc->unmanaged);
+
+    send_reply(msacc, msaccrp);
+
+    erts_proc_dec_refc(msaccrp->proc);
+
+    if (erts_smp_atomic32_dec_read_nob(&msaccrp->refc) == 0)
+      erts_free(ERTS_ALC_T_MSACC, vmsaccrp);
+}
+
+static void erts_msacc_reset(ErtsMsAcc *msacc) {
+  int i;
+  if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
+
+  for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
+      msacc->perf_counters[i] = 0;
+#ifdef ERTS_MSACC_STATE_COUNTERS
+      msacc->state_counters[i] = 0;
+#endif
+  }
+
+  if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
+}
+
+#endif /* ERTS_ENABLE_MSACC */
+
+
+/*
+ * This function is responsible for enabling, disabling, resetting and
+ * gathering data related to microstate accounting.
+ *
+ * Managed threads and unmanaged threads are handled differently.
+ *   - managed threads get a misc_aux job telling them to switch on msacc
+ *   - unmanaged have some fields protected by a mutex that has to be taken
+ *     before any values can be updated
+ *
+ * For performance reasons there is also a global value erts_msacc_enabled
+ * that controls the state of all threads. Statistics gathering is only on
+ * if erts_msacc_enabled && msacc is true.
+ */
+Eterm
+erts_msacc_request(Process *c_p, int action, Eterm *threads)
+{
+#ifdef ERTS_ENABLE_MSACC
+    ErtsMsAcc *msacc =  ERTS_MSACC_TSD_GET();
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    Eterm ref;
+    ErtsMSAccReq *msaccrp;
+    Eterm *hp;
+
+
+#ifdef ERTS_MSACC_ALWAYS_ON
+    if (action == ERTS_MSACC_ENABLE || action == ERTS_MSACC_DISABLE)
+        return THE_NON_VALUE;
+#else
+    /* take care of double enable, and double disable here */
+    if (msacc && action == ERTS_MSACC_ENABLE) {
+        return THE_NON_VALUE;
+    } else if (!msacc && action == ERTS_MSACC_DISABLE) {
+        return THE_NON_VALUE;
+    }
+#endif
+
+    ref = erts_make_ref(c_p);
+
+    msaccrp = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMSAccReq));
+    hp = &msaccrp->ref_heap[0];
+
+    msaccrp->action = action;
+    msaccrp->proc = c_p;
+    msaccrp->ref = STORE_NC(&hp, NULL, ref);
+    msaccrp->req_sched = esdp->no;
+
+#ifdef ERTS_SMP
+    *threads = erts_no_schedulers;
+    *threads += 1; /* aux thread */
+#else
+    *threads = 1;
+#endif
+
+    erts_smp_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads);
+
+    erts_proc_add_refc(c_p, *threads);
+
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+                                          erts_no_schedulers,
+                                          reply_msacc,
+                                          (void *) msaccrp);
+#ifdef ERTS_SMP
+    /* aux thread */
+    erts_schedule_misc_aux_work(0, reply_msacc, (void *) msaccrp);
+#endif
+
+#ifdef USE_THREADS
+    /* Manage unmanaged threads */
+    switch (action) {
+    case ERTS_MSACC_GATHER: {
+        Uint unmanaged_count;
+        ErtsMsAcc *msacc, **unmanaged;
+        int i = 0;
+
+        /* we copy a list of pointers here so that we do not have to have
+           the msacc_mutex when sending messages */
+        erts_rwmtx_rlock(&msacc_mutex);
+        unmanaged_count = msacc_unmanaged_count;
+        unmanaged = erts_alloc(ERTS_ALC_T_MSACC,
+                               sizeof(ErtsMsAcc*)*unmanaged_count);
+
+        for (i = 0, msacc = msacc_unmanaged;
+             i < unmanaged_count;
+             i++, msacc = msacc->next) {
+            unmanaged[i] = msacc;
+        }
+        erts_rwmtx_runlock(&msacc_mutex);
+
+        for (i = 0; i < unmanaged_count; i++) {
+            erts_mtx_lock(&unmanaged[i]->mtx);
+            if (unmanaged[i]->perf_counter) {
+                ErtsSysHrTime perf_counter;
+                /* if enabled update stats */
+                sys_perf_counter(&perf_counter);
+                unmanaged[i]->perf_counters[unmanaged[i]->state] +=
+                    perf_counter - unmanaged[i]->perf_counter;
+                unmanaged[i]->perf_counter = perf_counter;
+            }
+            erts_mtx_unlock(&unmanaged[i]->mtx);
+            send_reply(unmanaged[i],msaccrp);
+        }
+        erts_free(ERTS_ALC_T_MSACC,unmanaged);
+        /* We have just sent unmanaged_count messages, so bump no of threads */
+        *threads += unmanaged_count;
+        break;
+    }
+    case ERTS_MSACC_RESET: {
+        ErtsMsAcc *msacc;
+        erts_rwmtx_rlock(&msacc_mutex);
+        for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next)
+            erts_msacc_reset(msacc);
+        erts_rwmtx_runlock(&msacc_mutex);
+        break;
+    }
+    case ERTS_MSACC_ENABLE: {
+        erts_rwmtx_rlock(&msacc_mutex);
+        for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) {
+            erts_mtx_lock(&msacc->mtx);
+            sys_perf_counter(&msacc->perf_counter);
+            /* we assume the unmanaged thread is sleeping */
+            msacc->state = ERTS_MSACC_STATE_SLEEP;
+            erts_mtx_unlock(&msacc->mtx);
+        }
+        erts_rwmtx_runlock(&msacc_mutex);
+        break;
+    }
+    case ERTS_MSACC_DISABLE: {
+        ErtsSysHrTime perf_counter;
+        erts_rwmtx_rlock(&msacc_mutex);
+        /* make sure to update stats with latest results */
+        for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) {
+            erts_mtx_lock(&msacc->mtx);
+            sys_perf_counter(&perf_counter);
+            msacc->perf_counters[msacc->state] += perf_counter - msacc->perf_counter;
+            msacc->perf_counter = 0;
+            erts_mtx_unlock(&msacc->mtx);
+        }
+        erts_rwmtx_runlock(&msacc_mutex);
+        break;
+    }
+    default: { ASSERT(0); }
+    }
+
+#endif
+
+    *threads = make_small(*threads);
+
+    reply_msacc((void *) msaccrp);
+
+#ifndef ERTS_MSACC_ALWAYS_ON
+    /* enable/disable the global value */
+    if (action == ERTS_MSACC_ENABLE) {
+        erts_msacc_enabled = 1;
+    } else if (action == ERTS_MSACC_DISABLE) {
+        erts_msacc_enabled = 0;
+    }
+#endif
+
+    return ref;
+#else
+    return THE_NON_VALUE;
+#endif
+}
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
new file mode 100644
index 0000000000..de72fe9597
--- /dev/null
+++ b/erts/emulator/beam/erl_msacc.h
@@ -0,0 +1,409 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2014-2015. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef ERL_MSACC_H__
+#define ERL_MSACC_H__
+
+/* Can be enabled/disabled via configure */
+#if ERTS_ENABLE_MSACC == 2
+#define ERTS_MSACC_EXTENDED_STATES 1
+#endif
+
+/* Uncomment this to also count the number of
+   transitions to a state. This will add a count
+   to the counter map. */
+/* #define ERTS_MSACC_STATE_COUNTERS 1 */
+
+/* Uncomment this to make msacc to always be on,
+   this reduces overhead a little bit when profiling */
+/* #define ERTS_MSACC_ALWAYS_ON 1 */
+
+#define ERTS_MSACC_DISABLE 0
+#define ERTS_MSACC_ENABLE  1
+#define ERTS_MSACC_RESET   2
+#define ERTS_MSACC_GATHER  3
+
+/*
+ * When adding a new state, you have to:
+ * * Add it here
+ * * Increment ERTS_MSACC_STATE_COUNT
+ * * Add string value to erts_msacc_states
+ * * Have to be in alphabetical order!
+ * * Only add states to the non-extended section after
+ *   careful benchmarking to make sure the overhead
+ *   when disabled is minimal.
+ */
+
+#ifndef ERTS_MSACC_EXTENDED_STATES
+#define ERTS_MSACC_STATE_AUX       0
+#define ERTS_MSACC_STATE_CHECK_IO  1
+#define ERTS_MSACC_STATE_EMULATOR  2
+#define ERTS_MSACC_STATE_GC        3
+#define ERTS_MSACC_STATE_OTHER     4
+#define ERTS_MSACC_STATE_PORT      5
+#define ERTS_MSACC_STATE_SLEEP     6
+
+#define ERTS_MSACC_STATE_COUNT 7
+
+#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC
+static char *erts_msacc_states[] = {
+    "aux",
+    "check_io",
+    "emulator",
+    "gc",
+    "other",
+    "port",
+    "sleep"
+};
+#endif
+
+#else
+
+#define ERTS_MSACC_STATE_ALLOC     0
+#define ERTS_MSACC_STATE_AUX       1
+#define ERTS_MSACC_STATE_BIF       2
+#define ERTS_MSACC_STATE_BUSY_WAIT 3
+#define ERTS_MSACC_STATE_CHECK_IO  4
+#define ERTS_MSACC_STATE_EMULATOR  5
+#define ERTS_MSACC_STATE_ETS       6
+#define ERTS_MSACC_STATE_GC        7
+#define ERTS_MSACC_STATE_GC_FULL   8
+#define ERTS_MSACC_STATE_NIF       9
+#define ERTS_MSACC_STATE_OTHER     10
+#define ERTS_MSACC_STATE_PORT      11
+#define ERTS_MSACC_STATE_SEND      12
+#define ERTS_MSACC_STATE_SLEEP     13
+#define ERTS_MSACC_STATE_TIMERS    14
+
+#define ERTS_MSACC_STATE_COUNT 15
+
+#if ERTS_MSACC_STATE_STRINGS
+static char *erts_msacc_states[] = {
+    "alloc",
+    "aux",
+    "bif",
+    "busy_wait",
+    "check_io",
+    "emulator",
+    "ets",
+    "gc",
+    "gc_full",
+    "nif",
+    "other",
+    "port",
+    "send",
+    "sleep",
+    "timers"
+};
+#endif
+
+#endif
+
+typedef struct erl_msacc_t_ ErtsMsAcc;
+
+struct erl_msacc_t_ {
+
+    /* the the values below are protected by mtx iff unmanaged = 1 */
+    ErtsSysHrTime perf_counter;
+    ErtsSysHrTime perf_counters[ERTS_MSACC_STATE_COUNT];
+#ifdef ERTS_MSACC_STATE_COUNTERS
+    Uint64 state_counters[ERTS_MSACC_STATE_COUNT];
+#endif
+    Uint state;
+
+    /* protected by msacc_mutex in erl_msacc.c, and should be constant */
+    int unmanaged;
+    erts_mtx_t mtx;
+    ErtsMsAcc *next;
+    erts_tid_t tid;
+    Eterm id;
+    char *type;
+};
+
+#if ERTS_ENABLE_MSACC
+
+#define ERTS_MSACC_INLINE ERTS_GLB_INLINE
+
+#ifdef USE_THREADS
+extern erts_tsd_key_t erts_msacc_key;
+#else
+extern ErtsMsAcc *erts_msacc;
+#endif
+
+#ifdef ERTS_MSACC_ALWAYS_ON
+#define erts_msacc_enabled 1
+#else
+extern int erts_msacc_enabled;
+#endif
+
+#ifdef USE_THREADS
+#define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key)
+#define ERTS_MSACC_TSD_SET(tsd) erts_tsd_set(erts_msacc_key,tsd)
+#else
+#define ERTS_MSACC_TSD_GET() erts_msacc
+#define ERTS_MSACC_TSD_SET(tsd) erts_msacc = tsd
+#endif
+
+void erts_msacc_early_init(void);
+void erts_msacc_init(void);
+void erts_msacc_init_thread(char *type, int id, int liberty);
+
+/* The defines below are used to instrument the vm code
+ * with different state changes. There are two variants
+ * of each define. One that has a cached ErtsMsAcc *
+ * that it can use, and one that does not.
+ * The cached values are necessary to have in order to get
+ * low enough overhead when running without msacc enabled.
+ *
+ * The two most common patterns to use the function with are:
+ *
+ *   ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_NEWSTATE);
+ *     ... call other function in new state ...
+ *   ERTS_MSACC_POP_STATE();
+ *
+ * Note that the erts_msacc_push* function declare new variables, so
+ * to conform with C89 we have to call it in the beginning of a function.
+ * We might not want to change state it the beginning though, so we use this:
+ *
+ *   ERTS_MSACC_PUSH_STATE();
+ *     ... some other code ...
+ *   ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_NEWSTATE);
+ *     ... call other function in new state ...
+ *   ERTS_MSACC_POP_STATE();
+ *
+ * Notice that we used the cached version of set_state as push_state already
+ * read the erts_msacc_enabled to the cache.
+ *
+ * Most macros also have other variants with the suffix _m which means that
+ * they are known to only be called in managed threads, or with the _x suffix
+ * which means that it should only be used in an emulator compiled with
+ * extended states.
+ *
+ * Here is a listing of the entire api:
+ *
+ *  void ERTS_MSACC_DECLARE_CACHE()
+ *  void ERTS_MSACC_UPDATE_CACHE()
+ *  void ERTS_MSACC_IS_ENABLED()
+ *  void ERTS_MSACC_IS_ENABLED_CACHED()
+ *
+ *  void ERTS_MSACC_PUSH_STATE()
+ *  void ERTS_MSACC_SET_STATE(int state)
+ *  void ERTS_MSACC_PUSH_AND_SET_STATE(int state)
+ *
+ *  void ERTS_MSACC_PUSH_STATE_CACHED()
+ *  void ERTS_MSACC_SET_STATE_CACHED(int state)
+ *  void ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(int state)
+ *  void ERTS_MSACC_POP_STATE()
+ *
+ *  void ERTS_MSACC_PUSH_STATE_M()
+ *  void ERTS_MSACC_PUSH_STATE_CACHED_M()
+ *  void ERTS_MSACC_SET_STATE_CACHED_M(int state)
+ *  void ERTS_MSACC_SET_STATE_M(int state)
+ *  void ERTS_MSACC_POP_STATE_M()
+ *  void ERTS_MSACC_PUSH_AND_SET_STATE_M(int state)
+ *
+ *  Most functions are also available with an _x suffix that are only enabled
+ *  when using the extra states. If they are not, just add them to the end
+ *  of this file.
+ */
+
+/* cache handling functions */
+#define ERTS_MSACC_IS_ENABLED() ERTS_UNLIKELY(erts_msacc_enabled)
+#define ERTS_MSACC_DECLARE_CACHE()                                      \
+    ErtsMsAcc *ERTS_MSACC_UPDATE_CACHE();                                 \
+    ERTS_DECLARE_DUMMY(Uint __erts_msacc_state) = ERTS_MSACC_STATE_OTHER
+#define ERTS_MSACC_IS_ENABLED_CACHED() ERTS_UNLIKELY(__erts_msacc_cache != NULL)
+#define ERTS_MSACC_UPDATE_CACHE()                                       \
+    __erts_msacc_cache = erts_msacc_enabled ? ERTS_MSACC_TSD_GET() : NULL
+
+
+/* The defines below implicitly declare and load a new cache */
+#define ERTS_MSACC_PUSH_STATE()                           \
+    ERTS_MSACC_DECLARE_CACHE();                           \
+    ERTS_MSACC_PUSH_STATE_CACHED()
+#define ERTS_MSACC_SET_STATE(state)                                     \
+    ERTS_MSACC_DECLARE_CACHE();                                         \
+    ERTS_MSACC_SET_STATE_CACHED(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE(state)                    \
+    ERTS_MSACC_PUSH_STATE(); ERTS_MSACC_SET_STATE_CACHED(state)
+
+/* The defines below need an already declared cache to work */
+#define ERTS_MSACC_PUSH_STATE_CACHED()                                  \
+    __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ?               \
+        erts_msacc_get_state_um__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER
+#define ERTS_MSACC_SET_STATE_CACHED(state) \
+    if (ERTS_MSACC_IS_ENABLED_CACHED())                         \
+        erts_msacc_set_state_um__(__erts_msacc_cache, state, 1)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state) \
+    ERTS_MSACC_PUSH_STATE_CACHED(); ERTS_MSACC_SET_STATE_CACHED(state)
+#define ERTS_MSACC_POP_STATE()                                          \
+    if (ERTS_MSACC_IS_ENABLED_CACHED())                                 \
+        erts_msacc_set_state_um__(__erts_msacc_cache, __erts_msacc_state, 0)
+
+/* Only use these defines when we know that we have in a managed thread */
+#define ERTS_MSACC_PUSH_STATE_M()                         \
+    ERTS_MSACC_DECLARE_CACHE();                           \
+    ERTS_MSACC_PUSH_STATE_CACHED_M()
+#define ERTS_MSACC_PUSH_STATE_CACHED_M()                                \
+    __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ?    \
+        erts_msacc_get_state_m__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER
+#define ERTS_MSACC_SET_STATE_M(state)                   \
+    ERTS_MSACC_DECLARE_CACHE();                         \
+    ERTS_MSACC_SET_STATE_CACHED_M(state)
+#define ERTS_MSACC_SET_STATE_CACHED_M(state)            \
+    if (ERTS_MSACC_IS_ENABLED_CACHED())      \
+        erts_msacc_set_state_m__(__erts_msacc_cache, state, 1)
+#define ERTS_MSACC_POP_STATE_M()                                  \
+    if (ERTS_MSACC_IS_ENABLED_CACHED())                      \
+        erts_msacc_set_state_m__(__erts_msacc_cache, __erts_msacc_state, 0)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_M(state)                    \
+    ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state)
+
+ERTS_MSACC_INLINE
+void erts_msacc_set_state_um__(ErtsMsAcc *msacc,Uint state,int increment);
+ERTS_MSACC_INLINE
+void erts_msacc_set_state_m__(ErtsMsAcc *msacc,Uint state,int increment);
+
+ERTS_MSACC_INLINE
+Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc);
+ERTS_MSACC_INLINE
+Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc);
+
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_MSACC_INLINE
+Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) {
+    Uint state;
+    if (msacc->unmanaged)
+        erts_mtx_lock(&msacc->mtx);
+    state = msacc->state;
+    if (msacc->unmanaged)
+        erts_mtx_unlock(&msacc->mtx);
+    return state;
+}
+
+ERTS_MSACC_INLINE
+Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc) {
+    return msacc->state;
+}
+
+ERTS_MSACC_INLINE
+void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) {
+    if (ERTS_UNLIKELY(msacc->unmanaged)) {
+        erts_mtx_lock(&msacc->mtx);
+        msacc->state = new_state;
+        if (ERTS_LIKELY(!msacc->perf_counter)) {
+            erts_mtx_unlock(&msacc->mtx);
+            return;
+        }
+    }
+
+    erts_msacc_set_state_m__(msacc,new_state,increment);
+
+    if (ERTS_UNLIKELY(msacc->unmanaged))
+        erts_mtx_unlock(&msacc->mtx);
+}
+
+ERTS_MSACC_INLINE
+void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) {
+    ErtsSysHrTime prev_perf_counter;
+    Sint64 diff;
+
+    if (new_state == msacc->state)
+        return;
+
+    prev_perf_counter = msacc->perf_counter;
+    sys_perf_counter(&msacc->perf_counter);
+    diff = msacc->perf_counter - prev_perf_counter;
+    ASSERT(diff >= 0);
+    msacc->perf_counters[msacc->state] += diff;
+#ifdef ERTS_MSACC_STATE_COUNTERS
+    msacc->state_counters[new_state] += increment;
+#endif
+    msacc->state = new_state;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#else
+
+#define ERTS_MSACC_IS_ENABLED() 0
+#define erts_msacc_early_init()
+#define erts_msacc_init()
+#define erts_msacc_init_thread(type, id, liberty)
+#define ERTS_MSACC_PUSH_STATE()
+#define ERTS_MSACC_PUSH_STATE_CACHED()
+#define ERTS_MSACC_POP_STATE()
+#define ERTS_MSACC_SET_STATE(state)
+#define ERTS_MSACC_SET_STATE_CACHED(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state)
+#define ERTS_MSACC_UPDATE_CACHE()
+#define ERTS_MSACC_IS_ENABLED_CACHED()
+#define ERTS_MSACC_DECLARE_CACHE()
+#define ERTS_MSACC_PUSH_STATE_M()
+#define ERTS_MSACC_PUSH_STATE_CACHED_M()
+#define ERTS_MSACC_SET_STATE_CACHED_M(state)
+#define ERTS_MSACC_POP_STATE_M()
+#define ERTS_MSACC_PUSH_AND_SET_STATE_M(state)
+
+
+#endif /* ERTS_ENABLE_MSACC */
+
+#ifndef ERTS_MSACC_EXTENDED_STATES
+
+#define ERTS_MSACC_PUSH_STATE_X()
+#define ERTS_MSACC_POP_STATE_X()
+#define ERTS_MSACC_SET_STATE_X(state)
+#define ERTS_MSACC_SET_STATE_M_X(state)
+#define ERTS_MSACC_SET_STATE_CACHED_X(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_X(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(state)
+#define ERTS_MSACC_UPDATE_CACHE_X()
+#define ERTS_MSACC_IS_ENABLED_CACHED_X() 0
+#define ERTS_MSACC_DECLARE_CACHE_X()
+#define ERTS_MSACC_PUSH_STATE_M_X()
+#define ERTS_MSACC_PUSH_STATE_CACHED_M_X()
+#define ERTS_MSACC_SET_STATE_CACHED_M_X(state)
+#define ERTS_MSACC_POP_STATE_M_X()
+#define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state)
+
+#else
+
+#define ERTS_MSACC_PUSH_STATE_X() ERTS_MSACC_PUSH_STATE()
+#define ERTS_MSACC_POP_STATE_X() ERTS_MSACC_POP_STATE()
+#define ERTS_MSACC_SET_STATE_X(state) ERTS_MSACC_SET_STATE(state)
+#define ERTS_MSACC_SET_STATE_M_X(state) ERTS_MSACC_SET_STATE_M(state)
+#define ERTS_MSACC_SET_STATE_CACHED_X(state) ERTS_MSACC_SET_STATE_CACHED(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_X(state) ERTS_MSACC_PUSH_AND_SET_STATE(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state)
+#define ERTS_MSACC_UPDATE_CACHE_X() ERTS_MSACC_UPDATE_CACHE()
+#define ERTS_MSACC_IS_ENABLED_CACHED_X() ERTS_MSACC_IS_ENABLED_CACHED()
+#define ERTS_MSACC_DECLARE_CACHE_X() ERTS_MSACC_DECLARE_CACHE()
+#define ERTS_MSACC_PUSH_STATE_M_X() ERTS_MSACC_PUSH_STATE_M()
+#define ERTS_MSACC_PUSH_STATE_CACHED_M_X() ERTS_MSACC_PUSH_STATE_CACHED_M()
+#define ERTS_MSACC_SET_STATE_CACHED_M_X(state) ERTS_MSACC_SET_STATE_CACHED_M(state)
+#define ERTS_MSACC_POP_STATE_M_X() ERTS_MSACC_POP_STATE_M()
+#define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_M(state)
+
+#endif /* !ERTS_MSACC_EXTENDED_STATES */
+
+#endif /* ERL_MSACC_H__ */
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 2c09834d19..197b328fe2 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1648,6 +1648,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     int active;
     Uint64 start_time = 0;
     ErtsSchedulerData *esdp = runq->scheduler;
+    ERTS_MSACC_PUSH_STATE_M();
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
 
@@ -1690,6 +1691,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
     state = erts_atomic32_read_nob(&pp->state);
     pp->reds = ERTS_PORT_REDS_EXECUTE;
+    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
     goto begin_handle_tasks;
 
     while (1) {
@@ -1822,6 +1824,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     }
 
     erts_unblock_fpe(fpe_was_unmasked);
+    ERTS_MSACC_POP_STATE_M();
 
 
     if (io_tasks_executed) {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 3902632ef8..8e1ebe795c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -941,6 +941,12 @@ sched_wall_time_change(ErtsSchedulerData *esdp, int working)
 	    }
 	}
     }
+    if (!working) {
+        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT);
+    } else {
+        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER);
+    }
+
 }
 
 typedef struct {
@@ -1696,15 +1702,17 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
     int need_thr_progress = 0;
     ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     int more_work = 0;
-
+    ERTS_MSACC_PUSH_STATE_M_X();
 #ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
 #endif
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
+    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC);
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
 						&wakeup,
 						&more_work);
+    ERTS_MSACC_POP_STATE_M_X();
     if (more_work) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
 	    & ERTS_SSI_AUX_WORK_DD_THR_PRGR) {
@@ -2176,12 +2184,15 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
 	if (!(aux_work & ~ignore)) { \
 	    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+            ERTS_MSACC_UPDATE_CACHE();       \
+            ERTS_MSACC_POP_STATE_M();              \
 	    return aux_work; \
 	} \
     }
 
     erts_aint32_t aux_work = orig_aux_work;
     erts_aint32_t ignore = 0;
+    ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_AUX);
 
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
 #ifdef ERTS_SMP
@@ -2272,6 +2283,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 	haw_thr_prgr_current_check_progress(awdp);
 #endif
 
+    ERTS_MSACC_UPDATE_CACHE();
+    ERTS_MSACC_POP_STATE_M();
     return aux_work;
 
 #undef HANDLE_AUX_WORK
@@ -2779,6 +2792,8 @@ aux_thread(void *unused)
 
     ssi->event = erts_tse_fetch();
 
+    erts_msacc_init_thread("aux", 1, 1);
+
     callbacks.arg = (void *) ssi;
     callbacks.wakeup = thr_prgr_wakeup;
     callbacks.prepare_wait = thr_prgr_prep_wait;
@@ -2789,6 +2804,7 @@ aux_thread(void *unused)
     init_aux_work_data(awdp, NULL, NULL);
     awdp->ssi = ssi;
 
+
     sched_prep_spin_wait(ssi);
 
     while (1) {
@@ -2842,6 +2858,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 #ifdef ERTS_SMP
     int thr_prgr_active = 1;
     erts_aint32_t flgs;
+#endif
+    ERTS_MSACC_PUSH_STATE_M();
+#ifdef ERTS_SMP
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
@@ -2898,6 +2917,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    sched_wall_time_change(esdp, 1);
 		}
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
+                ERTS_MSACC_UPDATE_CACHE();
 		if (aux_work && erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
 	    }
@@ -2959,7 +2979,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 								     - 1) + 1;
 				} else
 				    timeout = -1;
+                                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
 				res = erts_tse_twait(ssi->event, timeout);
+                                ERTS_MSACC_POP_STATE_M();
 				current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
 				    erts_get_monotonic_time(esdp);
 			    } while (res == EINTR);
@@ -3028,9 +3050,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    if (working)
 		sched_wall_time_change(esdp, working = 0);
 
+	    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
 	    erl_sys_schedule(1); /* Might give us something to do */
 
+	    ERTS_MSACC_POP_STATE_M();
+
 	    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 		current_time = erts_get_monotonic_time(esdp);
 		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
@@ -3051,6 +3077,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 #endif
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
+                ERTS_MSACC_UPDATE_CACHE();
 #ifdef ERTS_SMP
 		if (aux_work && erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
@@ -3148,8 +3175,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	ASSERT(!erts_port_task_have_outstanding_io_tasks());
 
+	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+
 	erl_sys_schedule(0);
 
+	ERTS_MSACC_POP_STATE_M();
+
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
@@ -7926,6 +7957,8 @@ sched_thread_func(void *vesdp)
     callbacks.wait = thr_prgr_wait;
     callbacks.finalize_wait = thr_prgr_fin_wait;
 
+    erts_msacc_init_thread("scheduler", no, 1);
+
     erts_thr_progress_register_managed_thread(esdp, &callbacks, 0);
     erts_alloc_register_scheduler(vesdp);
 #endif
@@ -9204,6 +9237,8 @@ Process *schedule(Process *p, int calls)
     Uint32 flags;
     erts_aint32_t state = 0; /* Supress warning... */
 
+    ERTS_MSACC_DECLARE_CACHE();
+
 #ifdef USE_VM_PROBES
     if (p != NULL && DTRACE_ENABLED(process_unscheduled)) {
         DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
@@ -9308,6 +9343,8 @@ Process *schedule(Process *p, int calls)
 
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 
+        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
+
 	if (state & ERTS_PSFLG_FREE) {
 #ifdef ERTS_SMP
 	    ASSERT(esdp->free_process == p);
@@ -9468,7 +9505,7 @@ Process *schedule(Process *p, int calls)
 	    scheduler_wait(&fcalls, esdp, rq);
 	    flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
 	    flags |= ERTS_RUNQ_FLG_EXEC;
-
+            ERTS_MSACC_UPDATE_CACHE();
 #ifdef ERTS_SMP
 	    non_empty_runq(rq);
 #endif
@@ -9483,6 +9520,8 @@ Process *schedule(Process *p, int calls)
 	     * Schedule system-level activities.
 	     */
 
+	    ERTS_MSACC_PUSH_STATE_CACHED_M();
+
 	    erts_smp_atomic32_set_relb(&function_calls, 0);
 	    fcalls = 0;
 
@@ -9490,7 +9529,9 @@ Process *schedule(Process *p, int calls)
 	    erts_sys_schedule_interrupt(0);
 #endif
 	    erts_smp_runq_unlock(rq);
+	    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
 	    erl_sys_schedule(1);
+	    ERTS_MSACC_POP_STATE_M();
 
 	    current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
@@ -9568,9 +9609,12 @@ Process *schedule(Process *p, int calls)
 	    case 0:			/* No process at all */
 	    default:
 		ASSERT(qmask == 0);
+                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
 		goto check_activities_to_run;
 	    }
 
+            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR);
+
 	    BM_START_TIMER(system);
 
 	    /*
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 34f91e2ec8..e689547d1d 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -2056,6 +2056,8 @@ erts_atomic64_read_dirty(erts_atomic64_t *var)
 
 #endif /* !USE_THREADS */
 
+#include "erl_msacc.h"
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
@@ -2414,6 +2416,7 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
 {
 #ifdef USE_THREADS
     int res;
+    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock(&mtx->lc);
 #endif
@@ -2432,6 +2435,7 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
 #endif
     if (res != 0 && res != EINTR)
 	erts_thr_fatal_error(res, "wait on condition variable");
+    ERTS_MSACC_POP_STATE();
 #endif
 }
 
@@ -3488,7 +3492,11 @@ ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
 ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    return ethr_event_wait(&((ethr_ts_event *) ep)->event);
+    int res;
+    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
+    res = ethr_event_wait(&((ethr_ts_event *) ep)->event);
+    ERTS_MSACC_POP_STATE();
+    return res;
 #else
     return ENOTSUP;
 #endif
@@ -3497,7 +3505,11 @@ ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
 ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
 {
 #ifdef USE_THREADS
-    return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
+    int res;
+    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
+    res = ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
+    ERTS_MSACC_POP_STATE();
+    return res;
 #else
     return ENOTSUP;
 #endif
@@ -3506,8 +3518,12 @@ ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
 ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
 {
 #ifdef USE_THREADS
-    return ethr_event_twait(&((ethr_ts_event *) ep)->event,
-			    (ethr_sint64_t) tmo);
+    int res;
+    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
+    res = ethr_event_twait(&((ethr_ts_event *) ep)->event,
+                           (ethr_sint64_t) tmo);
+    ERTS_MSACC_POP_STATE();
+    return res;
 #else
     return ENOTSUP;
 #endif
@@ -3516,9 +3532,13 @@ ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
 ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
 {
 #ifdef USE_THREADS
-    return ethr_event_stwait(&((ethr_ts_event *) ep)->event,
-			     spincount,
-			     (ethr_sint64_t) tmo);
+    int res;
+    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
+    res = ethr_event_stwait(&((ethr_ts_event *) ep)->event,
+                            spincount,
+                            (ethr_sint64_t) tmo);
+    ERTS_MSACC_POP_STATE();
+    return res;
 #else
     return ENOTSUP;
 #endif
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 3f5925765d..e8a7573e86 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1448,6 +1448,8 @@ ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr)
 
 #endif
 
+Eterm erts_msacc_request(Process *c_p, int action, Eterm *threads);
+
 /*
 ** Call_trace uses this API for the parameter matching functions
 */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 07578fd24e..41b0fcdd1b 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -706,6 +706,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 
     error_number = error_type = 0;
     if (driver->start) {
+        ERTS_MSACC_PUSH_STATE_M();
 	if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(port, am_in, am_start);
 	}
@@ -716,6 +717,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
             DTRACE3(driver_start, process_str, driver->name, port_str);
         }
 #endif
+	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
 	fpe_was_unmasked = erts_block_fpe();
 	drv_data = (*driver->start)(ERTS_Port2ErlDrvPort(port), name, opts);
 	if (((SWord) drv_data) == -1)
@@ -735,6 +737,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	}
 
 	erts_unblock_fpe(fpe_was_unmasked);
+	ERTS_MSACC_POP_STATE_M();
 	port->caller = NIL;
 	if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(port, am_out, am_start);
@@ -1710,6 +1713,7 @@ call_driver_outputv(int bang_op,
     else {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
 	ErlDrvSizeT size = evp->size;
+	ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT);
 
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)	
 			   || ERTS_IS_CRASH_DUMPING);
@@ -1730,6 +1734,8 @@ call_driver_outputv(int bang_op,
 	    esdp->io.out += (Uint64) size;
 	else
 	    erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size);
+
+	ERTS_MSACC_POP_STATE_M();
     }
 }
 
@@ -1810,6 +1816,7 @@ call_driver_output(int bang_op,
 	send_badsig(prt);
     else {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+        ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT);
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)	
 			   || ERTS_IS_CRASH_DUMPING);
 
@@ -1829,6 +1836,8 @@ call_driver_output(int bang_op,
 	    esdp->io.out += (Uint64) size;
 	else
 	    erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size);
+
+	ERTS_MSACC_POP_STATE_M();
     }
 }
 
@@ -3473,6 +3482,7 @@ static void flush_port(Port *p)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
 
     if (p->drv_ptr->flush != NULL) {
+        ERTS_MSACC_PUSH_STATE_M();
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(driver_flush)) {
             DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p)
@@ -3482,9 +3492,11 @@ static void flush_port(Port *p)
         if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(p, am_in, am_flush);
 	}
+        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
 	fpe_was_unmasked = erts_block_fpe();
 	(*p->drv_ptr->flush)((ErlDrvData)p->drv_data);
 	erts_unblock_fpe(fpe_was_unmasked);
+	ERTS_MSACC_POP_STATE_M();
         if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(p, am_out, am_flush);
 	}
@@ -3532,6 +3544,7 @@ terminate_port(Port *prt)
     drv = prt->drv_ptr;
     if ((drv != NULL) && (drv->stop != NULL)) {
 	int fpe_was_unmasked = erts_block_fpe();
+	ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT);
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(driver_stop)) {
             DTRACE_FORMAT_COMMON_PID_AND_PORT(connected_id, prt)
@@ -3540,6 +3553,7 @@ terminate_port(Port *prt)
 #endif
 	(*drv->stop)((ErlDrvData)prt->drv_data);
 	erts_unblock_fpe(fpe_was_unmasked);
+	ERTS_MSACC_POP_STATE_M();
 #ifdef ERTS_SMP
 	if (prt->xports)
 	    erts_port_handle_xports(prt);
@@ -3850,6 +3864,7 @@ call_driver_control(Eterm caller,
 		    ErlDrvSizeT *from_size)
 {
     ErlDrvSSizeT cres;
+    ERTS_MSACC_PUSH_STATE_M();
 
     if (!prt->drv_ptr->control)
 	return ERTS_PORT_OP_BADARG;
@@ -3863,6 +3878,8 @@ call_driver_control(Eterm caller,
                 command, size);
     }
 #endif
+    
+    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
 
     prt->caller = caller;
     cres = prt->drv_ptr->control((ErlDrvData) prt->drv_data,
@@ -3873,6 +3890,8 @@ call_driver_control(Eterm caller,
 				 *from_size);
     prt->caller = NIL;
 
+    ERTS_MSACC_POP_STATE_M();
+
     if (cres < 0)
 	return ERTS_PORT_OP_BADARG;
 
@@ -4260,6 +4279,7 @@ call_driver_call(Eterm caller,
 		 unsigned *ret_flagsp)
 {
     ErlDrvSSizeT cres;
+    ERTS_MSACC_PUSH_STATE_M();
 
     if (!prt->drv_ptr->call)
 	return ERTS_PORT_OP_BADARG;
@@ -4275,6 +4295,8 @@ call_driver_call(Eterm caller,
     }
 #endif
 
+    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
+
     prt->caller = caller;
     cres = prt->drv_ptr->call((ErlDrvData) prt->drv_data,
 			      command,
@@ -4285,6 +4307,8 @@ call_driver_call(Eterm caller,
 			      ret_flagsp);
     prt->caller = NIL;
 
+    ERTS_MSACC_POP_STATE_M();
+
     if (cres <= 0
 	|| ((byte) (*resp_bufp)[0]) != VERSION_MAGIC)
 	return ERTS_PORT_OP_BADARG;
@@ -4988,6 +5012,7 @@ int get_port_flags(ErlDrvPort ix)
 void erts_raw_port_command(Port* p, byte* buf, Uint len)
 {
     int fpe_was_unmasked;
+    ERTS_MSACC_PUSH_STATE_M();
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
@@ -5008,9 +5033,11 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len)
         DTRACE4(driver_output, "-raw-", port_str, p->name, len);
     }
 #endif
+    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
     fpe_was_unmasked = erts_block_fpe();
     (*p->drv_ptr->output)((ErlDrvData)p->drv_data, (char*) buf, (int) len);
     erts_unblock_fpe(fpe_was_unmasked);
+    ERTS_MSACC_POP_STATE_M();
 }
 
 int async_ready(Port *p, void* data)
@@ -5022,6 +5049,7 @@ int async_ready(Port *p, void* data)
     if (p) {
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
 	if (p->drv_ptr->ready_async != NULL) {
+	    ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT);
 #ifdef USE_VM_PROBES
             if (DTRACE_ENABLED(driver_ready_async)) {
                 DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p)
@@ -5030,6 +5058,7 @@ int async_ready(Port *p, void* data)
 #endif
 	    (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data);
 	    need_free = 0;
+	    ERTS_MSACC_POP_STATE_M();
 
 	}
 	erts_port_driver_callback_epilogue(p, NULL);
@@ -7053,6 +7082,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
     void (*callback)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
     ErlDrvMonitor drv_monitor;
     int fpe_was_unmasked;
+    ERTS_MSACC_PUSH_STATE_M();
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
     ASSERT(prt->drv_ptr != NULL);    
@@ -7064,6 +7094,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
     callback = prt->drv_ptr->process_exit;
     ASSERT(callback != NULL);
     ref_to_driver_monitor(ref,&drv_monitor);
+    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
     DRV_MONITOR_UNLOCK_PDL(prt);
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(driver_process_exit)) {
@@ -7075,6 +7106,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
     (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor);
     erts_unblock_fpe(fpe_was_unmasked);
     DRV_MONITOR_LOCK_PDL(prt);
+    ERTS_MSACC_POP_STATE_M();
     /* remove monitor *after* callback */
     rmon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref);
     DRV_MONITOR_UNLOCK_PDL(prt);
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 0ab6661c9f..48cb39333a 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -278,9 +278,14 @@ ErtsMonotonicTime
 erts_check_next_timeout_time(ErtsSchedulerData *esdp)
 {
     ErtsTimerWheel *tiw = esdp->timer_wheel;
+    ErtsMonotonicTime time;
+    ERTS_MSACC_DECLARE_CACHE_X();
     if (tiw->true_next_timeout_time)
 	return tiw->next_timeout_time;
-    return find_next_timeout(esdp, tiw, 1, 0, 0);
+    ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS);
+    time = find_next_timeout(esdp, tiw, 1, 0, 0);
+    ERTS_MSACC_POP_STATE_M_X();
+    return time;
 }
 
 #ifndef ERTS_TW_DEBUG
@@ -336,6 +341,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
     int tiw_pos_ix, slots, yielded_slot_restarted, yield_count;
     ErtsMonotonicTime bump_to, tmp_slots, old_pos;
+    ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT;
 
@@ -371,6 +377,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 		tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY;
 		tiw->pos = bump_to;
 		tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+                ERTS_MSACC_POP_STATE_M_X();
 		return;
 	    }
 
@@ -382,6 +389,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 		    tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE;
 		    tiw->true_next_timeout_time = 1;
 		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos);
+                    ERTS_MSACC_POP_STATE_M_X();
 		    return;
 		}
 
@@ -400,8 +408,10 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 		p = tiw->at_once.head;
 	    }
 
-	    if (tiw->pos >= bump_to)
+	    if (tiw->pos >= bump_to) {
+                ERTS_MSACC_POP_STATE_M_X();
 		break;
+            }
 
 	    if (tiw->nto == 0)
 		goto empty_wheel;
@@ -478,6 +488,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 			    tiw->yield_slot = tiw_pos_ix;
 			    tiw->yield_slots_left = slots;
 			    tiw->yield_start_pos = old_pos;
+                            ERTS_MSACC_POP_STATE_M_X();
 			    return; /* Yield! */
 			}
 
@@ -500,6 +511,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 
     /* Search at most two seconds ahead... */
     (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2));
+    ERTS_MSACC_POP_STATE_M_X();
 }
 
 Uint
@@ -569,6 +581,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 		      ErtsMonotonicTime timeout_pos)
 {
     ErtsMonotonicTime timeout_time;
+    ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     p->u.func.timeout = timeout;
     p->u.func.cancel = cancel;
@@ -612,6 +625,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 	tiw->true_next_timeout_time = 1;
 	tiw->next_timeout_time = timeout_time;
     }
+    ERTS_MSACC_POP_STATE_M_X();
 }
 
 void
@@ -620,11 +634,13 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
     if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) {
 	ErlCancelProc cancel;
 	void *arg;
+        ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 	remove_timer(tiw, p);
 	cancel = p->u.func.cancel;
 	arg = p->u.func.arg;
 	if (cancel)
 	    (*cancel)(arg);
+        ERTS_MSACC_POP_STATE_M_X();
     }
 }
 
-- 
cgit v1.2.3


From c62f0216c37340e8716938b9408a40cd5b4e1d62 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 15 Jul 2015 11:34:13 +0200
Subject: erts: Fix msacc win32 debug compile error

---
 erts/emulator/beam/beam_emu.c  | 2 +-
 erts/emulator/beam/erl_msacc.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 14b81a0a6e..4be311ae82 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1252,7 +1252,7 @@ void process_main(void)
     Uint64 start_time = 0;          /* Monitor long schedule */
     BeamInstr* start_time_i = NULL;
 
-    ERTS_MSACC_DECLARE_CACHE_X(); /* a cached value of the tsd pointer for msacc */
+    ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
 
     ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
 
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index de72fe9597..1b4b7a408a 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -228,7 +228,7 @@ void erts_msacc_init_thread(char *type, int id, int liberty);
 #define ERTS_MSACC_IS_ENABLED() ERTS_UNLIKELY(erts_msacc_enabled)
 #define ERTS_MSACC_DECLARE_CACHE()                                      \
     ErtsMsAcc *ERTS_MSACC_UPDATE_CACHE();                                 \
-    ERTS_DECLARE_DUMMY(Uint __erts_msacc_state) = ERTS_MSACC_STATE_OTHER
+    ERTS_DECLARE_DUMMY(Uint __erts_msacc_state) = ERTS_MSACC_STATE_OTHER;
 #define ERTS_MSACC_IS_ENABLED_CACHED() ERTS_UNLIKELY(__erts_msacc_cache != NULL)
 #define ERTS_MSACC_UPDATE_CACHE()                                       \
     __erts_msacc_cache = erts_msacc_enabled ? ERTS_MSACC_TSD_GET() : NULL
-- 
cgit v1.2.3


From 0399f5fc547ef035c4eb5e383f30b28ae73d936e Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Thu, 16 Jul 2015 11:27:00 +0200
Subject: erts: Refactor perf counter internal interface

perf counter is now part of the function pointer interface
and also the function returns the value instead of writing
to a memory buffer.
---
 erts/emulator/beam/beam_emu.c     |  4 +--
 erts/emulator/beam/erl_msacc.c    | 14 ++++-----
 erts/emulator/beam/erl_msacc.h    |  8 ++---
 erts/emulator/beam/erl_time.h     |  2 +-
 erts/emulator/beam/erl_time_sup.c | 66 ++-------------------------------------
 5 files changed, 16 insertions(+), 78 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4be311ae82..9f143c22bf 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4938,10 +4938,10 @@ do {						\
     it has to be very very fast */
  OpCase(i_perf_counter): {
     BeamInstr* next;
-    ErtsSysHrTime ts;
+    ErtsSysPerfCounter ts;
     PreFetch(0, next);
 
-    sys_perf_counter(&ts);
+    ts = erts_sys_perf_counter();
 
     if (IS_SSMALL(ts)) {
         r(0) = make_small((Sint)ts);
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index bf1c06dea7..71e3fd8b6e 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -117,7 +117,7 @@ void erts_msacc_init_thread(char *type, int id, int managed) {
 
 #ifdef ERTS_MSACC_ALWAYS_ON
     ERTS_MSACC_TSD_SET(msacc);
-    sys_perf_counter(&msacc->perf_counter);
+    msacc->perf_counter = erts_sys_perf_counter();
     msacc->state = ERTS_MSACC_STATE_OTHER;
 #endif
 }
@@ -278,7 +278,7 @@ reply_msacc(void *vmsaccrp)
     if (msaccrp->action == ERTS_MSACC_ENABLE && !msacc) {
         msacc = get_msacc();
 
-        sys_perf_counter(&msacc->perf_counter);
+        msacc->perf_counter = erts_sys_perf_counter();
 
         msacc->state = ERTS_MSACC_STATE_OTHER;
 
@@ -412,9 +412,9 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
         for (i = 0; i < unmanaged_count; i++) {
             erts_mtx_lock(&unmanaged[i]->mtx);
             if (unmanaged[i]->perf_counter) {
-                ErtsSysHrTime perf_counter;
+                ErtsSysPerfCounter perf_counter;
                 /* if enabled update stats */
-                sys_perf_counter(&perf_counter);
+                perf_counter = erts_sys_perf_counter();
                 unmanaged[i]->perf_counters[unmanaged[i]->state] +=
                     perf_counter - unmanaged[i]->perf_counter;
                 unmanaged[i]->perf_counter = perf_counter;
@@ -439,7 +439,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
         erts_rwmtx_rlock(&msacc_mutex);
         for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) {
             erts_mtx_lock(&msacc->mtx);
-            sys_perf_counter(&msacc->perf_counter);
+            msacc->perf_counter = erts_sys_perf_counter();
             /* we assume the unmanaged thread is sleeping */
             msacc->state = ERTS_MSACC_STATE_SLEEP;
             erts_mtx_unlock(&msacc->mtx);
@@ -448,12 +448,12 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
         break;
     }
     case ERTS_MSACC_DISABLE: {
-        ErtsSysHrTime perf_counter;
+        ErtsSysPerfCounter perf_counter;
         erts_rwmtx_rlock(&msacc_mutex);
         /* make sure to update stats with latest results */
         for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) {
             erts_mtx_lock(&msacc->mtx);
-            sys_perf_counter(&perf_counter);
+            perf_counter = erts_sys_perf_counter();
             msacc->perf_counters[msacc->state] += perf_counter - msacc->perf_counter;
             msacc->perf_counter = 0;
             erts_mtx_unlock(&msacc->mtx);
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index 1b4b7a408a..284388f7aa 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -121,8 +121,8 @@ typedef struct erl_msacc_t_ ErtsMsAcc;
 struct erl_msacc_t_ {
 
     /* the the values below are protected by mtx iff unmanaged = 1 */
-    ErtsSysHrTime perf_counter;
-    ErtsSysHrTime perf_counters[ERTS_MSACC_STATE_COUNT];
+    ErtsSysPerfCounter perf_counter;
+    ErtsSysPerfCounter perf_counters[ERTS_MSACC_STATE_COUNT];
 #ifdef ERTS_MSACC_STATE_COUNTERS
     Uint64 state_counters[ERTS_MSACC_STATE_COUNT];
 #endif
@@ -324,14 +324,14 @@ void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment)
 
 ERTS_MSACC_INLINE
 void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) {
-    ErtsSysHrTime prev_perf_counter;
+    ErtsSysPerfCounter prev_perf_counter;
     Sint64 diff;
 
     if (new_state == msacc->state)
         return;
 
     prev_perf_counter = msacc->perf_counter;
-    sys_perf_counter(&msacc->perf_counter);
+    msacc->perf_counter = erts_sys_perf_counter();
     diff = msacc->perf_counter - prev_perf_counter;
     ASSERT(diff >= 0);
     msacc->perf_counters[msacc->state] += diff;
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 446adcf4af..5242063550 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -142,7 +142,7 @@ erts_time_unit_conversion(Uint64 value,
 			  Uint32 from_time_unit,
 			  Uint32 to_time_unit);
 
-ErtsSysHrTime erts_perf_counter_unit(void);
+ErtsSysPerfCounter erts_perf_counter_unit(void);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 6509c6a805..98159fdf72 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -2444,72 +2444,10 @@ BIF_RETTYPE os_system_time_1(BIF_ALIST_1)
 BIF_RETTYPE
 os_perf_counter_0(BIF_ALIST_0)
 {
-    ErtsSysHrTime pcounter;
-    sys_perf_counter(&pcounter);
-    BIF_RET(make_time_val(BIF_P, pcounter));
+    BIF_RET(make_time_val(BIF_P, erts_sys_perf_counter()));
 }
 
 BIF_RETTYPE erts_internal_perf_counter_unit_0(BIF_ALIST_0)
 {
-    BIF_RET(make_time_val(BIF_P, SYS_PERF_COUNTER_UNIT));
-}
-
-/* What resolution to spin to in micro seconds */
-#define RESOLUTION 100
-/* How many iterations to spin */
-#define ITERATIONS 1
-/* How many significant figures to round to */
-#define SIGFIGS 3
-
-static ErtsSysHrTime perf_counter_unit = 0;
-
-static ErtsSysHrTime erts_calculate_perf_counter_unit(void);
-static ErtsSysHrTime erts_calculate_perf_counter_unit() {
-    int i;
-    ErtsSysHrTime pre, post;
-    double value = 0;
-    double round_factor;
-#if defined(HAVE_GETHRTIME) && defined(GETHRTIME_WITH_CLOCK_GETTIME)
-    struct timespec basetime,comparetime;
-#define __GETTIME(arg) clock_gettime(CLOCK_MONOTONIC,arg)
-#define __GETUSEC(arg) (arg.tv_nsec / 1000)
-#else
-    SysTimeval basetime,comparetime;
-#define __GETTIME(arg) sys_gettimeofday(arg)
-#define __GETUSEC(arg) arg.tv_usec
-#endif
-
-    for (i = 0; i < ITERATIONS; i++) {
-        /* Make sure usec just flipped over at current resolution */
-        __GETTIME(&basetime);
-        do {
-            __GETTIME(&comparetime);
-        } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
-
-        sys_perf_counter(&pre);
-
-        __GETTIME(&basetime);
-        do {
-            __GETTIME(&comparetime);
-        } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
-
-        sys_perf_counter(&post);
-
-        value += post - pre;
-    }
-    /* After this value is ticks per us */
-    value /= (RESOLUTION*ITERATIONS);
-
-    /* We round to 3 significant figures */
-    round_factor = pow(10.0, SIGFIGS - ceil(log10(value)));
-    value = ((ErtsSysHrTime)(value * round_factor + 0.5)) / round_factor;
-
-    /* convert to ticks per second */
-    return 1000000 * value;
-}
-
-ErtsSysHrTime erts_perf_counter_unit() {
-    if (perf_counter_unit == 0)
-        perf_counter_unit = erts_calculate_perf_counter_unit();
-    return perf_counter_unit;
+    BIF_RET(make_time_val(BIF_P, erts_sys_perf_counter_unit()));
 }
-- 
cgit v1.2.3


From 0bf37ca78b6200f8b82b20234f0b882154a094ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 18 Dec 2015 15:26:35 +0100
Subject: Silence compiler

---
 erts/emulator/beam/beam_bp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 2a23da4f25..5d471d168b 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -864,7 +864,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
     Eterm* cpp;
     int return_to_trace = 0;
     BeamInstr w;
-    BeamInstr *cp_save;
+    BeamInstr *cp_save = c_p->cp;
     Uint32 flags;
     Uint need = 0;
     Eterm* E = c_p->stop;
-- 
cgit v1.2.3


From 5e60a53cf306eb29a1f8396e773205d02a419ef6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 21 Jan 2016 19:57:08 +0100
Subject: erts: Make msacc alloctor type thread safe

LONG_LIVED is not thread safe on non-smp and
can only be used by scheduler.
---
 erts/emulator/beam/erl_alloc.types | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 6b7eff1428..5f153ac0ab 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -361,7 +361,7 @@ type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
-type    MSACC           LONG_LIVED      SYSTEM          microstate_accounting
+type    MSACC           DRIVER          SYSTEM          microstate_accounting
 
 #
 # Types used by system specific code
-- 
cgit v1.2.3


From c96b6c2f58642b457d806c0a8a5bed03d16e35f1 Mon Sep 17 00:00:00 2001
From: Dmytro Lytovchenko <dmytro.lytovchenko@gmail.com>
Date: Thu, 21 Jan 2016 17:20:47 +0100
Subject: Better list_to_integer

Now tries to use whole width of signed long (Sint) and this halves amount of
multiplications needed to parse long integers. New code is 2-3 times faster
than the old code for large inputs (tens and hundreds of digits), behavior
should not change for small inputs.

Test ran 10k times with GC forced between attempts.

Was (R17):
720 el base 10: 0.14682 sec; base 16: 0.192722 sec; base 36: 0.337118 sec.
2800 el base 10: 1.794133 sec; base 16: 2.735106 sec; base 36: 4.761108 sec.
6500 el base 10: 9.316434 sec; base 16: 14.109469 sec; base 36: 25.319263 sec.

Now (R19 Dev)
720 el base 10: 0.10265 sec; base 16: 0.10851 sec; base 36: 0.160478 sec.
2800 el base 10: 1.002793 sec; base 16: 1.360649 sec; base 36: 2.174309 sec.
6500 el base 10: 4.722197 sec; base 16: 6.60522 sec; base 36: 10.552795 sec.

Added test for corner cases and sign bit corruption. Replaced macros with
inline and hid it inside C file to not pollute global namespace

Old bug in #define LG2_LOOKUP: Replaced with inline function and table
recalculated for all bases 2 to 36 (was 2 to 64)
---
 erts/emulator/beam/bif.c |  48 +++----
 erts/emulator/beam/big.c | 346 +++++++++++++++++++++++++++++------------------
 erts/emulator/beam/big.h |  22 +--
 3 files changed, 241 insertions(+), 175 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d519216fbd..34611ad6ab 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2904,8 +2904,11 @@ BIF_RETTYPE integer_to_list_1(BIF_ALIST_1)
 
 /**********************************************************************/
 
-/* convert a list of ascii ascii integer value to an integer */
-
+/*
+ * Converts a list of ascii base10 digits to an integer fully or partially.
+ * Returns result and the remaining tail.
+ * On error returns: {error,not_a_list}, or {error, no_integer}
+ */
 
 BIF_RETTYPE string_to_integer_1(BIF_ALIST_1)
 {
@@ -2913,9 +2916,8 @@ BIF_RETTYPE string_to_integer_1(BIF_ALIST_1)
      Eterm tail;
      Eterm *hp;
      /* must be a list */
-     switch (do_list_to_integer(BIF_P,BIF_ARG_1,&res,&tail)) {
-	 /* HAlloc after do_list_to_integer as it 
-	    might HAlloc itself (bignum) */
+     switch (erts_list_to_integer(BIF_P, BIF_ARG_1, 10, &res, &tail)) {
+     /* HAlloc after erts_list_to_integer as it might HAlloc itself (bignum) */
      case LTI_BAD_STRUCTURE:
 	 hp = HAlloc(BIF_P,3);
 	 BIF_RET(TUPLE2(hp, am_error, am_not_a_list));
@@ -2930,13 +2932,14 @@ BIF_RETTYPE string_to_integer_1(BIF_ALIST_1)
 
 BIF_RETTYPE list_to_integer_1(BIF_ALIST_1)
  {
-   /* Using do_list_to_integer is about twice as fast as using 
+   /* Using erts_list_to_integer is about twice as fast as using
       erts_chars_to_integer because we do not have to copy the 
       entire list */
      Eterm res;
      Eterm dummy;
      /* must be a list */
-     if (do_list_to_integer(BIF_P,BIF_ARG_1,&res,&dummy) != LTI_ALL_INTEGER) {
+     if (erts_list_to_integer(BIF_P, BIF_ARG_1, 10,
+                              &res, &dummy) != LTI_ALL_INTEGER) {
 	 BIF_ERROR(BIF_P,BADARG);
      }
      BIF_RET(res);
@@ -2944,14 +2947,12 @@ BIF_RETTYPE list_to_integer_1(BIF_ALIST_1)
 
 BIF_RETTYPE list_to_integer_2(BIF_ALIST_2)
 {
-
   /* Bif implementation is about 50% faster than pure erlang,
      and since we have erts_chars_to_integer now it is simpler
      as well. This could be optmized further if we did not have to
      copy the list to buf. */
     int i;
-    Eterm res;
-    char *buf = NULL;
+    Eterm res, dummy;
     int base;
 
     i = erts_list_length(BIF_ARG_1);
@@ -2959,31 +2960,16 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2)
       BIF_ERROR(BIF_P, BADARG);
     
     base = signed_val(BIF_ARG_2);
-  
+
     if (base < 2 || base > 36) 
       BIF_ERROR(BIF_P, BADARG);
 
-    /* Take fast path if base it 10 */
-    if (base == 10)
-      return list_to_integer_1(BIF_P,&BIF_ARG_1);
-    
-    buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
-    
-    if (intlist_to_buf(BIF_ARG_1, buf, i) < 0)
-      goto list_to_integer_1_error;
-    buf[i] = '\0';		/* null terminal */
-    
-    if ((res = erts_chars_to_integer(BIF_P,buf,i,base)) == THE_NON_VALUE)
-      goto list_to_integer_1_error;
-    
-    erts_free(ERTS_ALC_T_TMP, (void *) buf);
+    if (erts_list_to_integer(BIF_P, BIF_ARG_1, base,
+                             &res, &dummy) != LTI_ALL_INTEGER) {
+        BIF_ERROR(BIF_P,BADARG);
+    }
     BIF_RET(res);
-    
- list_to_integer_1_error:
-    erts_free(ERTS_ALC_T_TMP, (void *) buf);
-    BIF_ERROR(BIF_P, BADARG);
-     
- }
+}
 
 /**********************************************************************/
 
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 29e677d2e5..11838e24ef 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -48,7 +48,7 @@
 	    _t_dst = (dst)+((sz)-1);				\
 	    _t_src = (src)+((sz)-1);				\
 	    while(_t_sz--) *_t_dst-- = *_t_src--;		\
-	}							\
+        }							\
     } while(0)
 
 /* add a and b with carry in + out */
@@ -423,6 +423,25 @@
 
 #endif
 
+/* Forward declaration of lookup tables (See below in this file) used in list to
+ * integer conversions for different bases. Also used in bignum printing.
+ */
+static const byte digits_per_sint_lookup[36-1];
+static const byte digits_per_small_lookup[36-1];
+static const Sint largest_power_of_base_lookup[36-1];
+
+static ERTS_INLINE byte get_digits_per_signed_int(Uint base) {
+    return digits_per_sint_lookup[base-2];
+}
+
+static ERTS_INLINE byte get_digits_per_small(Uint base) {
+    return digits_per_small_lookup[base-2];
+}
+
+static ERTS_INLINE Sint get_largest_power_of_base(Uint base) {
+    return largest_power_of_base_lookup[base-2];
+}
+
 /*
 ** compare two number vectors
 */
@@ -1719,8 +1738,10 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg)
     short sign = BIG_SIGN(xp);
     ErtsDigit rem;
     Uint n = 0;
+    const Uint digits_per_Sint = get_digits_per_signed_int(10);
+    const Sint largest_pow_of_base = get_largest_power_of_base(10);
 
-    if (xl == 1 && *dx < D_DECIMAL_BASE) {
+    if (xl == 1 && *dx < largest_pow_of_base) {
 	rem = *dx;
 	if (rem == 0) {
 	    (*write_func)(arg, '0'); n++;
@@ -1738,7 +1759,7 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg)
 	MOVE_DIGITS(tmp, dx, xl);
 
 	while(1) {
-	    tmpl = D_div(tmp, tmpl, D_DECIMAL_BASE, tmp, &rem);
+            tmpl = D_div(tmp, tmpl, largest_pow_of_base, tmp, &rem);
 	    if (tmpl == 1 && *tmp == 0) {
 		while(rem) {
 		    (*write_func)(arg, (rem % 10)+'0'); n++;
@@ -1746,7 +1767,7 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg)
 		}
 		break;
 	    } else {
-		int i = D_DECIMAL_EXP;
+                Uint i = digits_per_Sint;
 		while(i--) {
 		    (*write_func)(arg, (rem % 10)+'0'); n++;
 		    rem /= 10;
@@ -2522,63 +2543,100 @@ int term_equals_2pow32(Eterm x)
     }
 }
 
+static ERTS_INLINE int c2int_is_invalid_char(byte ch, int base) {
+    return (ch < '0'
+            || (ch > ('0' + base - 1)
+                && !(base > 10
+                     && ((ch >= 'a' && ch < ('a' + base - 10))
+                         || (ch >= 'A' && ch < ('A' + base - 10))))));
+}
 
-#define IS_VALID_CHARACTER(CHAR,BASE) \
-  (CHAR < '0'				    \
-   || (CHAR > ('0' + BASE - 1)		    \
-       && !(BASE > 10							\
-	    && ((CHAR >= 'a' && CHAR < ('a' + BASE - 10))		\
-		|| (CHAR >= 'A' && CHAR < ('A' + BASE - 10))))))
-#define CHARACTER_FROM_BASE(CHAR)					\
-  ((CHAR <= '9') ? CHAR - '0' : 10 + ((CHAR <= 'Z') ? CHAR - 'A' : CHAR - 'a'))
-#define D_BASE_EXP(BASE) (d_base_exp_lookup[BASE-2])
-#define D_BASE_BASE(BASE) (d_base_base_lookup[BASE-2])
-#define LG2_LOOKUP(BASE)  (lg2_lookup[base-2])
+static ERTS_INLINE byte c2int_digit_from_base(byte ch) {
+    return ch <= '9' ? ch - '0'
+            : (10 + (ch <= 'Z' ? ch - 'A' : ch - 'a'));
+}
 
 /*
- * for i in 2..64 do
- *   lg2_lookup[i-2] = log2(i)
- * end
- * How many bits are needed to store string of size n
+ * How many bits are needed to store 1 digit of given base in binary
+ * Wo.Alpha formula: Table [log2[n], {n,2,36}]
  */
-const double lg2_lookup[] = { 1.0, 1.58496, 2, 2.32193, 2.58496, 2.80735, 3.0,
-	3.16993, 3.32193, 3.45943, 3.58496, 3.70044, 3.80735, 3.90689, 4.0,
-	4.08746, 4.16993, 4.24793, 4.32193, 4.39232, 4.45943, 4.52356, 4.58496,
-	4.64386, 4.70044, 4.75489, 4.80735, 4.85798, 4.90689, 4.9542, 5.0,
-	5.04439, 5.08746, 5.12928, 5.16993, 5.20945, 5.24793, 5.2854, 5.32193,
-	5.35755, 5.39232, 5.42626, 5.45943, 5.49185, 5.52356, 5.55459, 5.58496,
-	5.61471, 5.64386, 5.67243, 5.70044, 5.72792, 5.75489, 5.78136, 5.80735,
-	5.83289, 5.85798, 5.88264, 5.90689, 5.93074, 5.9542, 5.97728, 6.0 };
+static const double lg2_lookup[36-1] = {
+    1.0, 1.58496, 2.0, 2.32193, 2.58496, 2.80735, 3.0, 3.16993, 3.32193,
+    3.45943, 3.58496, 3.70044, 3.80735, 3.90689, 4.0, 4.08746, 4.16993, 4.24793,
+    4.32193, 4.39232, 4.45943, 4.52356, 4.58496, 4.64386, 4.70044, 4.75489,
+    4.80735, 4.85798, 4.90689, 4.9542, 5.0, 5.04439, 5.08746, 5.12928, 5.16993
+};
+static ERTS_INLINE double lookup_log2(Uint base) {
+    return lg2_lookup[base - 2];
+}
 
 /*
- * for i in 2..64 do
- *   d_base_exp_lookup[i-2] = 31 / lg2_lookup[i-2];
- * end
- * How many characters can fit in 31 bits
+ * How many digits can fit into a signed int (Sint) for given base, we take
+ * one digit away just to be on the safer side (some corner cases).
  */
-const byte d_base_exp_lookup[] = { 31, 19, 15, 13, 11, 11, 10, 9, 9, 8, 8, 8, 8,
-	7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5,
-	5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
-	5, 5 };
+static const byte digits_per_sint_lookup[36-1] = {
+#if (SIZEOF_VOID_P == 4)
+    /* Wo.Alpha formula: Table [Trunc[31 / log[2,n]]-1, {n, 2, 36}] */
+    30, 18, 14, 12, 10, 10, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5,
+    5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4
+#elif (SIZEOF_VOID_P == 8)
+    /* Wo.Alpha formula: Table [Trunc[63 / log[2,n]]-1, {n, 2, 36}] */
+    62, 38, 30, 26, 23, 21, 20, 18, 17, 17, 16, 16, 15, 15, 14, 14, 14, 13, 13,
+    13, 13, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11
+#else
+    #error "Please produce a lookup table for the new architecture"
+#endif
+};
 
 /*
- * for i in 2..64 do
- *   d_base_base_lookup[i-2] = pow(i,d_base_exp_lookup[i-2]);
- * end
- * How much can the characters which fit in 31 bit represent
+ * How many digits can fit into Erlang Small (SMALL_BITS-1) counting sign bit
  */
-const Uint d_base_base_lookup[] = { 2147483648u, 1162261467u, 1073741824u,
-	1220703125u, 362797056u, 1977326743u, 1073741824u, 387420489u,
-	1000000000u, 214358881u, 429981696u, 815730721u, 1475789056u,
-	170859375u, 268435456u, 410338673u, 612220032u, 893871739u, 1280000000u,
-	1801088541u, 113379904u, 148035889u, 191102976u, 244140625u, 308915776u,
-	387420489u, 481890304u, 594823321u, 729000000u, 887503681u, 1073741824u,
-	1291467969u, 1544804416u, 1838265625u, 60466176u, 69343957u, 79235168u,
-	90224199u, 102400000u, 115856201u, 130691232u, 147008443u, 164916224u,
-	184528125u, 205962976u, 229345007u, 254803968u, 282475249u, 312500000u,
-	345025251u, 380204032u, 418195493u, 459165024u, 503284375u, 550731776u,
-	601692057u, 656356768u, 714924299u, 777600000u, 844596301u, 916132832u,
-	992436543u, 1073741824u };
+static const byte digits_per_small_lookup[36-1] = {
+#if (SIZEOF_VOID_P == 4)
+    /* Wo.Alpha formula: Table [Trunc[27 / log[2,n]]-1, {n, 2, 36}] */
+    27, 17, 13, 11, 10, 9, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5,
+    5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
+#elif (SIZEOF_VOID_P == 8)
+    /* Wo.Alpha formula: Table [Trunc[59 / log[2,n]]-1, {n, 2, 36}] */
+    59, 37, 29, 25, 22, 21, 19, 18, 17, 17, 16, 15, 15, 15, 14, 14, 14, 13, 13,
+    13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11
+#else
+    #error "Please produce a lookup table for the new architecture"
+#endif
+};
+
+/*
+ * Largest power of base which can be represented in a signed int (Sint).
+ * Calculated by base 2..36 to the power of corresponding element from
+ * digits_per_sint_lookup.
+ */
+static const Sint largest_power_of_base_lookup[36-1] = {
+#if (SIZEOF_VOID_P == 4)
+    /* Wo.Alpha formula: Table [Pow[n, Trunc[31 / log[2,n]]-1], {n, 2, 36}] */
+    1073741824, 387420489, 268435456, 244140625, 60466176, 282475249, 134217728,
+    43046721, 100000000, 19487171, 35831808, 62748517, 105413504, 11390625,
+    16777216, 24137569, 34012224, 47045881, 64000000, 85766121, 5153632,
+    6436343,7962624, 9765625, 11881376, 14348907, 17210368, 20511149, 24300000,
+    28629151, 33554432, 39135393, 45435424, 52521875, 1679616
+#elif (SIZEOF_VOID_P == 8)
+    /* Wo.Alpha formula: Table [Pow[n, Trunc[63 / log[2,n]]-1], {n, 2, 36}]
+     * with LL added after each element manually */
+    4611686018427387904LL, 1350851717672992089LL, 1152921504606846976LL,
+    1490116119384765625LL, 789730223053602816LL, 558545864083284007LL,
+    1152921504606846976LL, 150094635296999121LL, 100000000000000000LL,
+    505447028499293771LL, 184884258895036416LL, 665416609183179841LL,
+    155568095557812224LL, 437893890380859375LL, 72057594037927936LL,
+    168377826559400929LL, 374813367582081024LL, 42052983462257059LL,
+    81920000000000000LL, 154472377739119461LL, 282810057883082752LL,
+    21914624432020321LL, 36520347436056576LL, 59604644775390625LL,
+    95428956661682176LL, 150094635296999121LL, 232218265089212416LL,
+    12200509765705829LL, 17714700000000000LL, 25408476896404831LL,
+    36028797018963968LL, 50542106513726817LL, 70188843638032384LL,
+    96549157373046875LL, 131621703842267136LL
+#else
+    #error "Please produce a lookup table for the new architecture"
+#endif
+};
 
 Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, 
 			   Uint size, const int base) {
@@ -2588,8 +2646,11 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes,
     int neg = 0;
     byte b;
     Eterm *hp, *hp_end;
-    int m;
+    Sint m;
     int lg2;
+    const Uint digits_per_small = get_digits_per_small(base);
+    const Uint digits_per_Sint = get_digits_per_signed_int(base);
+    const Sint largest_pow_of_base = get_largest_power_of_base(base);
 
     if (size == 0)
 	goto bytebuf_to_integer_1_error;
@@ -2604,57 +2665,68 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes,
 	size--;
     }
 
+    /* Trim leading zeroes */
+    if (size) {
+        while (*bytes == '0') {
+            bytes++;
+            size--;
+            if (!size) {
+                /* All zero! */
+                res = make_small(0);
+                goto bytebuf_to_integer_1_done;
+            }
+        }
+    }
+
     if (size == 0)
 	goto bytebuf_to_integer_1_error;
 
-    if (size < SMALL_DIGITS && base <= 10) {
-	/* *
-	 * Take shortcut if we know that all chars are '0' < b < '9' and
-	 * fit in a small. This improves speed by about 10% over the generic
-	 * small case.
-	 * */
-	while (size--) {
-	    b = *bytes++;
+    if (size < digits_per_small) {
+        if (base <= 10) {
+            /* *
+         * Take shortcut if we know that all chars are '0' < b < '9' and
+         * fit in a small. This improves speed by about 10% over the generic
+         * small case.
+         * */
+            while (size--) {
+                b = *bytes++;
 
-	    if (b < '0' || b > ('0'+base-1))
-		goto bytebuf_to_integer_1_error;
+                if (b < '0' || b > ('0'+base-1))
+                    goto bytebuf_to_integer_1_error;
 
-	    i = i * base + b - '0';
-	}
+                i = i * base + b - '0';
+            }
 
-	if (neg)
-	    i = -i;
-	res = make_small(i);
-	goto bytebuf_to_integer_1_done;
+            if (neg)
+                i = -i;
+            res = make_small(i);
+            goto bytebuf_to_integer_1_done;
+        }
+
+        /* Take shortcut if we know it will fit in a small.
+         * This improves speed by about 30%.
+         */
+        while (size) {
+            b = *bytes++;
+            size--;
+
+            if (c2int_is_invalid_char(b, base))
+                goto bytebuf_to_integer_1_error;
+
+            i = i * base + c2int_digit_from_base(b);
+        }
+
+        if (neg)
+            i = -i;
+        res = make_small(i);
+        goto bytebuf_to_integer_1_done;
     }
 
     /*
      * Calculate the maximum number of bits which will
      * be needed to represent the binary
      */
-    lg2 = ((size+2)*LG2_LOOKUP(base)+1);
-
-    if (lg2 < SMALL_BITS) {
-	/* Take shortcut if we know it will fit in a small.
-	 * This improves speed by about 30%.
-	 */
-	while (size) {
-	    b = *bytes++;
-	    size--;
-
-	    if (IS_VALID_CHARACTER(b,base))
-		goto bytebuf_to_integer_1_error;
-
-	    i = i * base + CHARACTER_FROM_BASE(b);
-
-	}
-
-	if (neg)
-	    i = -i;
-	res = make_small(i);
-	goto bytebuf_to_integer_1_done;
-
-    }
+    lg2 = ((size+2)*lookup_log2(base)+1);
 
     /* Start calculating bignum */
     m = (lg2 + D_EXP-1)/D_EXP;
@@ -2663,8 +2735,8 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes,
     hp = HAlloc(BIF_P, m);
     hp_end = hp + m;
 
-    if ((i = (size % D_BASE_EXP(base))) == 0)
-	i = D_BASE_EXP(base);
+    if ((i = (size % digits_per_Sint)) == 0)
+        i = digits_per_Sint;
 
     n = size - i;
     m = 0;
@@ -2672,34 +2744,34 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes,
     while (i--) {
 	b = *bytes++;
 
-	if (IS_VALID_CHARACTER(b,base)) {
+        if (c2int_is_invalid_char(b,base)) {
 	    HRelease(BIF_P, hp_end, hp);
 	    goto bytebuf_to_integer_1_error;
 	}
 
-	m = base * m + CHARACTER_FROM_BASE(b);
+        m = base * m + c2int_digit_from_base(b);
     }
 
     res = small_to_big(m, hp);
 
     while (n) {
-	i = D_BASE_EXP(base);
-	n -= D_BASE_EXP(base);
+        i = digits_per_Sint;
+        n -= digits_per_Sint;
 	m = 0;
 	while (i--) {
 	    b = *bytes++;
 
-	    if (IS_VALID_CHARACTER(b,base)) {
+            if (c2int_is_invalid_char(b,base)) {
 	      HRelease(BIF_P, hp_end, hp);
 	      goto bytebuf_to_integer_1_error;
 	    }
 
-	    m = base * m + CHARACTER_FROM_BASE(b);
+            m = base * m + c2int_digit_from_base(b);
 	}
 	if (is_small(res)) {
 	    res = small_to_big(signed_val(res), hp);
 	}
-	res = big_times_small(res, D_BASE_BASE(base), hp);
+        res = big_times_small(res, largest_pow_of_base, hp);
 	if (is_small(res)) {
 	    res = small_to_big(signed_val(res), hp);
 	}
@@ -2730,31 +2802,35 @@ bytebuf_to_integer_1_error:
 
 bytebuf_to_integer_1_done:
     return res;
-
 }
 
-int do_list_to_integer(Process *p, Eterm orig_list,
-                       Eterm *integer, Eterm *rest)
+/* Converts list of digits with given 'base' to integer sequentially. Returns
+ * result in 'integer_out', remaining tail goes to 'tail_out' and returns result
+ * code if the list was consumed fully or partially or there was an error
+ */
+LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list,
+                                  const Uint base,
+                                  Eterm *integer_out, Eterm *tail_out)
 {
      Sint i = 0;
      Uint ui = 0;
      int skip = 0;
      int neg = 0;
      Sint n = 0;
-     int m;
+     Sint m;
      int lg2;
      Eterm res;
-     Eterm* hp;
-     Eterm *hp_end;
      Eterm lst = orig_list;
      Eterm tail = lst;
      int error_res = LTI_BAD_STRUCTURE;
+     const Uint digits_per_small = get_digits_per_small(base);
+     const Uint digits_per_Sint = get_digits_per_signed_int(base);
 
      if (is_nil(lst)) {
        error_res = LTI_NO_INTEGER;
      error:
-         *rest = tail;
-         *integer = make_small(0);
+         *tail_out = tail;
+         *integer_out = make_small(0);
          return error_res;
      }
      if (is_not_list(lst))
@@ -2784,15 +2860,16 @@ int do_list_to_integer(Process *p, Eterm orig_list,
      /* Calculate size and do type check */
 
      while(1) {
+         byte ch;
          if (is_not_small(CAR(list_val(lst)))) {
              break;
          }
-         if (unsigned_val(CAR(list_val(lst))) < '0' ||
-             unsigned_val(CAR(list_val(lst))) > '9') {
+         ch = unsigned_val(CAR(list_val(lst)));
+         if (c2int_is_invalid_char(ch, base)) {
              break;
          }
-         ui = ui * 10;
-         ui = ui + unsigned_val(CAR(list_val(lst))) - '0';
+         ui = ui * base;
+         ui = ui + c2int_digit_from_base(ch);
          n++;
          lst = CDR(list_val(lst));
          if (is_nil(lst)) {
@@ -2810,23 +2887,24 @@ int do_list_to_integer(Process *p, Eterm orig_list,
      }
 
 
-      /* If n <= 8 then we know it's a small int
-      ** since 2^27 = 134217728. If n > 8 then we must
-      ** construct a bignum and let that routine do the checking
-      */
+      /* If length fits inside Sint then we know it's a small int. Else we
+       * must construct a bignum and let that routine do the checking
+       */
 
-     if (n <= SMALL_DIGITS) {  /* It must be small */
-         if (neg) i = -(Sint)ui;
-         else i = (Sint)ui;
+     if (n <= digits_per_small) {  /* It must be small */
+         i = neg ? -(Sint)ui : (Sint)ui;
          res = make_small(i);
      } else {
-         /* Convert from log10 to log2 by multiplying with 1/log10(2)=3.3219
-            which we round up to (3 + 1/3) */
-         lg2 = (n+1)*3 + (n+1)/3 + 1;
+         const Sint largest_pow_of_base = get_largest_power_of_base(base);
+         Eterm *hp;
+         Eterm *hp_end;
+
+         /* Convert from log_base to log2 using lookup table */
+         lg2 = ((n+2)*lookup_log2(base)+1);
          m  = (lg2+D_EXP-1)/D_EXP; /* number of digits */
          m  = BIG_NEED_SIZE(m);    /* number of words + thing */
 
-         hp = HAlloc(p, m);
+         hp = HAlloc(BIF_P, m);
          hp_end = hp + m;
 
          lst = orig_list;
@@ -2834,27 +2912,29 @@ int do_list_to_integer(Process *p, Eterm orig_list,
              lst = CDR(list_val(lst));
 
          /* load first digits (at least one digit) */
-         if ((i = (n % D_DECIMAL_EXP)) == 0)
-             i = D_DECIMAL_EXP;
+         if ((i = (n % digits_per_Sint)) == 0)
+             i = digits_per_Sint;
          n -= i;
          m = 0;
          while(i--) {
-             m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0');
+             m *= base;
+             m += c2int_digit_from_base(unsigned_val(CAR(list_val(lst))));
              lst = CDR(list_val(lst));
          }
          res = small_to_big(m, hp);  /* load first digits */
 
          while(n) {
-             i = D_DECIMAL_EXP;
-             n -= D_DECIMAL_EXP;
+             i = digits_per_Sint;
+             n -= digits_per_Sint;
              m = 0;
              while(i--) {
-                 m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0');
+                 m *= base;
+                 m += c2int_digit_from_base(unsigned_val(CAR(list_val(lst))));
                  lst = CDR(list_val(lst));
              }
              if (is_small(res))
                  res = small_to_big(signed_val(res), hp);
-             res = big_times_small(res, D_DECIMAL_BASE, hp);
+             res = big_times_small(res, largest_pow_of_base, hp);
              if (is_small(res))
                  res = small_to_big(signed_val(res), hp);
              res = big_plus_small(res, m, hp);
@@ -2876,10 +2956,10 @@ int do_list_to_integer(Process *p, Eterm orig_list,
                  hp += (big_arity(res)+1);
              }
          }
-         HRelease(p,hp_end,hp);
+         HRelease(BIF_P, hp_end, hp);
      }
-     *integer = res;
-     *rest = tail;
+     *integer_out = res;
+     *tail_out = tail;
      if (tail != NIL) {
          return LTI_SOME_INTEGER;
      }
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 85807d6eea..9c92de6b55 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -54,9 +54,6 @@ typedef Uint32   ErtsHalfDigit;
 #error "can not determine machine size"
 #endif
 
-#define D_DECIMAL_EXP	9
-#define D_DECIMAL_BASE	1000000000
-
 typedef Uint  dsize_t;	 /* Vector size type */
 
 #define D_EXP (ERTS_SIZEOF_ETERM*8)
@@ -173,12 +170,15 @@ Eterm erts_sint64_to_big(Sint64, Eterm **);
 
 Eterm erts_chars_to_integer(Process *, char*, Uint, const int);
 
-#define LTI_BAD_STRUCTURE 0
-#define LTI_NO_INTEGER 1
-#define LTI_SOME_INTEGER 2
-#define LTI_ALL_INTEGER 3
-
-int do_list_to_integer(Process *p, Eterm orig_list,
-                       Eterm *integer, Eterm *rest);
-
+/* How list_to_integer classifies the input, was it even a string? */
+typedef enum {
+    LTI_BAD_STRUCTURE = 0,
+    LTI_NO_INTEGER    = 1,
+    LTI_SOME_INTEGER  = 2,
+    LTI_ALL_INTEGER   = 3
+} LTI_result_t;
+
+LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list,
+                                  const Uint base,
+                                  Eterm *integer_out, Eterm *tail_out);
 #endif
-- 
cgit v1.2.3


From 18f0707c218ebdeb6024ecffd7704d3582e0b91c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 2 Feb 2016 16:20:19 +0100
Subject: Use nano second time unit in tracing

---
 erts/emulator/beam/erl_trace.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 2243639099..8a4c0ab1f2 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -165,7 +165,8 @@ take_timestamp(ErtsTraceTimeStamp *tsp, int ts_type)
     case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: {
 	Uint hsz = 0;
 	ErtsMonotonicTime mtime = erts_get_monotonic_time(NULL);
-	mtime += ERTS_MONOTONIC_OFFSET_NATIVE;
+	mtime = ERTS_MONOTONIC_TO_NSEC(mtime);
+	mtime += ERTS_MONOTONIC_OFFSET_NSEC;
 	hsz = (IS_SSMALL(mtime) ?
 	       (Uint) 0
 	       : ERTS_SINT64_HEAP_SIZE((Sint64) mtime));
-- 
cgit v1.2.3


From a4d6c798135440eadfba4832ac3cf77b06b6ac4f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 4 Feb 2016 15:59:33 +0100
Subject: Fix bug causing run-queue mask to become inconsistent

---
 erts/emulator/beam/erl_process.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index f50b217d4a..0db81de570 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1091,7 +1091,7 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
     (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
 #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
-    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
+    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
 
 /* The sequential tracing token is a tuple of size 5:
  *
-- 
cgit v1.2.3


From 14680fcc3fb9d0357fe33a94525d08896afed1c5 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Thu, 3 Dec 2015 14:33:53 +0100
Subject: erts: Use Sint instead of int for list lengths

This avoids potential integer arithmetic overflow for very large lists.
---
 erts/emulator/beam/beam_bif_load.c    |  6 +++---
 erts/emulator/beam/beam_load.c        |  4 ++--
 erts/emulator/beam/bif.c              | 18 +++++++++---------
 erts/emulator/beam/erl_bif_info.c     |  4 ++--
 erts/emulator/beam/erl_bif_lists.c    |  8 ++++----
 erts/emulator/beam/erl_bif_port.c     |  8 ++++----
 erts/emulator/beam/erl_process_dict.c |  4 ++--
 erts/emulator/beam/erl_utils.h        |  2 +-
 erts/emulator/beam/global.h           |  4 ++--
 erts/emulator/beam/utils.c            | 17 +++++++++--------
 10 files changed, 38 insertions(+), 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index a000935388..1b4c022370 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -179,8 +179,8 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions)
 BIF_RETTYPE
 finish_loading_1(BIF_ALIST_1)
 {
-    int i;
-    int n;
+    Sint i;
+    Sint n;
     struct m* p = NULL;
     Uint exceptions;
     Eterm res;
@@ -201,7 +201,7 @@ finish_loading_1(BIF_ALIST_1)
      */
 
     n = erts_list_length(BIF_ARG_1);
-    if (n == -1) {
+    if (n < 0) {
 	ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
 	goto done;
     }
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d367cce212..10f9f7bc2f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6219,10 +6219,10 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     BeamInstr* code_base;
     BeamInstr* fp;
     byte* info;
-    int n;
+    Sint n;
     int code_size;
     int rval;
-    int i;
+    Sint i;
     byte* temp_alloc = NULL;
     byte* bytes;
     Uint size;
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index e116b10b95..2654785ffc 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2837,7 +2837,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
 {
     Eterm res;
     char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS);
-    int i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS);
+    Sint i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS);
 
     if (i < 0) {
 	erts_free(ERTS_ALC_T_TMP, (void *) buf);
@@ -2857,7 +2857,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
  
 BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1)
 {
-    int i;
+    Sint i;
     char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS);
 
     if ((i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS)) < 0) {
@@ -2964,7 +2964,7 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2)
      and since we have erts_chars_to_integer now it is simpler
      as well. This could be optmized further if we did not have to
      copy the list to buf. */
-    int i;
+    Sint i;
     Eterm res, dummy;
     int base;
 
@@ -3288,7 +3288,7 @@ static BIF_RETTYPE do_charbuf_to_float(Process *BIF_P,char *buf) {
 
 BIF_RETTYPE list_to_float_1(BIF_ALIST_1)
 {
-    int i;
+    Sint i;
     Eterm res;
     char *buf = NULL;
 
@@ -3405,7 +3405,7 @@ BIF_RETTYPE list_to_tuple_1(BIF_ALIST_1)
     Eterm* cons;
     Eterm res;
     Eterm* hp;
-    int len;
+    Sint len;
 
     if ((len = erts_list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) {
 	BIF_ERROR(BIF_P, BADARG);
@@ -3752,7 +3752,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1)
 {
     Process* p = BIF_P;
     Eterm string = BIF_ARG_1;
-    int len = is_string(string);
+    Sint len = is_string(string);
     char *str;
 
     if (len <= 0) {
@@ -3806,7 +3806,7 @@ BIF_RETTYPE halt_1(BIF_ALIST_1)
 	erl_exit(ERTS_ABORT_EXIT, "");
     }
     else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
-	int i;
+	Sint i;
 
 	if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) {
 	    goto error;
@@ -3875,7 +3875,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 	erl_exit(ERTS_ABORT_EXIT, "");
     }
     else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
-	int i;
+	Sint i;
 
 	if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) {
 	    goto error;
@@ -4018,7 +4018,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
 {
     Uint a = 0, b = 0, c = 0;
     char* cp;
-    int i;
+    Sint i;
     DistEntry *dep = NULL;
     char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, 65);
     /*
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 017339e1f6..bc5c83e542 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1721,7 +1721,7 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	if (arity == 2) {
 	    Eterm res = THE_NON_VALUE;
 	    char *buf;
-	    int len = is_string(*tp);
+	    Sint len = is_string(*tp);
 	    if (len <= 0)
 		return res;
 	    buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
@@ -1740,7 +1740,7 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	    else {
 		Eterm res = THE_NON_VALUE;
 		char *buf;
-		int len = is_string(tp[1]);
+		Sint len = is_string(tp[1]);
 		if (len <= 0)
 		    return res;
 		buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 5583dcb371..fe64e76575 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -42,7 +42,7 @@ static BIF_RETTYPE append(Process* p, Eterm A, Eterm B)
     Eterm last;
     size_t need;
     Eterm* hp;
-    int i;
+    Sint i;
 
     if ((i = erts_list_length(A)) < 0) {
 	BIF_ERROR(p, BADARG);
@@ -99,9 +99,9 @@ static Eterm subtract(Process* p, Eterm A, Eterm B)
     Eterm small_vec[SMALL_VEC_SIZE];	/* Preallocated memory for small lists */
     Eterm* vec_p;
     Eterm* vp;
-    int     i;
-    int     n;
-    int     m;
+    Sint i;
+    Sint n;
+    Sint m;
     
     if ((n = erts_list_length(A)) < 0) {
 	BIF_ERROR(p, BADARG);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 839abd0424..3acc1d7bbd 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -649,7 +649,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
 static Port *
 open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 {
-    int i;
+    Sint i;
     Eterm option;
     Uint arity;
     Eterm* tp;
@@ -977,8 +977,8 @@ static char **convert_args(Eterm l)
 {
     char **pp;
     char *b;
-    int n;
-    int i = 0;
+    Sint n;
+    Sint i = 0;
     Eterm str;
     if (is_not_list(l) && is_not_nil(l)) {
 	return NULL;
@@ -1024,7 +1024,7 @@ static byte* convert_environment(Process* p, Eterm env)
     Eterm* temp_heap;
     Eterm* hp;
     Uint heap_size;
-    int n;
+    Sint n;
     Sint size;
     byte* bytes;
     int encoding = erts_get_native_filename_encoding();
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 84cd81aecf..36d16f7f42 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -749,7 +749,7 @@ static void shrink(Process *p, Eterm* ret)
 	    if (lo == NIL) {
 		ARRAY_PUT(pd, pd->splitPosition, hi);
 	    } else {
-		int needed = 4;
+		Sint needed = 4;
 		if (is_list(hi) && is_list(lo)) {
 		    needed = 2*erts_list_length(hi);
 		}
@@ -814,7 +814,7 @@ static void grow(Process *p)
     Eterm *hp;
     unsigned int pos;
     unsigned int homeSize;
-    int needed = 0;
+    Sint needed = 0;
     ProcDict *pd = p->dictionary;
 #ifdef DEBUG
     Eterm *hp_limit;
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 4058d63eaf..b86786a9c0 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -115,7 +115,7 @@ void erts_silence_warn_unused_result(long unused);
 int erts_fit_in_bits_int64(Sint64);
 int erts_fit_in_bits_int32(Sint32);
 int erts_fit_in_bits_uint(Uint);
-int erts_list_length(Eterm);
+Sint erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
 Uint32 make_broken_hash(Eterm);
 Uint32 block_hash(byte *, unsigned, Uint32);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index e8a7573e86..628f36a35e 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1308,7 +1308,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen);
 #define ERTS_UTF8_OK_MAX_CHARS 4
 
 void bin_write(int, void*, byte*, size_t);
-int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */
+Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */
 
 struct Sint_buf {
 #if defined(ARCH_64)
@@ -1381,7 +1381,7 @@ ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT);
 ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *);
 int erts_iolist_size_yielding(ErtsIOListState *state);
 int erts_iolist_size(Eterm, ErlDrvSizeT *);
-int is_string(Eterm);
+Sint is_string(Eterm);
 void erl_at_exit(void (*) (void*), void*);
 Eterm collect_memory(Process *);
 void dump_memory_to_fd(int);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ef851d840d..35dfee49cd 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -317,10 +317,10 @@ erl_grow_equeue(ErtsEQueue* q, Eterm* default_equeue)
  * Calculate length of a list.
  * Returns -1 if not a proper list (i.e. not terminated with NIL)
  */
-int
+Sint
 erts_list_length(Eterm list)
 {
-    int i = 0;
+    Sint i = 0;
 
     while(is_list(list)) {
 	i++;
@@ -3930,11 +3930,11 @@ void bin_write(int to, void *to_arg, byte* buf, size_t sz)
 /* Fill buf with the contents of bytelist list 
    return number of chars in list or -1 for error */
 
-int
-intlist_to_buf(Eterm list, char *buf, int len)
+Sint
+intlist_to_buf(Eterm list, char *buf, Sint len)
 {
     Eterm* listptr;
-    int sz = 0;
+    Sint sz = 0;
 
     if (is_nil(list)) 
 	return 0;
@@ -4481,11 +4481,12 @@ int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
     return iolist_size(0, NULL, obj, sizep);
 }
 
-/* return 0 if item is not a non-empty flat list of bytes */
-int
+/* return 0 if item is not a non-empty flat list of bytes
+   otherwise return the nonzero length of the list */
+Sint
 is_string(Eterm list)
 {
-    int len = 0;
+    Sint len = 0;
 
     while(is_list(list)) {
 	Eterm* consp = list_val(list);
-- 
cgit v1.2.3


From 7a319cd96f7f4869300b32442ebe892ae557f41c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 8 Feb 2016 15:47:49 +0100
Subject: erts: Fix error cases in enif_get_list_length

false if improper list
false if length > UINT_MAX
---
 erts/emulator/beam/erl_nif.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index b057ec7770..12aaf4ce53 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -911,8 +911,13 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail)
 
 int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len)
 {
-    if (is_not_list(term) && is_not_nil(term)) return 0;
-    *len = erts_list_length(term);
+    Sint i;
+    Uint u;
+
+    if ((i = erts_list_length(term)) < 0) return 0;
+    u = (Uint)i;
+    if ((unsigned)u != u) return 0;
+    *len = u;
     return 1;
 }
 
-- 
cgit v1.2.3


From a7f48d4972be0c7984d5cb0e08e73260c0fdbe1b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 27 Jan 2016 14:45:32 +0100
Subject: Ensure that work is done on the correct type of schedulers

Only the actual call to the dirty nif is allowed to execute on
dirty schedulers. The dirty nif is not allowed to execute on
normal schedulers if dirty schedulers are available.

Arrival of exit signals and system tasks, while a process was
scheduled for execution on a dirty scheduler, could mess up the
process internal state.

Preparation for dirty system task has been made, but is currently
unused.
---
 erts/emulator/beam/erl_nif.c          |  15 +-
 erts/emulator/beam/erl_process.c      | 650 +++++++++++++++++++++-------------
 erts/emulator/beam/erl_process.h      |  48 ++-
 erts/emulator/beam/erl_process_dump.c |  10 +-
 4 files changed, 458 insertions(+), 265 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d7a2076d85..962cb4858f 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1772,12 +1772,10 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
     ep->fp = NULL;
+    erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
+						   | ERTS_PSFLG_DIRTY_IO_PROC));
     result = (*fp)(env, argc, argv);
-    erts_smp_atomic32_read_band_mb(&proc->state,
-				   ~(ERTS_PSFLG_DIRTY_CPU_PROC
-				     |ERTS_PSFLG_DIRTY_IO_PROC
-				     |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q
-				     |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
+
     if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL)
 	close_lib(env->mod_nif);
     /*
@@ -1825,13 +1823,6 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[
     a = erts_smp_atomic32_read_acqb(&proc->state);
     while (1) {
 	n = state = a;
-	/*
-	 * clear any current dirty flags and dirty queue indicators,
-	 * in case the application is shifting a job from one type
-	 * of dirty scheduler to the other
-	 */
-	n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC
-	       |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q);
 	if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
 	    n |= ERTS_PSFLG_DIRTY_CPU_PROC;
 	else
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d583118e7b..1765315ed3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2969,7 +2969,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    sched_wall_time_change(esdp, thr_prgr_active);
 
 	while (1) {
-	    ErtsMonotonicTime current_time;
+	    ErtsMonotonicTime current_time = 0;
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
@@ -3635,6 +3635,13 @@ check_requeue_process(ErtsRunQueue *rq, int prio_q)
     return 0;
 }
 
+static ERTS_INLINE void
+free_proxy_proc(Process *proxy)
+{
+    ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
+    erts_free(ERTS_ALC_T_PROC, proxy);
+}
+
 #ifdef ERTS_SMP
 
 static ErtsRunQueue *
@@ -3949,9 +3956,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	erts_aint32_t state;
 	Process *proc;
 	int notify = 0;
-#ifdef ERTS_DIRTY_SCHEDULERS
-	int requeue;
-#endif
 	to_rq = NULL;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -3966,49 +3970,94 @@ evacuate_run_queue(ErtsRunQueue *rq,
 
 	proc = dequeue_process(rq, prio_q, &state);
 	while (proc) {
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    requeue = 1;
+	    Process *real_proc;
+	    int prio;
+	    erts_aint32_t max_qbit, qbit, real_state;
+
+	    prio = ERTS_PSFLGS_GET_PRQ_PRIO(state);
+	    qbit = ((erts_aint32_t) 1) << prio;
+
+	    if (!(state & ERTS_PSFLG_PROXY)) {
+		real_proc = proc;
+		real_state = state;
+	    }
+	    else {
+		real_proc = erts_proc_lookup_raw(proc->common.id);
+		if (!real_proc) {
+		    free_proxy_proc(proc);
+		    goto handle_next_proc;
+		}
+		real_state =  erts_smp_atomic32_read_acqb(&real_proc->state);
+	    }
+
+	    max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET);
+	    max_qbit &= ERTS_PSFLGS_QMASK;
+	    max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
+	    max_qbit &= -max_qbit;
+
+	    if (qbit > max_qbit) {
+		/* Process already queued with higher prio; drop it... */
+		if (real_proc != proc)
+		    free_proxy_proc(proc);
+		else {
+		    erts_aint32_t clr_bits;
+#ifdef DEBUG
+		    erts_aint32_t old;
 #endif
-	    if (ERTS_PSFLG_BOUND & state) {
-		/* Bound processes get stuck here... */
-		proc->next = NULL;
-		if (sbpp->last)
-		    sbpp->last->next = proc;
-		else
-		    sbpp->first = proc;
-		sbpp->last = proc;
-#ifdef ERTS_DIRTY_SCHEDULERS
-		requeue = 0;
+
+		    clr_bits = ERTS_PSFLG_IN_RUNQ;
+		    clr_bits |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET;
+
+#ifdef DEBUG
+		    old =
+#else
+			(void)
 #endif
+			erts_smp_atomic32_read_band_mb(&proc->state,
+						       ~clr_bits);
+			ASSERT((old & clr_bits) == clr_bits);
+
+		}
+
+		goto handle_next_proc;
 	    }
+
 #ifdef ERTS_DIRTY_SCHEDULERS
-	    else if (state & ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q) {
+
+	    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
+		erts_aint32_t dqbit = qbit;
 #ifdef DEBUG
-		erts_aint32_t old =
-#else
-		(void)
+		erts_aint32_t old_dqbit;
 #endif
-		    erts_smp_atomic32_read_band_nob(&proc->state,
-						    ~(ERTS_PSFLG_DIRTY_CPU_PROC
-						      | ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q));
-		/* assert that no other dirty flags are set */
-		ASSERT(!(old & (ERTS_PSFLG_DIRTY_IO_PROC|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)));
-	    } else if (state & ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) {
+
+		if (rq == ERTS_DIRTY_CPU_RUNQ)
+		    dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET;
+		else {
+		    ASSERT(rq == ERTS_DIRTY_IO_RUNQ);
+		    dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET;
+		}
+
 #ifdef DEBUG
-		erts_aint32_t old =
+		old_dqbit = (int)
 #else
-		(void)
+		    (void)
 #endif
-		    erts_smp_atomic32_read_band_nob(&proc->state,
-						    ~(ERTS_PSFLG_DIRTY_IO_PROC
-						      | ERTS_PSFLG_DIRTY_IO_PROC_IN_Q));
-		/* assert that no other dirty flags are set */
-		ASSERT(!(old & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q)));
+		    erts_smp_atomic32_read_band_mb(&real_proc->dirty_state,
+						   ~dqbit);
+		ASSERT(old_dqbit & dqbit);
 	    }
-	    if (requeue) {
-#else
-	    else {
 #endif
+
+	    if (ERTS_PSFLG_BOUND & real_state) {
+		/* Bound processes get stuck here... */
+		proc->next = NULL;
+		if (sbpp->last)
+		    sbpp->last->next = proc;
+		else
+		    sbpp->first = proc;
+		sbpp->last = proc;
+	    }
+	    else {
 		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		erts_smp_runq_unlock(rq);
 
@@ -4031,6 +4080,8 @@ evacuate_run_queue(ErtsRunQueue *rq,
 
 		erts_smp_runq_lock(rq);
 	    }
+
+	handle_next_proc:
 	    proc = dequeue_process(rq, prio_q, &state);
 	}
 	if (notify)
@@ -5974,19 +6025,96 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
     return proxy;
 }
 
-static ERTS_INLINE void
-free_proxy_proc(Process *proxy)
-{
-    ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
-    erts_free(ERTS_ALC_T_PROC, proxy);
-}
-
 #define ERTS_ENQUEUE_NOT 0
 #define ERTS_ENQUEUE_NORMAL_QUEUE 1
-#ifdef ERTS_DIRTY_SCHEDULERS
 #define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2
 #define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static int
+check_dirty_enqueue_in_prio_queue(Process *c_p,
+				  erts_aint32_t *newp,
+				  erts_aint32_t actual,
+				  erts_aint32_t aprio,
+				  erts_aint32_t qbit)
+{
+    int queue;
+    erts_aint32_t dact, max_qbit;
+
+    /* Termination should be done on an ordinary scheduler */
+    if (actual & ERTS_PSFLG_EXITING) {
+	*newp &= ~ERTS_PSFLGS_DIRTY_WORK;
+	return ERTS_ENQUEUE_NORMAL_QUEUE;
+    }
+
+    /*
+     * If we have system tasks, we enqueue on ordinary run-queue
+     * and take care of those system tasks first.
+     */
+    if (actual & ERTS_PSFLG_ACTIVE_SYS)
+	return ERTS_ENQUEUE_NORMAL_QUEUE;
+
+    dact = erts_smp_atomic32_read_mb(&c_p->dirty_state);
+    if (actual & (ERTS_PSFLG_DIRTY_ACTIVE_SYS
+		  | ERTS_PSFLG_DIRTY_CPU_PROC)) {
+	max_qbit = ((dact >> ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET)
+		    & ERTS_PDSFLGS_QMASK);
+	queue = ERTS_ENQUEUE_DIRTY_CPU_QUEUE;
+    }
+    else {
+	ASSERT(actual & ERTS_PSFLG_DIRTY_IO_PROC);
+	max_qbit = ((dact >> ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET)
+		    & ERTS_PDSFLGS_QMASK);
+	queue = ERTS_ENQUEUE_DIRTY_IO_QUEUE;
+    }
+
+    max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
+    max_qbit &= -max_qbit;
+
+    if (qbit >= max_qbit)
+	return ERTS_ENQUEUE_NOT; /* Already queued in higher or equal prio */
+    if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK))
+	!= (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) {
+	/*
+	 * Process struct already enqueued, or actual prio not
+	 * equal to user prio, i.e., enqueue using proxy.
+	 */
+	return -1*queue;
+    }
+
+    *newp |= ERTS_PSFLG_IN_RUNQ;
+    return queue;
+}
+
+static ERTS_INLINE int
+fin_dirty_enq_s_change(Process *p,
+		       int pstruct_reserved,
+		       erts_aint32_t enq_prio,
+		       int qmask_offset)
+{
+    erts_aint32_t qbit = 1 << enq_prio;
+    qbit <<= qmask_offset;
+
+    if (qbit & erts_smp_atomic32_read_bor_mb(&p->dirty_state, qbit)) {
+	/* Already enqueue by someone else... */
+	if (pstruct_reserved) {
+	    /* We reserved process struct for enqueue; clear it... */
+#ifdef DEBUG
+	    erts_aint32_t old =
+#else
+		(void)
 #endif
+		erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ);
+	    ASSERT(old & ERTS_PSFLG_IN_RUNQ);
+	}
+	return 0;
+    }
+
+    return !0;
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
 
 static ERTS_INLINE int
 check_enqueue_in_prio_queue(Process *c_p,
@@ -6002,61 +6130,14 @@ check_enqueue_in_prio_queue(Process *c_p,
     *prq_prio_p = aprio;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-    if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) {
-        /*
-         * If we have system tasks of a priority higher
-         * or equal to the user priority, we enqueue
-         * on ordinary run-queue and take care of
-         * those system tasks first.
-         */
-        if (actual & ERTS_PSFLG_ACTIVE_SYS) {
-            erts_aint32_t uprio, stprio, qmask;
-            uprio = (actual >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK;
-            if (aprio < uprio)
-                goto enqueue_normal_runq; /* system tasks with higher prio */
-            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
-            qmask = c_p->sys_task_qs->qmask;
-            erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
-            switch (qmask & -qmask) {
-            case MAX_BIT:
-                stprio = PRIORITY_MAX;
-                break;
-            case HIGH_BIT:
-                stprio = PRIORITY_HIGH;
-                break;
-            case NORMAL_BIT:
-                stprio = PRIORITY_NORMAL;
-                break;
-            case LOW_BIT:
-                stprio = PRIORITY_LOW;
-                break;
-            default:
-                stprio = PRIORITY_LOW+1;
-                break;
-            }
-            if (stprio <= uprio)
-                goto enqueue_normal_runq; /* system tasks with higher prio */
-        }
-
-        /* Enqueue in dirty run queue if not already enqueued */
-	if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))
-	    return ERTS_ENQUEUE_NOT; /* already in queue */
-        if (actual & ERTS_PSFLG_DIRTY_CPU_PROC) {
-	    *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q;
-            if (actual & ERTS_PSFLG_IN_RUNQ)
-                return -ERTS_ENQUEUE_DIRTY_CPU_QUEUE; /* use proxy */
-	    *newp |= ERTS_PSFLG_IN_RUNQ;
-            return ERTS_ENQUEUE_DIRTY_CPU_QUEUE;
-        }
-	*newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q;
-        if (actual & ERTS_PSFLG_IN_RUNQ)
-            return -ERTS_ENQUEUE_DIRTY_IO_QUEUE; /* use proxy */
-	*newp |= ERTS_PSFLG_IN_RUNQ;
-        return ERTS_ENQUEUE_DIRTY_IO_QUEUE;
+    if (actual & ERTS_PSFLGS_DIRTY_WORK) {
+	int res = check_dirty_enqueue_in_prio_queue(c_p, newp, actual,
+						    aprio, qbit);
+	if (res != ERTS_ENQUEUE_NORMAL_QUEUE)
+	    return res;
     }
-
- enqueue_normal_runq:
 #endif
+
     max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK;
     max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
     max_qbit &= -max_qbit;
@@ -6088,6 +6169,65 @@ check_enqueue_in_prio_queue(Process *c_p,
     return ERTS_ENQUEUE_NORMAL_QUEUE;
 }
 
+static ERTS_INLINE ErtsRunQueue *
+select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t state)
+{
+
+    switch (enqueue) {
+
+    case ERTS_ENQUEUE_NOT:
+
+	return NULL;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+    case ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
+    case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
+
+	if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio,
+				   ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET))
+	    return ERTS_DIRTY_CPU_RUNQ;
+
+	return NULL;
+
+
+    case ERTS_ENQUEUE_DIRTY_IO_QUEUE:
+    case -ERTS_ENQUEUE_DIRTY_IO_QUEUE:
+
+	if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio,
+				   ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET))
+	    return ERTS_DIRTY_IO_RUNQ;
+
+	return NULL;
+
+#endif
+
+    default: {
+	ErtsRunQueue* runq;
+
+	ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE
+	       || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE);
+
+	runq = erts_get_runq_proc(p);
+
+#ifdef ERTS_SMP
+	if (!(ERTS_PSFLG_BOUND & state)) {
+	    ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
+	    if (new_runq) {
+		RUNQ_SET_RQ(&p->run_queue, new_runq);
+		runq = new_runq;
+	    }
+	}
+#endif
+
+	ASSERT(runq);
+
+	return runq;
+    }
+    }
+}
+
+
 /*
  * schedule_out_process() return with c_rq locked.
  */
@@ -6096,11 +6236,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 {
     erts_aint32_t a, e, n, enq_prio = -1;
     int enqueue; /* < 0 -> use proxy */
-    Process* sched_p;
     ErtsRunQueue* runq;
-#ifdef ERTS_SMP
-    int check_emigration_need;
-#endif
 
     a = state;
 
@@ -6112,7 +6248,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 	enqueue = ERTS_ENQUEUE_NOT;
 
 	n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS);
-	if (a & ERTS_PSFLG_ACTIVE_SYS
+	if (a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)
 	    || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
 	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
 	}
@@ -6121,16 +6257,17 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 	    break;
     }
 
-    switch (enqueue) {
-    case ERTS_ENQUEUE_NOT:
+    runq = select_enqueue_run_queue(enqueue, enq_prio, p, n);
+
+    if (!runq) {
+
 	if (erts_system_profile_flags.runnable_procs) {
 
 	    /* Status lock prevents out of order "runnable proc" trace msgs */
 	    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-	    if (!(a & ERTS_PSFLG_ACTIVE_SYS)
-		&& (!(a & ERTS_PSFLG_ACTIVE)
-		    || (a & ERTS_PSFLG_SUSPENDED))) {
+	    if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS))
+		&& (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) {
 		/* Process inactive */
 		profile_runnable_proc(p, am_inactive);
 	    }
@@ -6143,98 +6280,76 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
 
 	return 0;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef ERTS_SMP
-    case ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
-    case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
-	runq = ERTS_DIRTY_CPU_RUNQ;
-	ASSERT(ERTS_SCHEDULER_IS_DIRTY_CPU(runq->scheduler));
-#ifdef ERTS_SMP
-	check_emigration_need = 0;
-#endif
-	break;
+    }
+    else {
+	Process* sched_p;
 
-    case ERTS_ENQUEUE_DIRTY_IO_QUEUE:
-    case -ERTS_ENQUEUE_DIRTY_IO_QUEUE:
-	runq = ERTS_DIRTY_IO_RUNQ;
-	ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(runq->scheduler));
-#ifdef ERTS_SMP
-	check_emigration_need = 0;
-#endif
-	break;
-#endif
-#endif
+	ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS
+						    | ERTS_PSFLG_DIRTY_ACTIVE_SYS)));
 
-    default:
-	ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE
-	       || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE);
+	if (enqueue < 0)
+	    sched_p = make_proxy_proc(proxy, p, enq_prio);
+	else {
+	    sched_p = p;
+	    if (proxy)
+		free_proxy_proc(proxy);
+	}
 
-	runq = erts_get_runq_proc(p);
-#ifdef ERTS_SMP
-	check_emigration_need = !(ERTS_PSFLG_BOUND & n);
-#endif
-	break;
-    }
+	ASSERT(runq);
 
-    ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS));
+	erts_smp_runq_lock(runq);
 
-    if (enqueue < 0)
-	sched_p = make_proxy_proc(proxy, p, enq_prio);
-    else {
-	sched_p = p;
-	if (proxy)
-	    free_proxy_proc(proxy);
-    }
+	/* Enqueue the process */
+	enqueue_process(runq, (int) enq_prio, sched_p);
 
-#ifdef ERTS_SMP
-    if (check_emigration_need) {
-	ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
-	if (new_runq) {
-	    RUNQ_SET_RQ(&sched_p->run_queue, new_runq);
-	    runq = new_runq;
-	}
-    }
-#endif
+	if (runq == c_rq)
+	    return 1;
 
-    ASSERT(runq);
+	erts_smp_runq_unlock(runq);
 
-    erts_smp_runq_lock(runq);
+	smp_notify_inc_runq(runq);
 
-    /* Enqueue the process */
-    enqueue_process(runq, (int) enq_prio, sched_p);
+	erts_smp_runq_lock(c_rq);
 
-    if (runq == c_rq)
 	return 1;
-    erts_smp_runq_unlock(runq);
-    smp_notify_inc_runq(runq);
-    erts_smp_runq_lock(c_rq);
-    return 1;
+    }
+
 }
 
 static ERTS_INLINE void
-add2runq(Process *p, erts_aint32_t state, erts_aint32_t prio)
+add2runq(int enqueue, erts_aint32_t prio,
+	 Process *proc, erts_aint32_t state,
+	 Process **proxy)
 {
-    ErtsRunQueue *runq = erts_get_runq_proc(p);
+    ErtsRunQueue *runq;
 
-#ifdef ERTS_SMP
-    if (!(ERTS_PSFLG_BOUND & state)) {
-	ErtsRunQueue *new_runq = erts_check_emigration_need(runq, (int) prio);
-	if (new_runq) {
-	    RUNQ_SET_RQ(&p->run_queue, new_runq);
-	    runq = new_runq;
-	}
-    }
-#endif
-    ASSERT(runq);
+    runq = select_enqueue_run_queue(enqueue, prio, proc, state);
 
-    erts_smp_runq_lock(runq);
+    if (runq) {
+	Process *sched_p;
 
-    /* Enqueue the process */
-    enqueue_process(runq, (int) prio, p);
+	if (enqueue > 0)
+	    sched_p = proc;
+	else {
+	    Process *pxy;
 
-    erts_smp_runq_unlock(runq);
-    smp_notify_inc_runq(runq);
+	    if (!proxy)
+		pxy = NULL;
+	    else {
+		pxy = *proxy;
+		*proxy = NULL;
+	    }
+	    sched_p = make_proxy_proc(pxy, proc, prio);
+	}
+
+	erts_smp_runq_lock(runq);
+
+	/* Enqueue the process */
+	enqueue_process(runq, (int) prio, sched_p);
 
+	erts_smp_runq_unlock(runq);
+	smp_notify_inc_runq(runq);
+    }
 }
 
 static ERTS_INLINE int
@@ -6340,10 +6455,7 @@ schedule_process(Process *p, erts_aint32_t in_state, ErtsProcLocks locks)
 					     &state,
 					     &enq_prio,
 					     locks);
-    if (enqueue != ERTS_ENQUEUE_NOT)
-	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
-		 state,
-		 enq_prio);
+    add2runq(enqueue, enq_prio, p, state, NULL);
 }
 
 void
@@ -6403,16 +6515,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy)
 	prof_runnable_procs = 0;
     }
 
-    if (enqueue != ERTS_ENQUEUE_NOT) {
-	Process *sched_p;
-	if (enqueue > 0)
-	    sched_p = p;
-	else {
-	    sched_p = make_proxy_proc(proxy, p, enq_prio);
-	    proxy = NULL;
-	}
-	add2runq(sched_p, n, enq_prio);
-    }
+    add2runq(enqueue, enq_prio, p, n, &proxy);
 
 cleanup:
 
@@ -6508,10 +6611,7 @@ resume_process(Process *p, ErtsProcLocks locks)
 					 &state,
 					 &enq_prio,
 					 locks);
-    if (enqueue)
-	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
-		 state,
-		 enq_prio);
+    add2runq(enqueue, enq_prio, p, state, NULL);
 }
 
 int
@@ -9361,15 +9461,16 @@ Process *schedule(Process *p, int calls)
 #ifdef ERTS_SMP
 	ErtsMigrationPaths *mps;
 	ErtsMigrationPath *mp;
-	ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
-	if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
+
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	    ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
+	    if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
 	    rq->procs.pending_exiters = NULL;
 	    erts_smp_runq_unlock(rq);
 	    handle_pending_exiters(pnd_xtrs);
 	    erts_smp_runq_lock(rq);
 	}
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    if (rq->check_balance_reds <= 0)
 		check_balance(rq);
 
@@ -9564,11 +9665,12 @@ Process *schedule(Process *p, int calls)
     pick_next_process: {
 	    erts_aint32_t psflg_band_mask;
 	    int prio_q;
-	    int qmask;
+	    int qmask, qbit;
 
 	    flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 	    qmask = (int) (flags & ERTS_RUNQ_FLGS_PROCS_QMASK);
-	    switch (qmask & -qmask) {
+	    qbit = qmask & -qmask;
+	    switch (qbit) {
 	    case MAX_BIT:
 		prio_q = PRIORITY_MAX;
 		break;
@@ -9596,20 +9698,11 @@ Process *schedule(Process *p, int calls)
 
 	    ASSERT(p); /* Wrong qmask in rq->flags? */
 
-	    psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state)
-							+ ERTS_PSFLGS_IN_PRQ_MASK_OFFSET));
-
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    ASSERT((state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) !=
-		   (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC));
-	    if (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) {
-		ASSERT((ERTS_SCHEDULER_IS_DIRTY_CPU(esdp) && (state & ERTS_PSFLG_DIRTY_CPU_PROC)) ||
-		       (ERTS_SCHEDULER_IS_DIRTY_IO(esdp) && (state & ERTS_PSFLG_DIRTY_IO_PROC)));
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !(state & ERTS_PSFLG_ACTIVE_SYS))
-		    goto pick_next_process;
-		state &= ~(ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q);
-	    }
-#endif
+	    if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+		psflg_band_mask = ~((erts_aint32_t) 0);
+	    else
+		psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state)
+							    + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET));
 
 	    if (!(state & ERTS_PSFLG_PROXY))
 		psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ;
@@ -9632,9 +9725,11 @@ Process *schedule(Process *p, int calls)
 			       | ERTS_PSFLG_RUNNING_SYS))) {
 		    tmp = state & (ERTS_PSFLG_SUSPENDED
 				   | ERTS_PSFLG_PENDING_EXIT
-				   | ERTS_PSFLG_ACTIVE_SYS);
+				   | ERTS_PSFLG_ACTIVE_SYS
+				   | ERTS_PSFLG_DIRTY_ACTIVE_SYS);
 		    if (tmp != ERTS_PSFLG_SUSPENDED) {
-			if (state & ERTS_PSFLG_ACTIVE_SYS)
+			if (state & (ERTS_PSFLG_ACTIVE_SYS
+				     | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
 			    new |= ERTS_PSFLG_RUNNING_SYS;
 			else
 			    new |= ERTS_PSFLG_RUNNING;
@@ -9647,7 +9742,8 @@ Process *schedule(Process *p, int calls)
 				  | ERTS_PSFLG_FREE))
 			|| ((state & (ERTS_PSFLG_SUSPENDED
 				      | ERTS_PSFLG_PENDING_EXIT
-				      | ERTS_PSFLG_ACTIVE_SYS))
+				      | ERTS_PSFLG_ACTIVE_SYS
+				      | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
 			    == ERTS_PSFLG_SUSPENDED)) {
 			if (state & ERTS_PSFLG_FREE)
 			    erts_proc_dec_refc(p);
@@ -9666,10 +9762,42 @@ Process *schedule(Process *p, int calls)
 
 	    esdp->current_process = p;
 
+
+	    reds = context_reds;
+
+#ifdef ERTS_SMP
+
+	    erts_smp_runq_unlock(rq);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+#ifdef DEBUG
+		int old_dqbit;
+#endif
+		int dqbit = qbit;
+
+		if (rq == ERTS_DIRTY_CPU_RUNQ)
+		    dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET;
+		else {
+		    ASSERT(rq == ERTS_DIRTY_IO_RUNQ);
+		    dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET;
+		}
+
+#ifdef DEBUG
+		old_dqbit = (int)
+#else
+		    (void)
+#endif
+		    erts_smp_atomic32_read_band_mb(&p->dirty_state, ~dqbit);
+		ASSERT(old_dqbit & dqbit);
+	    }
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+#endif /* ERTS_SMP */
+
 	}
 
 #ifdef ERTS_SMP
-	erts_smp_runq_unlock(rq);
 
 	if (flags & ERTS_RUNQ_FLG_PROTECTED)
 	    (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
@@ -9702,15 +9830,56 @@ Process *schedule(Process *p, int calls)
 	    erts_smp_spin_unlock(&erts_sched_stat.lock);
 	}
 
-	if (ERTS_PROC_PENDING_EXIT(p)) {
+	ASSERT(!p->scheduler_data);
+	p->scheduler_data = esdp;
+
+	state = erts_smp_atomic32_read_nob(&p->state);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	    if (!!(state & ERTS_PSFLGS_DIRTY_WORK)
+		& !(state & ERTS_PSFLG_ACTIVE_SYS)) {
+		/* Migrate to dirty scheduler... */
+	    sunlock_sched_out_proc:
+		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+		p->fcalls = reds;
+		goto sched_out_proc;
+
+	    }
+	}
+	else {
+	    if (state & (ERTS_PSFLG_ACTIVE_SYS
+			 | ERTS_PSFLG_PENDING_EXIT
+			 | ERTS_PSFLG_EXITING)) {
+		/* Migrate to normal scheduler... */
+		goto sunlock_sched_out_proc;
+	    }
+	    if ((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS)
+		&& rq == ERTS_DIRTY_IO_RUNQ) {
+		/* Migrate to dirty cpu scheduler... */
+		goto sunlock_sched_out_proc;
+	    }
+
+	    ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS)
+		   || *p->i == (BeamInstr) em_call_nif);
+
+	    ASSERT(rq == ERTS_DIRTY_CPU_RUNQ
+		   ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC
+			       | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
+		   : (rq == ERTS_DIRTY_IO_RUNQ
+		      && (state & ERTS_PSFLG_DIRTY_IO_PROC)));	    
+	}
+#endif
+
+	if (state & ERTS_PSFLG_PENDING_EXIT) {
 	    erts_handle_pending_exit(p,
 				     ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 	    state = erts_smp_atomic32_read_nob(&p->state);
 	}
-	ASSERT(!p->scheduler_data);
-	p->scheduler_data = esdp;
-#endif
-	reds = context_reds;
+
+#endif /* ERTS_SMP */
+
+	p->fcalls = reds;
 
 	if (IS_TRACED(p)) {
 	    if (state & ERTS_PSFLG_EXITING) {
@@ -9739,7 +9908,8 @@ Process *schedule(Process *p, int calls)
 	    reds -= execute_sys_tasks(p, &state, reds);
 	    if (reds <= 0
 #ifdef ERTS_DIRTY_SCHEDULERS
-		|| (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))
+		|| ERTS_SCHEDULER_IS_DIRTY(esdp)
+		|| (state & ERTS_PSFLGS_DIRTY_WORK)
 #endif
 		) {
 		p->fcalls = reds;
@@ -9787,7 +9957,6 @@ Process *schedule(Process *p, int calls)
 	    proxy_p = NULL;
 	}
 	    
-	p->fcalls = reds;
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
 	/* Never run a suspended process */
@@ -10732,6 +10901,9 @@ static void early_init_process_struct(void *varg, Eterm data)
     Process *proc = arg->proc;
 
     proc->common.id = make_internal_pid(data);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_atomic32_init_nob(&proc->dirty_state, 0);
+#endif
     erts_smp_atomic32_init_relb(&proc->state, arg->state);
 
 #ifdef ERTS_SMP
@@ -11194,6 +11366,9 @@ void erts_init_empty_process(Process *p)
     p->last_old_htop = NULL;
 #endif
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_atomic32_init_nob(&p->dirty_state, 0);
+#endif
     erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
 #ifdef ERTS_SMP
@@ -11395,7 +11570,9 @@ set_proc_exiting(Process *p,
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL);
 
     enqueue = change_proc_schedule_state(p,
-					 ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT,
+					 (ERTS_PSFLG_SUSPENDED
+					  | ERTS_PSFLG_PENDING_EXIT
+					  | ERTS_PSFLGS_DIRTY_WORK),
 					 ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
 					 &state,
 					 &enq_prio,
@@ -11429,10 +11606,7 @@ set_proc_exiting(Process *p,
     }
 #endif
 
-    if (enqueue)
-	add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio),
-		 state,
-		 enq_prio);
+    add2runq(enqueue, enq_prio, p, state, NULL);
 }
 
 static ERTS_INLINE erts_aint32_t
@@ -11535,6 +11709,11 @@ save_pending_exiter(Process *p)
     else
         rq = esdp->run_queue;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+	rq = ERTS_RUNQ_IX(0); /* Handle on ordinary scheduler */
+#endif
+
     plp = proclist_create(p);
 
     erts_smp_runq_lock(rq);
@@ -11544,13 +11723,8 @@ save_pending_exiter(Process *p)
     non_empty_runq(rq);
 
     erts_smp_runq_unlock(rq);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-	wake_dirty_schedulers(rq, 0);
-    else
-#endif
-	wake_scheduler(rq);
 
+    wake_scheduler(rq);
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 10c6fa4a67..32e458aa1d 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1033,6 +1033,9 @@ struct process {
     ErtsProcSysTaskQs *sys_task_qs;
 
     erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */
+#endif
 
 #ifdef ERTS_SMP
     ErlMessageInQueue msg_inq;
@@ -1150,15 +1153,14 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
-#ifdef ERTS_DIRTY_SCHEDULERS
 #define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
 #define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
-#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
-#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
-#else
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
-#endif
+#define ERTS_PSFLG_DIRTY_ACTIVE_SYS	ERTS_PSFLG_BIT(20)
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 20)
+
+#define ERTS_PSFLGS_DIRTY_WORK		(ERTS_PSFLG_DIRTY_CPU_PROC	\
+					 | ERTS_PSFLG_DIRTY_IO_PROC	\
+					 | ERTS_PSFLG_DIRTY_ACTIVE_SYS)
 
 #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
 					 | ERTS_PSFLG_IN_PRQ_HIGH	\
@@ -1170,7 +1172,37 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
     (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
 #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
-    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
+    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+/*
+ * Flags in the dirty_state field.
+ */
+
+#define ERTS_PDSFLG_IN_CPU_PRQ_MAX 	(((erts_aint32_t) 1) << 0)
+#define ERTS_PDSFLG_IN_CPU_PRQ_HIGH	(((erts_aint32_t) 1) << 1)
+#define ERTS_PDSFLG_IN_CPU_PRQ_NORMAL	(((erts_aint32_t) 1) << 2)
+#define ERTS_PDSFLG_IN_CPU_PRQ_LOW 	(((erts_aint32_t) 1) << 3)
+#define ERTS_PDSFLG_IN_IO_PRQ_MAX 	(((erts_aint32_t) 1) << 4)
+#define ERTS_PDSFLG_IN_IO_PRQ_HIGH	(((erts_aint32_t) 1) << 5)
+#define ERTS_PDSFLG_IN_IO_PRQ_NORMAL	(((erts_aint32_t) 1) << 6)
+#define ERTS_PDSFLG_IN_IO_PRQ_LOW 	(((erts_aint32_t) 1) << 7)
+
+#define ERTS_PDSFLGS_QMASK 		ERTS_PSFLGS_QMASK
+#define ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET 0
+#define ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET ERTS_PSFLGS_QMASK_BITS
+
+#define ERTS_PDSFLG_IN_CPU_PRQ_MASK 	(ERTS_PDSFLG_IN_CPU_PRQ_MAX	\
+					 | ERTS_PDSFLG_IN_CPU_PRQ_HIGH	\
+					 | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\
+					 | ERTS_PDSFLG_IN_CPU_PRQ_LOW)
+#define ERTS_PDSFLG_IN_IO_PRQ_MASK 	(ERTS_PDSFLG_IN_CPU_PRQ_MAX	\
+					 | ERTS_PDSFLG_IN_CPU_PRQ_HIGH	\
+					 | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\
+					 | ERTS_PDSFLG_IN_CPU_PRQ_LOW)
+#endif
+
 
 /*
  * Static flags that do not change after process creation.
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 25f0b1ed38..598c1d66ba 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -617,7 +617,7 @@ erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) {
     if (psflg)
         erts_print(to, to_arg, " | ");
 
-    for (i = 0; i < ERTS_PSFLG_MAX && psflg; i++) {
+    for (i = 0; i <= ERTS_PSFLG_MAX && psflg; i++) {
         erts_aint32_t chk = (1 << i);
         if (psflg & chk) {
             switch (chk) {
@@ -657,16 +657,12 @@ erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) {
                 erts_print(to, to_arg, "PROXY"); break;
             case ERTS_PSFLG_DELAYED_SYS:
                 erts_print(to, to_arg, "DELAYED_SYS"); break;
-#ifdef ERTS_DIRTY_SCHEDULERS
             case ERTS_PSFLG_DIRTY_CPU_PROC:
                 erts_print(to, to_arg, "DIRTY_CPU_PROC"); break;
             case ERTS_PSFLG_DIRTY_IO_PROC:
                 erts_print(to, to_arg, "DIRTY_IO_PROC"); break;
-            case ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q:
-                erts_print(to, to_arg, "DIRTY_CPU_PROC_IN_Q"); break;
-            case ERTS_PSFLG_DIRTY_IO_PROC_IN_Q:
-                erts_print(to, to_arg, "DIRTY_IO_PROC_IN_Q"); break;
-#endif
+            case ERTS_PSFLG_DIRTY_ACTIVE_SYS:
+                erts_print(to, to_arg, "DIRTY_ACTIVE_SYS"); break;
             default:
                 erts_print(to, to_arg, "UNKNOWN(%d)", chk); break;
             }
-- 
cgit v1.2.3


From bc1e16e334afc8255b221e3ffbb6f7c54fb91a2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@plataformatec.com.br>
Date: Mon, 1 Feb 2016 10:50:32 +0100
Subject: Avoid erts_cmp jump in atom, int and float comparisons

Given the function definition below:

    check(X) when X >= 0, X <= 20 -> true.

@nox has originally noticed that perfoming lt and ge
guard tests were performing slower than they should be.

Further investigation revealed that most of the cost
was in jumping to the erts_cmp function. This patch
brings the operations already inlined in erts_cmp
into the emulator, removing the jump cost.

After applying these changes, invoking the check/1
function defined above 30000 times with different
values from 0 to 20 has fallen from 367us to 213us
(measured as average of 3 runs). This is a
considerably improvement over Erlang 18 which takes
556us on average.

Floats have also dropped their time from 1126us
(on Erlang 18) to 613us.
---
 erts/emulator/beam/atom.c      |  2 +-
 erts/emulator/beam/beam_emu.c  |  8 ++++----
 erts/emulator/beam/erl_utils.h | 34 ++++++++++++++++++++++++----------
 erts/emulator/beam/utils.c     | 18 +++++++++---------
 4 files changed, 38 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index fd2adac676..099c00bcf6 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -176,7 +176,7 @@ atom_alloc(Atom* tmpl)
 
     /*
      * Precompute ordinal value of first 3 bytes + 7 bits.
-     * This is used by utils.c:cmp_atoms().
+     * This is used by utils.c:erts_cmp_atoms().
      * We cannot use the full 32 bits of the first 4 bytes,
      * since we use the sign of the difference between two
      * ordinal values to represent their relative order.
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4d7b00b032..659ff3ad2f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -718,10 +718,10 @@ void** beam_ops;
 #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
 #define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; }
 #define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; }
-#define Equal(X, Y, Action) if (!CMP_EQ(X,Y)) { Action; }
-#define NotEqual(X, Y, Action) if (!CMP_NE(X,Y)) { Action; }
-#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; }
-#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; }
+#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action)
+#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action)
+#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action)
+#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action)
 
 #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
 
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 4058d63eaf..b8746f7edb 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -161,7 +161,9 @@ int eq(Eterm, Eterm);
 
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
+int erts_cmp_atoms(Eterm a, Eterm b);
 Sint erts_cmp(Eterm, Eterm, int, int);
+Sint erts_cmp_compound(Eterm, Eterm, int, int);
 Sint cmp(Eterm a, Eterm b);
 #define CMP(A,B)                         erts_cmp(A,B,0,0)
 #define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
@@ -174,17 +176,29 @@ Sint cmp(Eterm a, Eterm b);
 #define cmp_ge(a,b)          (CMP((a),(b)) >= 0)
 #define cmp_gt(a,b)          (CMP((a),(b)) >  0)
 
-#define cmp_lt_term(a,b)     (CMP_TERM((a),(b)) <  0)
-#define cmp_le_term(a,b)     (CMP_TERM((a),(b)) <= 0)
-#define cmp_ge_term(a,b)     (CMP_TERM((a),(b)) >= 0)
-#define cmp_gt_term(a,b)     (CMP_TERM((a),(b)) >  0)
-
-#define CMP_LT(a,b)          ((a) != (b) && cmp_lt((a),(b)))
-#define CMP_GE(a,b)          ((a) == (b) || cmp_ge((a),(b)))
 #define CMP_EQ(a,b)          ((a) == (b) || cmp_eq((a),(b)))
-#define CMP_NE(a,b)          ((a) != (b) && cmp_ne((a),(b)))
 
-#define CMP_LT_TERM(a,b)     ((a) != (b) && cmp_lt_term((a),(b)))
-#define CMP_GE_TERM(a,b)     ((a) == (b) || cmp_ge_term((a),(b)))
+#define CMP_EQ_ACTION(X,Y,Action)	\
+    if ((X) != (Y)) { CMP_SPEC((X),(Y),!=,Action,1); }
+#define CMP_NE_ACTION(X,Y,Action)	\
+    if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),==,Action,1); }
+#define CMP_GE_ACTION(X,Y,Action)	\
+    if ((X) != (Y)) { CMP_SPEC((X),(Y),<,Action,0); }
+#define CMP_LT_ACTION(X,Y,Action)	\
+    if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),>=,Action,0); }
+
+#define CMP_SPEC(X,Y,Op,Action,EqOnly)				\
+    if (is_atom(X) && is_atom(Y)) {				\
+	if (erts_cmp_atoms(X, Y) Op 0) { Action; };		\
+    } else if (is_both_small(X, Y)) {				\
+	if (signed_val(X) Op signed_val(Y)) { Action; };	\
+    } else if (is_float(X) && is_float(Y)) {			\
+        FloatDef af, bf;					\
+        GET_DOUBLE(X, af);					\
+        GET_DOUBLE(Y, bf);					\
+        if (af.fd Op bf.fd) { Action; };			\
+    } else {							\
+	if (erts_cmp_compound(X,Y,0,EqOnly) Op 0) { Action; };	\
+    }
 
 #endif
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ef851d840d..6aad1ff778 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2991,7 +2991,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2)
 
 #define float_comp(x,y)    (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1))
 
-static int cmp_atoms(Eterm a, Eterm b)
+int erts_cmp_atoms(Eterm a, Eterm b)
 {
     Atom *aa = atom_tab(atom_val(a));
     Atom *bb = atom_tab(atom_val(b));
@@ -3010,12 +3010,12 @@ Sint cmp(Eterm a, Eterm b)
     return erts_cmp(a, b, 0, 0);
 }
 
-static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
+Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
 
 Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
 {
     if (is_atom(a) && is_atom(b)) {
-        return cmp_atoms(a, b);
+        return erts_cmp_atoms(a, b);
     } else if (is_both_small(a, b)) {
         return (signed_val(a) - signed_val(b));
     } else if (is_float(a) && is_float(b)) {
@@ -3032,7 +3032,7 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
  * exact = 1 -> term-based compare
  * exact = 0 -> arith-based compare
  */
-static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
+Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
 {
 #define PSTACK_TYPE struct erts_cmp_hashmap_state
     struct erts_cmp_hashmap_state {
@@ -3089,7 +3089,7 @@ static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
     do {								\
 	if((AN) != (BN)) {						\
             if((AN)->sysname != (BN)->sysname)				\
-                RETURN_NEQ(cmp_atoms((AN)->sysname, (BN)->sysname));	\
+                RETURN_NEQ(erts_cmp_atoms((AN)->sysname, (BN)->sysname));	\
 	    ASSERT((AN)->creation != (BN)->creation);			\
 	    RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1);	\
 	}								\
@@ -3107,7 +3107,7 @@ tailrecur_ne:
     /* deal with majority (?) cases by brute-force */
     if (is_atom(a)) {
 	if (is_atom(b)) {
-	    ON_CMP_GOTO(cmp_atoms(a, b));
+	    ON_CMP_GOTO(erts_cmp_atoms(a, b));
 	}
     } else if (is_both_small(a, b)) {
 	ON_CMP_GOTO(signed_val(a) - signed_val(b));
@@ -3341,10 +3341,10 @@ tailrecur_ne:
 		    Export* a_exp = *((Export **) (export_val(a) + 1));
 		    Export* b_exp = *((Export **) (export_val(b) + 1));
 
-		    if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
+		    if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
 			RETURN_NEQ(j);
 		    }
-		    if ((j = cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) {
+		    if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) {
 			RETURN_NEQ(j);
 		    }
 		    ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]);
@@ -3659,7 +3659,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
 	b = *bb++;
 	if (!is_same(a, b)) {
 	    if (is_atom(a) && is_atom(b)) {
-		if ((j = cmp_atoms(a, b)) != 0) {
+		if ((j = erts_cmp_atoms(a, b)) != 0) {
 		    goto not_equal;
 		}
 	    } else if (is_both_small(a, b)) {
-- 
cgit v1.2.3


From 345651148351c57b26347ee9a9b50e6e2732e79a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@plataformatec.com.br>
Date: Mon, 15 Feb 2016 11:00:26 +0100
Subject: Unify comparison macros in erl_utils.h

This removes the duplication in having both `cmp_eq`
and `CMP_EQ` and normalizes their name to uppercase.
---
 erts/emulator/beam/erl_bif_op.c | 12 ++++++------
 erts/emulator/beam/erl_utils.h  | 14 ++++++--------
 2 files changed, 12 insertions(+), 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index d53a9e11ca..0f20ded1d6 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -89,22 +89,22 @@ BIF_RETTYPE not_1(BIF_ALIST_1)
 
 BIF_RETTYPE sgt_2(BIF_ALIST_2)
 {
-    BIF_RET(cmp_gt(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    BIF_RET(CMP_GT(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
 }
 
 BIF_RETTYPE sge_2(BIF_ALIST_2)
 {
-    BIF_RET(cmp_ge(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    BIF_RET(CMP_GE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
 }
 
 BIF_RETTYPE slt_2(BIF_ALIST_2)
 {
-    BIF_RET(cmp_lt(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    BIF_RET(CMP_LT(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
 }
 
 BIF_RETTYPE sle_2(BIF_ALIST_2)
 {
-    BIF_RET(cmp_le(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    BIF_RET(CMP_LE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
 }
 
 BIF_RETTYPE seq_2(BIF_ALIST_2)
@@ -114,7 +114,7 @@ BIF_RETTYPE seq_2(BIF_ALIST_2)
 
 BIF_RETTYPE seqeq_2(BIF_ALIST_2)
 {
-    BIF_RET(cmp_eq(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    BIF_RET(CMP_EQ(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
 }
 
 BIF_RETTYPE sneq_2(BIF_ALIST_2)
@@ -124,7 +124,7 @@ BIF_RETTYPE sneq_2(BIF_ALIST_2)
 
 BIF_RETTYPE sneqeq_2(BIF_ALIST_2)
 {
-    BIF_RET(cmp_ne(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
+    BIF_RET(CMP_NE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false);
 }
 
 BIF_RETTYPE is_atom_1(BIF_ALIST_1)
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index b8746f7edb..dd7b925bd3 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -169,14 +169,12 @@ Sint cmp(Eterm a, Eterm b);
 #define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
 #define CMP_EQ_ONLY(A,B)                 erts_cmp(A,B,0,1)
 
-#define cmp_lt(a,b)          (CMP((a),(b)) <  0)
-#define cmp_le(a,b)          (CMP((a),(b)) <= 0)
-#define cmp_eq(a,b)          (CMP_EQ_ONLY((a),(b)) == 0)
-#define cmp_ne(a,b)          (CMP_EQ_ONLY((a),(b)) != 0)
-#define cmp_ge(a,b)          (CMP((a),(b)) >= 0)
-#define cmp_gt(a,b)          (CMP((a),(b)) >  0)
-
-#define CMP_EQ(a,b)          ((a) == (b) || cmp_eq((a),(b)))
+#define CMP_LT(a,b)          ((a) != (b) && CMP((a),(b)) <  0)
+#define CMP_LE(a,b)          ((a) == (b) || CMP((a),(b)) <= 0)
+#define CMP_EQ(a,b)          ((a) == (b) || CMP_EQ_ONLY((a),(b)) == 0)
+#define CMP_NE(a,b)          ((a) != (b) && CMP_EQ_ONLY((a),(b)) != 0)
+#define CMP_GE(a,b)          ((a) == (b) || CMP((a),(b)) >= 0)
+#define CMP_GT(a,b)          ((a) != (b) && CMP((a),(b)) >  0)
 
 #define CMP_EQ_ACTION(X,Y,Action)	\
     if ((X) != (Y)) { CMP_SPEC((X),(Y),!=,Action,1); }
-- 
cgit v1.2.3


From 4ea67ffdac2629255b1b0ed4e9423823f62c0947 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 15 Feb 2016 18:12:48 +0100
Subject: erts: Add BIF erts_internal:system_check/1

This commit implements erts_internal:system_check(schedulers) with the
intent of a basic responsiveness test check of the schedulers.
---
 erts/emulator/beam/atom.names      |  1 +
 erts/emulator/beam/bif.tab         |  2 +
 erts/emulator/beam/erl_alloc.types |  2 +
 erts/emulator/beam/erl_bif_info.c  | 15 ++++++
 erts/emulator/beam/erl_process.c   | 97 ++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_process.h   |  1 +
 6 files changed, 118 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 07f6492948..cb6d294a41 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -269,6 +269,7 @@ atom getenv
 atom gather_gc_info_result
 atom gather_io_bytes
 atom gather_sched_wall_time_result
+atom gather_system_check_result
 atom getting_linked
 atom getting_unlinked
 atom global
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 4f0656d174..3177d5dae7 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -174,6 +174,8 @@ bif erts_internal:time_unit/0
 
 bif erts_internal:is_system_process/1
 
+bif erts_internal:system_check/1
+
 # inet_db support
 bif erlang:port_set_data/2
 bif erlang:port_get_data/1
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 1ecebdeb07..7738531142 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -369,6 +369,7 @@ type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	STANDARD_LOW	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD_LOW	SYSTEM		port_data_heap
+type	SYS_CHECK_REQ	STANDARD_LOW	SYSTEM		system_check_request
 
 +else # "fullword"
 
@@ -389,6 +390,7 @@ type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
+type	SYS_CHECK_REQ	SHORT_LIVED	SYSTEM		system_check_request
 
 +endif
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 414ff6711a..8bf6877bea 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -65,6 +65,7 @@ static Export* gather_io_bytes_trap = NULL;
 
 static Export *gather_sched_wall_time_res_trap;
 static Export *gather_gc_info_res_trap;
+static Export *gather_system_check_res_trap;
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
@@ -3784,6 +3785,18 @@ BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1)
+{
+    Eterm res;
+    if (ERTS_IS_ATOM_STR("schedulers", BIF_ARG_1)) {
+	res = erts_system_check_request(BIF_P);
+	if (is_non_value(res))
+	    BIF_RET(am_undefined);
+	BIF_TRAP1(gather_system_check_res_trap, BIF_P, res);
+    }
+
+    BIF_ERROR(BIF_P, BADARG);
+}
 
 static erts_smp_atomic_t hipe_test_reschedule_flag;
 
@@ -4391,6 +4404,8 @@ erts_bif_info_init(void)
 	= erts_export_put(am_erlang, am_gather_gc_info_result, 1);
     gather_io_bytes_trap
 	= erts_export_put(am_erts_internal, am_gather_io_bytes, 2);
+    gather_system_check_res_trap
+	= erts_export_put(am_erts_internal, am_gather_system_check_result, 1);
     process_info_init();
     os_info_init();
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b24f26138c..8e4a42fbe4 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -967,11 +967,25 @@ typedef struct {
     erts_smp_atomic32_t refc;
 } ErtsSchedWallTimeReq;
 
+typedef struct {
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    Uint req_sched;
+    erts_smp_atomic32_t refc;
+} ErtsSystemCheckReq;
+
+
 #if !HALFWORD_HEAP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq,
 				 ErtsSchedWallTimeReq,
 				 5,
 				 ERTS_ALC_T_SCHED_WTIME_REQ)
+
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq,
+				 ErtsSystemCheckReq,
+				 5,
+				 ERTS_ALC_T_SYS_CHECK_REQ)
 #else
 static ERTS_INLINE ErtsSchedWallTimeReq *
 swtreq_alloc(void)
@@ -985,6 +999,19 @@ swtreq_free(ErtsSchedWallTimeReq *ptr)
 {
     erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr);
 }
+
+static ERTS_INLINE ErtsSystemCheckReq *
+screq_alloc(void)
+{
+    return erts_alloc(ERTS_ALC_T_SYS_CHECK_REQ,
+		      sizeof(ErtsSystemCheckReq));
+}
+
+static ERTS_INLINE void
+screq_free(ErtsSystemCheckReq *ptr)
+{
+    erts_free(ERTS_ALC_T_SYS_CHECK_REQ, ptr);
+}
 #endif
 
 static void
@@ -1118,6 +1145,75 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable)
     return ref;
 }
 
+static void
+reply_system_check(void *vscrp)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsSystemCheckReq *scrp = (ErtsSystemCheckReq *) vscrp;
+    ErtsProcLocks rp_locks = (scrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0);
+    Process *rp = scrp->proc;
+    Eterm msg;
+    Eterm *hp = NULL;
+    Eterm **hpp;
+    Uint sz;
+    ErlOffHeap *ohp = NULL;
+    ErlHeapFragment *bp = NULL;
+
+    ASSERT(esdp);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+#endif
+
+    sz = REF_THING_SIZE;
+    hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+    hpp = &hp;
+    msg = STORE_NC(hpp, ohp, scrp->ref);
+
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+
+    if (scrp->req_sched == esdp->no)
+	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+
+    if (rp_locks)
+	erts_smp_proc_unlock(rp, rp_locks);
+
+    erts_proc_dec_refc(rp);
+
+    if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0)
+	screq_free(vscrp);
+}
+
+
+Eterm erts_system_check_request(Process *c_p) {
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    Eterm ref;
+    ErtsSystemCheckReq *scrp;
+    Eterm *hp;
+
+    scrp = screq_alloc();
+    ref = erts_make_ref(c_p);
+    hp = &scrp->ref_heap[0];
+
+    scrp->proc = c_p;
+    scrp->ref = STORE_NC(&hp, NULL, ref);
+    scrp->req_sched = esdp->no;
+    erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers);
+
+    erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+					  erts_no_schedulers,
+					  reply_system_check,
+					  (void *) scrp);
+#endif
+
+    reply_system_check((void *) scrp);
+
+    return ref;
+}
+
 static ERTS_INLINE ErtsProcList *
 proclist_create(Process *p)
 {
@@ -5792,6 +5888,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     init_misc_aux_work();
 #if !HALFWORD_HEAP
     init_swtreq_alloc();
+    init_screq_alloc();
 #endif
 
     erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 799e49005c..f1570a835c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1483,6 +1483,7 @@ void erts_init_scheduling(int, int
 
 int erts_set_gc_state(Process *c_p, int enable);
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+Eterm erts_system_check_request(Process *c_p);
 Eterm erts_gc_info_request(Process *c_p);
 Uint64 erts_get_proc_interval(void);
 Uint64 erts_ensure_later_proc_interval(Uint64);
-- 
cgit v1.2.3


From 184a4d38db21cdc3011a5c906f38c880c1ffdc58 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 18 Feb 2016 19:44:24 +0100
Subject: erts: Fix lock checker for process locks

Do lock order check *before* trying to seize lock... duh!
---
 erts/emulator/beam/erl_process_lock.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index a64c993e8f..9c59301086 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -523,6 +523,10 @@ erts_smp_proc_lock__(Process *p,
 
     ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);
 
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_proc_lc_lock(p, locks, file, line);
+#endif
+
     old_lflgs = erts_smp_proc_raw_trylock__(p, locks);
 
     if (old_lflgs != 0) {
@@ -544,9 +548,6 @@ erts_smp_proc_lock__(Process *p,
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line);
 #endif
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_proc_lc_lock(p, locks, file, line);
-#endif
 
 #ifdef ERTS_PROC_LOCK_DEBUG
     erts_proc_lock_op_debug(p, locks, 1);
-- 
cgit v1.2.3


From e1be12434b06fb2594af5cdafc5efc5b9182d8b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 30 Nov 2015 15:28:37 +0100
Subject: beam_load.c: Add a function to check for an on_load function

We will need a way to check whether an prepared BEAM modules has
an on_load function.
---
 erts/emulator/beam/beam_load.c | 17 +++++++++++++++++
 erts/emulator/beam/global.h    |  1 +
 2 files changed, 18 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 1511a4f935..f115df935f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -888,6 +888,23 @@ erts_module_for_prepared_code(Binary* magic)
     }
 }
 
+/*
+ * Return a non-zero value if the module has an on_load function,
+ * or 0 if it does not.
+ */
+
+Eterm
+erts_has_code_on_load(Binary* magic)
+{
+    LoaderState* stp;
+
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) {
+	return NIL;
+    }
+    stp = ERTS_MAGIC_BIN_DATA(magic);
+    return stp->on_load ? am_true : am_false;
+}
+
 static void
 free_loader_state(Binary* magic)
 {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2a8bdb6ee3..b174fa3e8a 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1008,6 +1008,7 @@ typedef struct {
 
 Binary* erts_alloc_loader_state(void);
 Eterm erts_module_for_prepared_code(Binary* magic);
+Eterm erts_has_code_on_load(Binary* magic);
 Eterm erts_prepare_loading(Binary* loader_state,  Process *c_p,
 			   Eterm group_leader, Eterm* modp,
 			   byte* code, Uint size);
-- 
cgit v1.2.3


From 9f60b6ea108b625affe5aef745443c8422c8d64f Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Sun, 21 Feb 2016 20:32:40 -0500
Subject: Skip run queue lock check for dirty schedulers

The wake_scheduler function asserts that the run queue is not locked,
but this assertion sometimes fails for dirty schedulers (in January
2016 a user in erlang-questions reported a dirty schedulers problem
related to this). After discussing it with Rickard, we decided
modifying the assertion was the most practical way to address the
problem.
---
 erts/emulator/beam/erl_process.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b7499c5b5a..fd29c22810 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -3250,7 +3250,8 @@ wake_scheduler(ErtsRunQueue *rq)
      * so all code *should* handle this without having
      * the lock on the run queue.
      */
-    ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq));
+    ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)
+		       || ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
     ssi = rq->scheduler->ssi;
 
-- 
cgit v1.2.3


From 6cec93eb4e697fc1451b0b4729fc3d4874581ebd Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 23 Feb 2016 11:24:12 +0100
Subject: erts: Fix install of suspend handler

This commit makes sure to setup the suspend handler
to matter what +B option is given at the command line.
---
 erts/emulator/beam/break.c    | 4 ++--
 erts/emulator/beam/erl_init.c | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 4ce9d24479..8647b621d5 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -684,7 +684,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
        crash dump. */
     erts_thr_progress_fatal_error_block(&tpd_buf);
 
-#ifdef ERTS_THR_HAVE_SIG_FUNCS
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
     /*
      * We suspend all scheduler threads so that we can dump some
      * data about the currently running processes and scheduler data.
@@ -818,7 +818,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 
 #ifdef ERTS_SMP
 
-#if defined(ERTS_THR_HAVE_SIG_FUNCS)
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
 
     /* We resume all schedulers so that we are in a known safe state
        when we write the rest of the crash dump */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index d9c3b0dcf4..bcc54491bd 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2134,6 +2134,7 @@ erl_start(int argc, char **argv)
 	init_break_handler();
     if (replace_intr)
 	erts_replace_intr();
+    sys_init_suspend_handler();
 #endif
 
     boot_argc = argc - i;  /* Number of arguments to init */
-- 
cgit v1.2.3


From db241c69cef8774b9b7afa7e0f0f8dbdcf528a07 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 17 Feb 2016 21:17:38 +0100
Subject: erts: Make literal_alloc documented and configurable

Except it cannot be disabled and cannot be multi-threaded.

The bit-vector 'erts_literal_vspace_map' on 32-bit is currently only
protected by the literal allocator mutex. We could allow multiple
instances on 64-bit (I think), but what would be the point?
---
 erts/emulator/beam/erl_alloc.c | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 806f569c38..14ff13fdb2 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -165,6 +165,8 @@ enum allctr_type {
 struct au_init {
     int enable;
     int thr_spec;
+    int disable_allowed;
+    int thr_spec_allowed;
     int carrier_migration_allowed;
     enum allctr_type	atype;
     struct {
@@ -217,7 +219,7 @@ typedef struct {
     struct au_init test_alloc;
 } erts_alc_hndl_args_init_t;
 
-#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
+#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
 
 #define SET_DEFAULT_ALLOC_OPTS(IP)					\
 do {									\
@@ -294,6 +296,9 @@ set_default_literal_alloc_opts(struct au_init *ip)
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= 1;
     ip->thr_spec		= 0;
+    ip->disable_allowed         = 0;
+    ip->thr_spec_allowed        = 0;
+    ip->carrier_migration_allowed = 0;
     ip->atype			= BESTFIT;
     ip->init.bf.ao		= 1;
     ip->init.util.ramv		= 0;
@@ -1352,9 +1357,17 @@ handle_au_arg(struct au_init *auip,
 	else
 	    goto bad_switch;
 	break;
-    case 'e':
-	auip->enable = get_bool_value(sub_param+1, argv, ip);
+    case 'e': {
+	int e = get_bool_value(sub_param + 1, argv, ip);
+        if (!auip->disable_allowed && !e) {
+            if (!u_switch)
+                bad_value(param, sub_param + 1, "false");
+	    else
+		ASSERT(auip->enable); /* ignore */
+        }
+	else auip->enable = e;
 	break;
+    }
     case 'l':
 	if (has_prefix("lmbcs", sub_param)) {
 	    auip->default_.lmbcs = 0;
@@ -1423,7 +1436,14 @@ handle_au_arg(struct au_init *auip,
     case 't': {
 	int res = get_bool_value(sub_param+1, argv, ip);
 	if (res > 0) {
-	    auip->thr_spec = 1;
+	    if (!auip->thr_spec_allowed) {
+		if (!u_switch)
+                    bad_value(param, sub_param + 1, "true");
+		else
+		    ASSERT(!auip->thr_spec); /* ignore */
+	    }
+	    else
+		auip->thr_spec = 1;
 	    break;
 	}
 	else if (res == 0) {
@@ -1472,6 +1492,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		case 'B':
 		    handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
 		    break;
+		case 'I':
+                    handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
+		    break;
 		case 'D':
 		    handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
 		    break;
-- 
cgit v1.2.3


From abe5967c1964a4ca93f321c6cd564c8650f11a53 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 19 Feb 2016 15:45:18 +0100
Subject: erts: Refactor init of erts_literal_mmapper

---
 erts/emulator/beam/erl_alloc.c | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 14ff13fdb2..7c880342bd 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -774,9 +774,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #if HAVE_ERTS_MSEG
     init.mseg.nos = erts_no_schedulers;
     erts_mseg_init(&init.mseg);
-# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-    erts_mmap_init(&erts_literal_mmapper, &init.mseg.literal_mmap);
-# endif
 #endif
 
     erts_alcu_init(&init.alloc_util);
-- 
cgit v1.2.3


From 8e2a21f1df1140867d0b074ec7a86610d1e1b51e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 22 Feb 2016 18:18:15 +0100
Subject: erts: Add emulator flag +MIscs for literal super carrier size

---
 erts/emulator/beam/erl_alloc.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 7c880342bd..a266ea6d19 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1489,8 +1489,15 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		case 'B':
 		    handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
 		    break;
-		case 'I':
-                    handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
+                case 'I':
+                    if (has_prefix("scs", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+			init->mseg.literal_mmap.scs =
+#endif
+			    get_mb_value(argv[i]+6, argv, &i);
+		    }
+                    else
+                        handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
 		    break;
 		case 'D':
 		    handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
-- 
cgit v1.2.3


From a5e66520d92d283d14f1352dcfd0ce889c27a0c5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 23 Feb 2016 18:31:52 +0100
Subject: Do not wait for main lock when looking up process not running

---
 erts/emulator/beam/erl_process.c | 102 +++++++++++++++++++++++++++------------
 1 file changed, 71 insertions(+), 31 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b24f26138c..adea745471 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6462,9 +6462,6 @@ suspend_process(Process *c_p, Process *p)
 
     if (suspended) {
 
-	ASSERT(!(ERTS_PSFLG_RUNNING & state)
-	       || p == erts_get_current_process());
-
 	if (suspended > 0 && erts_system_profile_flags.runnable_procs) {
 
 	    /* 'state' is before our change... */
@@ -8411,9 +8408,8 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	    resume_process(rp, rp_locks);
     }
     else {
-
 	rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
-			   pid, pid_locks|ERTS_PROC_LOCK_STATUS);
+			   pid, ERTS_PROC_LOCK_STATUS);
 
 	if (!rp) {
 	    c_p->flags &= ~F_P2PNR_RESCHED;
@@ -8422,40 +8418,84 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 
 	ASSERT(!(c_p->flags & F_P2PNR_RESCHED));
 
-	if (suspend) {
-	    if (suspend_process(c_p, rp))
-		goto done;
-	}
-	else {
-	    if (!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
-		  & erts_smp_atomic32_read_acqb(&rp->state)))
-		goto done;
+	/*
+	 * Suspend the other process in order to prevent
+	 * it from being selected for normal execution.
+	 * This will however not prevent it from being
+	 * selected for execution of a system task. If
+	 * it is selected for execution of a system task
+	 * we might be blocked for quite a while if the
+	 * try-lock below fails. That is, there is room
+	 * for improvement here...
+	 */
 
-	}
+	if (!suspend_process(c_p, rp)) {
+	    /* Other process running */
 
-	/* Other process running */
+	    ASSERT(ERTS_PSFLG_RUNNING
+		   & erts_smp_atomic32_read_nob(&rp->state));
 
-	/*
-	 * If we got pending suspenders and suspend ourselves waiting
-	 * to suspend another process we might deadlock.
-	 * In this case we have to yield, be suspended by
-	 * someone else and then do it all over again.
-	 */
-	if (!c_p->pending_suspenders) {
-	    /* Mark rp pending for suspend by c_p */
-	    add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend);
-	    ASSERT(is_nil(c_p->suspendee));
+	running:
 
-	    /* Suspend c_p; when rp is suspended c_p will be resumed. */
-	    suspend_process(c_p, c_p);
-	    c_p->flags |= F_P2PNR_RESCHED;
+	    /*
+	     * If we got pending suspenders and suspend ourselves waiting
+	     * to suspend another process we might deadlock.
+	     * In this case we have to yield, be suspended by
+	     * someone else and then do it all over again.
+	     */
+	    if (!c_p->pending_suspenders) {
+		/* Mark rp pending for suspend by c_p */
+		add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend);
+		ASSERT(is_nil(c_p->suspendee));
+
+		/* Suspend c_p; when rp is suspended c_p will be resumed. */
+		suspend_process(c_p, c_p);
+		c_p->flags |= F_P2PNR_RESCHED;
+	    }
+	    /* Yield (caller is assumed to yield immediately in bif). */
+	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+	    rp = ERTS_PROC_LOCK_BUSY;
+	}
+	else {
+	    ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
+	    if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
+		if (ERTS_PSFLG_RUNNING_SYS
+		    & erts_smp_atomic32_read_nob(&rp->state)) {
+		    /* Executing system task... */
+		    resume_process(rp, ERTS_PROC_LOCK_STATUS);
+		    goto running;
+		}
+		erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+		/*
+		 * If we are unlucky, the process just got selected for
+		 * execution of a system task. In this case we may be
+		 * blocked here for quite a while... Execution of system
+		 * tasks are fortunately quite rare events. We try to
+		 * avoid this by checking if it is in a state executing
+		 * system tasks (above), but it will not prevent all
+		 * scenarios for a long block here...
+		 */
+		rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
+				   pid, pid_locks|ERTS_PROC_LOCK_STATUS);
+		if (!rp)
+		    goto done;
+	    }
+
+	    /*
+	     * The previous suspend has prevented the process
+	     * from being selected for normal execution regardless
+	     * of locks held or not held on it...
+	     */
+	    ASSERT(!(ERTS_PSFLG_RUNNING
+		     & erts_smp_atomic32_read_nob(&rp->state)));
+
+	    if (!suspend)
+		resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
 	}
-	/* Yield (caller is assumed to yield immediately in bif). */
-	erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
-	rp = ERTS_PROC_LOCK_BUSY;
     }
 
  done:
+	
     if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS))
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
     if (unlock_c_p_status)
-- 
cgit v1.2.3


From 56090db3ea417157a749bdd810fc61d117493f1f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 23 Feb 2016 19:45:01 +0100
Subject: erts: Change erl_exit into erts_exit

This is mostly a pure refactoring.

Except for the buggy cases when calling erlang:halt() with a positive
integer in the range -(INT_MIN+2) to -INT_MIN that got confused with
ERTS_ABORT_EXIT, ERTS_DUMP_EXIT and ERTS_INTR_EXIT.

Outcome             OLD erl_exit(n, )         NEW erts_exit(n, )
-------             -------------------       -------------------------------------------
exit(Status)        n = -Status <= 0          n = Status >= 0
crashdump+abort     n > 0, ignore n           n = ERTS_ERROR_EXIT < 0

The outcome of the old ERTS_ABORT_EXIT, ERTS_INTR_EXIT and
ERTS_DUMP_EXIT are the same as before (even though their values have
changed).
---
 erts/emulator/beam/beam_catches.c          |  6 ++---
 erts/emulator/beam/beam_emu.c              | 14 +++++-----
 erts/emulator/beam/bif.c                   | 34 ++++++++++++------------
 erts/emulator/beam/binary.c                |  2 +-
 erts/emulator/beam/break.c                 | 12 ++++-----
 erts/emulator/beam/copy.c                  | 20 +++++++-------
 erts/emulator/beam/dist.c                  | 10 +++----
 erts/emulator/beam/erl_afit_alloc.c        |  2 +-
 erts/emulator/beam/erl_alloc.c             | 38 +++++++++++++--------------
 erts/emulator/beam/erl_alloc_util.c        |  8 +++---
 erts/emulator/beam/erl_ao_firstfit_alloc.c |  2 +-
 erts/emulator/beam/erl_bestfit_alloc.c     |  2 +-
 erts/emulator/beam/erl_bif_ddll.c          |  2 +-
 erts/emulator/beam/erl_bif_info.c          | 12 ++++-----
 erts/emulator/beam/erl_bif_port.c          |  4 +--
 erts/emulator/beam/erl_cpu_topology.c      | 10 +++----
 erts/emulator/beam/erl_db.c                | 14 +++++-----
 erts/emulator/beam/erl_db_hash.c           |  2 +-
 erts/emulator/beam/erl_db_tree.c           |  2 +-
 erts/emulator/beam/erl_db_util.c           | 18 ++++++-------
 erts/emulator/beam/erl_debug.c             | 14 +++++-----
 erts/emulator/beam/erl_drv_thread.c        |  2 +-
 erts/emulator/beam/erl_fun.c               |  2 +-
 erts/emulator/beam/erl_gc.c                |  6 ++---
 erts/emulator/beam/erl_goodfit_alloc.c     |  4 +--
 erts/emulator/beam/erl_init.c              | 36 +++++++++++--------------
 erts/emulator/beam/erl_map.c               | 32 +++++++++++------------
 erts/emulator/beam/erl_monitors.c          |  2 +-
 erts/emulator/beam/erl_nif.c               |  2 +-
 erts/emulator/beam/erl_node_tables.c       |  4 +--
 erts/emulator/beam/erl_port_task.c         |  4 +--
 erts/emulator/beam/erl_process.c           | 42 +++++++++++++++---------------
 erts/emulator/beam/erl_process.h           |  2 +-
 erts/emulator/beam/erl_process_dict.c      |  8 +++---
 erts/emulator/beam/erl_ptab.c              |  2 +-
 erts/emulator/beam/erl_term.c              |  2 +-
 erts/emulator/beam/erl_thr_progress.c      | 12 ++++-----
 erts/emulator/beam/erl_thr_queue.c         |  6 ++---
 erts/emulator/beam/erl_time_sup.c          |  6 ++---
 erts/emulator/beam/erl_trace.c             |  4 +--
 erts/emulator/beam/erl_unicode.c           |  2 +-
 erts/emulator/beam/external.c              | 18 ++++++-------
 erts/emulator/beam/global.h                | 12 ++++-----
 erts/emulator/beam/hash.c                  |  2 +-
 erts/emulator/beam/index.c                 |  4 +--
 erts/emulator/beam/io.c                    |  6 ++---
 erts/emulator/beam/register.c              |  2 +-
 erts/emulator/beam/sys.h                   | 25 +++++++++---------
 erts/emulator/beam/time.c                  |  2 +-
 erts/emulator/beam/utils.c                 | 20 +++++++-------
 50 files changed, 248 insertions(+), 251 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c
index c1fd17c65d..7a1f4901aa 100644
--- a/erts/emulator/beam/beam_catches.c
+++ b/erts/emulator/beam/beam_catches.c
@@ -143,7 +143,7 @@ BeamInstr *beam_catches_car(unsigned i)
     struct bc_pool* p = &bccix[erts_active_code_ix()];
 
     if (i >= p->tabsize ) {
-	erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
+	erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i);
     }
     return p->beam_catches[i].cp;
 }
@@ -157,10 +157,10 @@ void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes,
     ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging);
     for(i = head; i != (unsigned)-1;) {
 	if (i >= p->tabsize) {
-	    erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
+	    erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i);
 	}
 	if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) {
-	    erl_exit(1,
+	    erts_exit(ERTS_ERROR_EXIT,
 		    "beam_catches_delmod: item %#x has cp %p which is not "
 		    "in module's range [%p,%p[\r\n",
 		    i, p->beam_catches[i].cp, code, ((char*)code + code_bytes));
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 38def5d89f..da201d9b36 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5018,7 +5018,7 @@ do {								\
 #ifdef NO_FPE_SIGNALS
      OpCase(fclearerror):
      OpCase(i_fcheckerror):
-	 erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
+	 erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
 #  define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT
 #  define ERTS_NO_FPE_ERROR ERTS_FP_ERROR
 #else
@@ -5176,7 +5176,7 @@ do {								\
 	 I = handle_error(c_p, I, reg, NULL);
 	 goto post_error_handling;
        default:
-	 erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
+	 erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
      }
  }
  OpCase(hipe_call_count): {
@@ -5256,7 +5256,7 @@ do {								\
  OpCase(label_L):
  OpCase(on_load):
  OpCase(line_I):
-    erl_exit(1, "meta op\n");
+    erts_exit(ERTS_ERROR_EXIT, "meta op\n");
 
     /*
      * One-time initialization of Beam emulator.
@@ -5310,7 +5310,7 @@ do {								\
  }
 #ifdef NO_JUMP_TABLE
  default:
-    erl_exit(1, "unexpected op code %d\n",Go);
+    erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go);
   }
 #endif
     return;			/* Never executed */
@@ -5355,7 +5355,7 @@ translate_gc_bif(void* gcf)
     } else if (gcf == erts_gc_binary_part_3) {
 	return binary_part_3;
     } else {
-	erl_exit(1, "bad gc bif");
+	erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
     }
 }
 
@@ -5420,7 +5420,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
     Eterm* hp;
     Eterm Value = c_p->fvalue;
     Eterm Args = am_true;
-    c_p->i = pc;    /* In case we call erl_exit(). */
+    c_p->i = pc;    /* In case we call erts_exit(). */
 
     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
 
@@ -5484,7 +5484,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
 	    c_p->cp = 0;	/* To avoid keeping stale references. */
 	    return new_pc;
 	}
-	if (c_p->catches > 0) erl_exit(1, "Catch not found");
+	if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found");
     }
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     terminate_proc(c_p, Value);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index a8cc19ee1f..a61025d19a 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2216,7 +2216,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
 			    erts_dsend_export_trap_context(p, ctx));
 	break;
     default:
-	erl_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result);
+	erts_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result);
 	break;
     }
 
@@ -2260,7 +2260,7 @@ static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1)
 	BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1);
     }
     default:
-	erl_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result);
+	erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result);
 	break;
     }
     ASSERT(! "Can not arrive here");
@@ -2332,7 +2332,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
 			    erts_dsend_export_trap_context(p, ctx));
 	break;
     default:
-	erl_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result);
+	erts_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result);
 	break;
     }
 
@@ -3883,7 +3883,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1)
     erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);       
     pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1);
     if (pres < 0)
-	erl_exit(1, "Failed to convert term to string: %d (%s)\n",
+	erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to string: %d (%s)\n",
 		 -pres, erl_errno_id(-pres));
     hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */
     res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
@@ -3905,7 +3905,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1)
     }
     str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1));
     if (intlist_to_buf(string, str, len) != len)
-	erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+	erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
     str[len] = '\0';
     erts_fprintf(stderr, "%s", str);
     erts_free(ERTS_ALC_T_TMP, (void *) str);
@@ -3925,7 +3925,7 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0)
 BIF_RETTYPE halt_0(BIF_ALIST_0)
 {
     VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n"));
-    erl_halt(0);
+    erts_halt(0);
     ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
 }
 
@@ -3941,14 +3941,15 @@ BIF_RETTYPE halt_1(BIF_ALIST_1)
     Sint code;
     
     if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
+	int pos_int_code = (int)code & INT_MAX;
 	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
-	erl_halt((int)(- code));
+	erts_halt(pos_int_code);
 	ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
     }
     else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
 	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erl_exit(ERTS_ABORT_EXIT, "");
+	erts_exit(ERTS_ABORT_EXIT, "");
     }
     else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
 	int i;
@@ -3959,11 +3960,11 @@ BIF_RETTYPE halt_1(BIF_ALIST_1)
 	halt_msg[i] = '\0';
 	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
+	erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
     }
     else
 	goto error;
-    return NIL;  /* Pedantic (lint does not know about erl_exit) */
+    return NIL;  /* Pedantic (lint does not know about erts_exit) */
  error:
 	BIF_ERROR(BIF_P, BADARG);
 }
@@ -4002,22 +4003,23 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 	goto error;
 
     if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
+	int pos_int_code = (int)code & INT_MAX;
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
 	if (flush) {
-	    erl_halt((int)(- code));
+	    erts_halt(pos_int_code);
 	    ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
 	}
 	else {
 	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	    erl_exit((int)(- code), "");
+            erts_exit(pos_int_code, "");
 	}
     }
     else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erl_exit(ERTS_ABORT_EXIT, "");
+	erts_exit(ERTS_ABORT_EXIT, "");
     }
     else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
 	int i;
@@ -4029,11 +4031,11 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
+	erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
     }
     else
 	goto error;
-    return NIL;  /* Pedantic (lint does not know about erl_exit) */
+    return NIL;  /* Pedantic (lint does not know about erts_exit) */
  error:
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -4089,7 +4091,7 @@ term2list_dsprintf(Process *p, Eterm term)
     erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);       
     pres = erts_dsprintf(dsbufp, "%T", term);
     if (pres < 0)
-	erl_exit(1, "Failed to convert term to list: %d (%s)\n",
+	erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to list: %d (%s)\n",
 		 -pres, erl_errno_id(-pres));
     hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */
     res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index e670fbf31c..247c8f1122 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -47,7 +47,7 @@ erts_init_binary(void)
     if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) {
 	/* I assume that any compiler should be able to optimize this
 	   away. If not, this test is not very expensive... */
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Internal error: Address of orig_bytes[0] of a Binary"
 		 " is *not* 8-byte aligned\n");
     }
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 4ce9d24479..0ddf7f4e6d 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -104,7 +104,7 @@ process_killer(void)
 	    erts_printf("(k)ill (n)ext (r)eturn:\n");
 	    while(1) {
 		if ((j = sys_get_key(0)) <= 0)
-		    erl_exit(0, "");
+		    erts_exit(0, "");
 		switch(j) {
 		case 'k': {
 		    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
@@ -493,7 +493,7 @@ do_break(void)
        halt immediately if break is called */
     mode = erts_read_env("ERL_CONSOLE_MODE");
     if (mode && strcmp(mode, "window") != 0)
-	erl_exit(0, "");
+	erts_exit(0, "");
     erts_free_read_env(mode);
 #endif /* __WIN32__ */
 
@@ -503,7 +503,7 @@ do_break(void)
 
     while (1) {
 	if ((i = sys_get_key(0)) <= 0)
-	    erl_exit(0, "");
+	    erts_exit(0, "");
 	switch (i) {
 	case 'q':
 	case 'a': 
@@ -513,9 +513,9 @@ do_break(void)
 		   * The usual reason for a read error is Ctrl-C. Treat this as
 		   * 'a' to avoid infinite loop.
 		   */
-	    erl_exit(0, "");
+	    erts_exit(0, "");
 	case 'A':		/* Halt generating crash dump */
-	    erl_exit(1, "Crash dump requested by user");
+	    erts_exit(ERTS_ERROR_EXIT, "Crash dump requested by user");
 	case 'c':
 	    return;
 	case 'p':
@@ -785,7 +785,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_fdprintf(fd, "Atoms: %d\n", atom_table_size());
 
 #ifdef USE_THREADS
-    /* We want to note which thread it was that called erl_exit */
+    /* We want to note which thread it was that called erts_exit */
     if (erts_get_scheduler_data()) {
         erts_fdprintf(fd, "Calling Thread: scheduler:%d\n",
                       erts_get_scheduler_data()->no);
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 8849dadd00..64be43edb4 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -59,7 +59,7 @@ copy_object(Eterm obj, Process* to)
     res = copy_struct(obj, size, &hp, &to->off_heap);
 #ifdef DEBUG
     if (eq(obj, res) == 0) {
-	erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n");
+	erts_exit(ERTS_ABORT_EXIT, "copy not equal to source\n");
     }
 #endif
     return res;
@@ -171,7 +171,7 @@ Uint size_object(Eterm obj)
 			    }
 			    break;
 			default:
-			    erl_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+			    erts_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
 		    }
 		    break;
 		case SUB_BINARY_SUBTAG:
@@ -202,7 +202,7 @@ Uint size_object(Eterm obj)
 		    }
 		    break;
                 case BIN_MATCHSTATE_SUBTAG:
-		    erl_exit(ERTS_ABORT_EXIT,
+		    erts_exit(ERTS_ABORT_EXIT,
 			     "size_object: matchstate term not allowed");
 		default:
 		    sum += thing_arityval(hdr) + 1;
@@ -219,7 +219,7 @@ Uint size_object(Eterm obj)
 	    obj = ESTACK_POP(s);
 	    break;
 	default:
-	    erl_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj);
+	    erts_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj);
 	}
     }
 }
@@ -272,7 +272,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	goto L_copy_list;
     case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed;
     default:
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "%s, line %d: Internal error in copy_struct: 0x%08x\n",
 		 __FILE__, __LINE__,obj);
     }
@@ -331,7 +331,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
 	    case TAG_PRIMARY_BOXED: argp = tailp; goto L_copy_boxed;
 	    default:
-		erl_exit(ERTS_ABORT_EXIT,
+		erts_exit(ERTS_ABORT_EXIT,
 			 "%s, line %d: Internal error in copy_struct: 0x%08x\n",
 			 __FILE__, __LINE__,obj);
 	    }
@@ -512,11 +512,11 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			*argp = make_hashmap_rel(tp, dst_base);
 			break;
 		    default:
-			erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
+			erts_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr));
 		}
 		break;
 	    case BIN_MATCHSTATE_SUBTAG:
-		erl_exit(ERTS_ABORT_EXIT,
+		erts_exit(ERTS_ABORT_EXIT,
 			 "copy_struct: matchstate term not allowed");
 	    default:
 		i = thing_arityval(hdr)+1;
@@ -540,13 +540,13 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 
 #ifdef DEBUG
     if (htop != hbot)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Internal error in copy_struct() when copying %T:"
 		 " htop=%p != hbot=%p (sz=%beu)\n",
 		 org_obj, htop, hbot, org_sz); 
 #else
     if (htop > hbot) {
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Internal error in copy_struct(): htop, hbot overrun\n");
     }
 #endif
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 7be2b77a3b..787241b960 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1959,7 +1959,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    goto done;
 	}
 	default:
-	    erl_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase);
+	    erts_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase);
 	}
     }
 
@@ -1980,7 +1980,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (size > (Uint) INT_MAX)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Absurdly large distribution output data buffer "
 		 "(%beu bytes) passed.\n",
 		 size);
@@ -2020,7 +2020,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (size > (Uint) INT_MAX)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Absurdly large distribution output data buffer "
 		 "(%beu bytes) passed.\n",
 		 size);
@@ -3382,7 +3382,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
 		    continue;
 		break;
 	    default:
-		erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
+		erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
 	    }
 	}
 
@@ -3691,7 +3691,7 @@ erts_processes_monitoring_nodes(Process *c_p)
 		case ERTS_NODES_MON_OPT_TYPES:        type = am_all;     break;
 		case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break;
 		case ERTS_NODES_MON_OPT_TYPE_HIDDEN:  type = am_hidden;  break;
-		default: erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
+		default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
 		}
 		olist = erts_bld_cons(hpp, szp, 
 				      erts_bld_tuple(hpp, szp, 2,
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index 47dafa53c0..4b0541c10e 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -241,7 +241,7 @@ info_options(Allctr_t *allctr,
     if (hpp || szp) {
 	
 	if (!atoms_initialized)
-	    erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+	    erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
 		     __FILE__, __LINE__);;
 
 	res = NIL;
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index c3f4fe5a63..0d88dfa363 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -653,11 +653,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) {
 	    int err = errno;
 	    char *errstr = err ? strerror(err) : "unknown";
-	    erl_exit(-1, "Failed to lock physical memory: %s (%d)\n",
+	    erts_exit(1, "Failed to lock physical memory: %s (%d)\n",
 		     errstr, err);
 	}
 #else
-	erl_exit(-1, "Failed to lock physical memory: Not supported\n");
+	erts_exit(1, "Failed to lock physical memory: Not supported\n");
 #endif
     }
 
@@ -806,13 +806,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	if (!erts_allctrs[i].alloc)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Missing alloc function for %s\n", ERTS_ALC_A2AD(i));
 	if (!erts_allctrs[i].realloc)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Missing realloc function for %s\n", ERTS_ALC_A2AD(i));
 	if (!erts_allctrs[i].free)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Missing free function for %s\n", ERTS_ALC_A2AD(i));
     }
 
@@ -890,7 +890,7 @@ erts_alloc_late_init(void)
 static void *
 erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
 {
-    erl_exit(ERTS_ABORT_EXIT,
+    erts_exit(ERTS_ABORT_EXIT,
 	     "Attempt to reallocate a block of the fixed size type %s\n",
 	     ERTS_ALC_T2TD(type));
 }
@@ -1012,7 +1012,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 					  * tspec->size)
 				       + ERTS_CACHE_LINE_SIZE - 1));
 	if (!states)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Failed to allocate allocator states for %salloc\n",
 		     init->init.util.name_prefix);
 	tspec->allctr = (Allctr_t **) states;
@@ -1040,7 +1040,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 				   (tot_fix_list_size
 				    + ERTS_CACHE_LINE_SIZE - 1));
 	if (!fix_lists)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Failed to allocate fix lists for %salloc\n",
 		     init->init.util.name_prefix);
 
@@ -1114,7 +1114,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 	}
 
 	if (!as)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Failed to start %salloc\n", init->init.util.name_prefix);
 
 	ASSERT(as == (void *) as0);
@@ -1909,7 +1909,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
 	case ERTS_ALC_O_FREE:		op_str = "free";	break;
 	default:			op_str = "UNKNOWN";	break;
 	}
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "%s: %s operation not supported (memory type: \"%s\")\n",
 		 allctr_str, op_str, t_str);
 	break;
@@ -1923,18 +1923,18 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
 	va_start(argp, n);
 	size = va_arg(argp, Uint);
 	va_end(argp);
-	erl_exit(1,
+	erts_exit(ERTS_ERROR_EXIT,
 		 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
 		 allctr_str, op, size, t_str);
 	break;
     }
     case ERTS_ALC_E_NOALLCTR:
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_alloc: Unknown allocator type: %d\n",
 		 ERTS_ALC_T2A(ERTS_ALC_N2T(n)));
 	break;
     default:
-	erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
+	erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
 	break;
     }
 }
@@ -3189,7 +3189,7 @@ reply_alloc_info(void *vair)
 					       make_small(0), ainfo);
 		    }
 		    else {
-			erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
+			erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
 				 __FILE__, __LINE__);
 		    }
 		}
@@ -3426,7 +3426,7 @@ void *safe_realloc(void *ptr, Uint sz)
  * Keep alloc_SUITE_data/allocator_test.h updated if changes are made        *
  * to erts_alc_test()                                                        *
 \*                                                                           */
-#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
+#define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
 
 UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 {
@@ -3888,7 +3888,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
     found_type = GET_TYPE_OF_PATTERN(pre_pattern);
     if (pre_pattern != MK_PATTERN(n)) {
 	if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "ERROR: Fence at beginning of memory block (p=0x%u) "
 		     "clobbered.\n",
 		     (UWord) ptr);
@@ -3905,12 +3905,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 	char *op_str;
 
 	if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "ERROR: Fence at end of memory block (p=0x%u, sz=%u) "
 		     "clobbered.\n",
 		     (UWord) ptr, (UWord) sz);
 	if (found_type != GET_TYPE_OF_PATTERN(post_pattern))
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "ERROR: Fence around memory block (p=0x%u, sz=%u) "
 		     "clobbered.\n",
 		     (UWord) ptr, (UWord) sz);
@@ -3933,7 +3933,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 	default:			op_str = "???";		break;
 	}
 
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\","
 		 " but %s as type \"%s\".\n",
 		 (UWord) ptr, (UWord) sz, ftype, op_str, otype);
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 8229a15824..18312eacde 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4518,7 +4518,7 @@ make_name_atoms(Allctr_t *allctr)
     size_t prefix_len = strlen(allctr->name_prefix);
 
     if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1)
-	erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix);
+	erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix);
 
     memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len);
 
@@ -5720,7 +5720,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     /* erts_alcu_start assumes that allctr has been zeroed */
 
     if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) {
-        erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n",
+        erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n",
                  __FILE__, __LINE__);
     }
 
@@ -5904,7 +5904,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	  if (allctr->thread_safe)
 	    erts_mtx_destroy(&allctr->mutex);
 #endif
-	  erl_exit(ERTS_ABORT_EXIT,
+	  erts_exit(ERTS_ABORT_EXIT,
 	    "Failed to create main carrier for %salloc\n",
 	    init->name_prefix);
 	}
@@ -6120,7 +6120,7 @@ erts_alcu_verify_unused(Allctr_t *allctr)
     if (no) {
 	UWord sz = allctr->sbcs.blocks.curr.size;
 	sz += allctr->mbcs.blocks.curr.size;
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "%salloc() used when expected to be unused!\n"
 		 "Total amount of blocks allocated: %bpu\n"
 		 "Total amount of bytes allocated: %bpu\n",
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 19420af8ab..5cd067ff54 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -1035,7 +1035,7 @@ info_options(Allctr_t *allctr,
     if (hpp || szp) {
 	
 	if (!atoms_initialized)
-	    erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+	    erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
 		     __FILE__, __LINE__);;
 
 	res = NIL;
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index fb853b65ab..f39a18ac88 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -940,7 +940,7 @@ info_options(Allctr_t *allctr,
     if (hpp || szp) {
 	
 	if (!atoms_initialized)
-	    erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+	    erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
 		     __FILE__, __LINE__);;
 
 	res = NIL;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 28bec6325c..f4ef59993a 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1309,7 +1309,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro
     case ERL_DE_FORCE_RELOAD:
 	break;
     default:
-	erl_exit(1,"Internal error, unknown state %u in dynamic driver.", drv->handle->status);
+	erts_exit(ERTS_ERROR_EXIT,"Internal error, unknown state %u in dynamic driver.", drv->handle->status);
     }
     p->flags |= F_USING_DDLL;
     r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b44382cde8..0341b809db 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -914,7 +914,7 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1)
 	case ERTS_PI_FAIL_TYPE_AWAIT_EXIT:
 	    ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
 	default:
-	    erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__);
+	    erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__);
 	}
     }
 
@@ -954,7 +954,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
 	    case ERTS_PI_FAIL_TYPE_AWAIT_EXIT:
 		ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
 	    default:
-		erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error",
+		erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error",
 			 __FILE__, __LINE__);
 	    }
 	}
@@ -1759,7 +1759,7 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 		return res;
 	    buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
 	    if (intlist_to_buf(*tp, buf, len) != len)
-		erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+		erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
 	    buf[len] = '\0';
 	    res = erts_instr_dump_memory_map(buf) ? am_true : am_false;
 	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
@@ -1778,7 +1778,7 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 		    return res;
 		buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
 		if (intlist_to_buf(tp[1], buf, len) != len)
-		    erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+		    erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
 		buf[len] = '\0';
 		res = erts_instr_dump_stat(buf, 1) ? am_true : am_false;
 		erts_free(ERTS_ALC_T_TMP, (void *) buf);
@@ -3765,7 +3765,7 @@ static void broken_halt_test(Eterm bif_arg_2)
 #if defined(ERTS_HAVE_TRY_CATCH)
     erts_get_scheduler_data()->run_queue = NULL;
 #endif
-    erl_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2);
+    erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2);
 }
 
 
@@ -4012,7 +4012,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		BIF_RET(am_true);
 	}
 	else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
-	    erl_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2);
+	    erts_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2);
 	}
 	else if (ERTS_IS_ATOM_STR("kill_dist_connection", BIF_ARG_1)) {
 	    DistEntry *dep = erts_sysname_to_connected_dist_entry(BIF_ARG_2);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index e47d7bcbbb..27c24197ea 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -791,7 +791,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 	} else {
 	    name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
 	    if (intlist_to_buf(name, name_buf, i) != i)
-		erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
+		erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
 	    name_buf[i] = '\0';
 	}
 	driver = &vanilla_driver;
@@ -1169,7 +1169,7 @@ static Eterm http_bld_uri(struct packet_callback_args* pca,
         return erts_bld_tuple(hpp, szp, 3, am_scheme, s1, s2);
                               
     default:
-        erl_exit(1, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type);
+        erts_exit(ERTS_ERROR_EXIT, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type);
     }
 }
 
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 8395f6ecc6..c263eebfcc 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -402,7 +402,7 @@ cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
 	    break;
 	default:
 	    cmp_func = NULL;
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Bad cpu bind type: %d\n",
 		     (int) cpu_bind_order);
 	    break;
@@ -1590,7 +1590,7 @@ get_cpu_topology_term(Process *c_p, int type)
 	}
 	break;
     default:
-	erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
+	erts_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
 	break;
     }
 
@@ -1967,7 +1967,7 @@ cpu_group_insert(erts_cpu_groups_map_t *map,
 	    ix = 0;
     } while (ix != start);
 
-    erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n");
+    erts_exit(ERTS_ABORT_EXIT, "Reader groups map full\n");
 }
 
 
@@ -2290,7 +2290,7 @@ remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg)
 	prev_cgm = cgm;
     }
 
-    erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n");
+    erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n");
 }
 
 static int
@@ -2320,7 +2320,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map,
 	    ix = 0;
     } while (ix != start);
 
-    erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical);
+    erts_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 878ee32b47..83c7ba7bbc 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1295,7 +1295,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 	goto badarg;
 
     if (!remove_named_tab(tb, 1))
-	erl_exit(1,"Could not find named tab %s", tb->common.id);
+	erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id);
 
     tb->common.id = tb->common.the_name = BIF_ARG_2;
 
@@ -1581,7 +1581,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 			   BIF_P->common.id,
 			   make_small(slot)),
 		    0) != DB_ERROR_NONE) {
-	erl_exit(1,"Could not update ets metadata.");
+	erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata.");
     }
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 
@@ -2940,7 +2940,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
 
     bits = erts_fit_in_bits_int32(db_max_tabs-1);
     if (bits > SMALL_BITS) {
-	erl_exit(1,"Max limit for ets tabled too high %u (max %u).",
+	erts_exit(ERTS_ERROR_EXIT,"Max limit for ets tabled too high %u (max %u).",
 		 db_max_tabs, ((Uint)1)<<SMALL_BITS);
     }
     meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1;
@@ -3001,7 +3001,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
     db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
 
     if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) {
-	erl_exit(1,"Unable to create ets metadata tables.");
+	erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables.");
     }
 
     erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0);
@@ -3032,7 +3032,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
     db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
 
     if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) {
-	erl_exit(1,"Unable to create ets metadata tables.");
+	erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables.");
     }
 
     /* Non visual BIF to trap to. */
@@ -3229,7 +3229,7 @@ retry:
  * yielding.
  */
 #define ERTS_DB_INTERNAL_ERROR(LSTR) \
-  erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \
+  erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \
 	   __FILE__, __LINE__)
 
 int
@@ -3501,7 +3501,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 			   make_small(tb->common.slot)),
 		    0) != DB_ERROR_NONE) {
 	UnUseTmpHeap(3,p);
-	erl_exit(1,"Could not insert ets metadata in safe_fixtable.");
+	erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable.");
     }	
     UnUseTmpHeap(3,p);
     db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 98a2e2842a..fa925c94a5 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -3028,7 +3028,7 @@ void db_check_table_hash(DbTable *tbl)
 	if ((list = BUCKET(tb,j)) != 0) {
 	    while (list != 0) {
 		if (!is_tuple(make_tuple(list->dbterm.tpl))) {
-		    erl_exit(1, "Bad term in slot %d of ets table", j);
+		    erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j);
 		}
 		list = list->next;
 	    }
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 465aa566ad..61293fbf9a 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1213,7 +1213,7 @@ static int db_select_count_continue_tree(Process *p,
     tptr = tuple_val(continuation);
 
     if (arityval(*tptr) != 5)
-	erl_exit(1,"Internal error in ets:select_count/1");
+	erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_count/1");
     
     lastkey = tptr[2];
     end_condition = tptr[3];
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index dab357a079..af5b611afd 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1946,11 +1946,11 @@ restart:
 
     #ifdef DMC_DEBUG
 	if (*heap_fence != FENCE_PATTERN) {
-	    erl_exit(1, "Heap fence overwritten in db_prog_match after op "
+	    erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op "
 		     "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
 	}
 	if (*stack_fence != FENCE_PATTERN) {
-	    erl_exit(1, "Stack fence overwritten in db_prog_match after op "
+	    erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op "
 		     "0x%08x, overwritten with 0x%08x.", save_op, 
 		     *stack_fence);
 	}
@@ -2615,7 +2615,7 @@ restart:
 	case matchHalt:
 	    goto success;
 	default:
-	    erl_exit(1, "Internal error: unexpected opcode in match program.");
+	    erts_exit(ERTS_ERROR_EXIT, "Internal error: unexpected opcode in match program.");
 	}
     }
 fail:
@@ -2639,11 +2639,11 @@ success:
 
 #ifdef DMC_DEBUG
     if (*heap_fence != FENCE_PATTERN) {
-	erl_exit(1, "Heap fence overwritten in db_prog_match after op "
+	erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op "
 		 "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
     }
     if (*stack_fence != FENCE_PATTERN) {
-	erl_exit(1, "Stack fence overwritten in db_prog_match after op "
+	erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op "
 		 "0x%08x, overwritten with 0x%08x.", save_op, 
 		 *stack_fence);
     }
@@ -3683,7 +3683,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 	break;
     }
     default:
-	erl_exit(1, "db_match_compile: "
+	erts_exit(ERTS_ERROR_EXIT, "db_match_compile: "
 		 "Bad object on heap: 0x%bex\n", c);
     }
     return retOk;
@@ -4930,7 +4930,7 @@ static DMCRet dmc_fun(DMCContext *context,
 	DMC_PUSH(*text, matchCall3);
 	break;
     default:
-	erl_exit(1,"ets:match() internal error, "
+	erts_exit(ERTS_ERROR_EXIT,"ets:match() internal error, "
 		 "guard with more than 3 arguments.");
     }
     DMC_PUSH(*text, (UWord) b->biff);
@@ -5210,7 +5210,7 @@ static Uint my_size_object(Eterm t)
 		   tmp == am_const) {
 	    sum += size_object(tuple_val(t)[2]);
 	} else {
-	    erl_exit(1,"Internal error, sizing unrecognized object in "
+	    erts_exit(ERTS_ERROR_EXIT,"Internal error, sizing unrecognized object in "
 		     "(d)ets:match compilation.");
 	}
 	break;
@@ -5255,7 +5255,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap)
 		sz = size_object(b);
 		ret = copy_struct(b,sz,hp,off_heap);
 	    } else {
-		erl_exit(1, "Trying to constant-copy non constant expression "
+		erts_exit(ERTS_ERROR_EXIT, "Trying to constant-copy non constant expression "
 			 "0x%bex in (d)ets:match compilation.", t);
 	    }
 	} else {
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 2dcfb79f00..4928aae9c2 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -255,14 +255,14 @@ void erts_check_stack(Process *p)
     Eterm *stack_end = p->htop;
 
     if (p->stop > stack_start)
-	erl_exit(1,
+	erts_exit(ERTS_ERROR_EXIT,
 		 "<%lu.%lu.%lu>: Stack underflow\n",
 		 internal_pid_channel_no(p->common.id),
 		 internal_pid_number(p->common.id),
 		 internal_pid_serial(p->common.id));
 
     if (p->stop < stack_end)
-	erl_exit(1,
+	erts_exit(ERTS_ERROR_EXIT,
 		 "<%lu.%lu.%lu>: Stack overflow\n",
 		 internal_pid_channel_no(p->common.id),
 		 internal_pid_number(p->common.id),
@@ -287,7 +287,7 @@ void erts_check_stack(Process *p)
 	if (in_mbuf)
 	    continue;
 
-	erl_exit(1,
+	erts_exit(ERTS_ERROR_EXIT,
 		 "<%lu.%lu.%lu>: Wild stack pointer\n",
 		 internal_pid_channel_no(p->common.id),
 		 internal_pid_number(p->common.id),
@@ -372,7 +372,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end)
 #ifdef DEBUG
         if (hval == DEBUG_BAD_WORD) {
             print_untagged_memory(start, end);
-            erl_exit(1, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n",
+            erts_exit(ERTS_ERROR_EXIT, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n",
                      PTR_SIZE,(unsigned long)(pos - 1));
         }
 #endif
@@ -385,7 +385,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end)
         if (verify_eterm(p,hval))
             continue;
 
-        erl_exit(1, "Wild pointer found @ 0x%0*lx!\n",
+        erts_exit(ERTS_ERROR_EXIT, "Wild pointer found @ 0x%0*lx!\n",
                  PTR_SIZE,(unsigned long)(pos - 1));
     }
 }
@@ -395,11 +395,11 @@ void verify_process(Process *p)
 #define VERIFY_AREA(name,ptr,sz) {                                      \
     int n = (sz);							\
     while (n--) if(!verify_eterm(p,*(ptr+n)))				\
-        erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); }
+        erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); }
 
 #define VERIFY_ETERM(name,eterm) {                                      \
     if(!verify_eterm(p,eterm))                                          \
-        erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); }
+        erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); }
 
 
     ErlMessage* mp = p->msg.first;
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index e0404eb5c9..184f8e8931 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -43,7 +43,7 @@ fatal_error(int err, char *func)
 	else
 	    estr = "Unknown error";
     }
-    erl_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err);
+    erts_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err);
 }
 
 #define ERL_DRV_TSD_KEYS_INC 10
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 4268e2d40a..2f13aa364f 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -185,7 +185,7 @@ erts_erase_fun_entry(ErlFunEntry* fe)
 #endif
     {
 	if (fe->address != unloaded_fun)
-	    erl_exit(1,
+	    erts_exit(ERTS_ERROR_EXIT,
 		     "Internal error: "
 		     "Invalid reference count found on #Fun<%T.%d.%d>: "
 		     " About to erase fun still referred by code.\n",
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 2f21111a2e..f48c46ca33 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -59,7 +59,7 @@
         erts_fprintf(stderr, "stop=%p\n", (p)->stop); \
         erts_fprintf(stderr, "htop=%p\n", (p)->htop); \
         erts_fprintf(stderr, "heap=%p\n", (p)->heap); \
-        erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \
+        erts_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \
 		 __FILE__,__LINE__,(P)->common.id); \
     }
 
@@ -268,7 +268,7 @@ erts_next_heap_size(Uint size, Uint offset)
 		low = mid + 1;
 	    }
 	}
-	erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset);
+	erts_exit(ERTS_ERROR_EXIT, "no next heap size found: %beu, offset %beu\n", size, offset);
     }
     return 0;
 }
@@ -2788,7 +2788,7 @@ within(Eterm *ptr, Process *p)
 #define ERTS_CHK_OFFHEAP_ASSERT(EXP)			\
 do {							\
     if (!(EXP))						\
-	erl_exit(ERTS_ABORT_EXIT,			\
+	erts_exit(ERTS_ABORT_EXIT,			\
 		 "%s:%d: Assertion failed: %s\n",	\
 		 __FILE__, __LINE__, #EXP);		\
 } while (0)
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index f89f8723d9..9b4aad9d91 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -571,8 +571,8 @@ info_options(Allctr_t *allctr,
     if (hpp || szp) {
 	
 	if (!atoms_initialized)
-	    erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
-		     __FILE__, __LINE__);;
+	    erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized",
+		     __FILE__, __LINE__);
 
 	res = NIL;
 	add_2tup(hpp, szp, &res, am.as, am.gf);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index d9c3b0dcf4..e729574ec7 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -269,7 +269,7 @@ this_rel_num(void)
 	    i++;
 	this_rel = atoi(&this_rel_str[i]); 
 	if (this_rel < 1)
-	    erl_exit(-1, "Unexpected ERLANG_OTP_RELEASE format\n");
+	    erts_exit(1, "Unexpected ERLANG_OTP_RELEASE format\n");
     }
     return this_rel;
 }
@@ -415,7 +415,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
     start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1);
     if (erts_find_function(start_mod, am_start, 2,
 			   erts_active_code_ix()) == NULL) {
-	erl_exit(5, "No function %s:start/2\n", modname);
+	erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname);
     }
 
     /*
@@ -512,12 +512,12 @@ load_preloaded(void)
 	length = preload_p[i].size;
 	module_name = erts_atom_put((byte *) name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1);
 	if ((code = sys_preload_begin(&preload_p[i])) == 0)
-	    erl_exit(1, "Failed to find preloaded code for module %s\n", 
+	    erts_exit(ERTS_ERROR_EXIT, "Failed to find preloaded code for module %s\n",
 		     name);
 	res = erts_preload_module(NULL, 0, NIL, &module_name, code, length);
 	sys_preload_end(&preload_p[i]);
 	if (res != NIL)
-	    erl_exit(1,"Failed loading preloaded module %s (%T)\n",
+	    erts_exit(ERTS_ERROR_EXIT,"Failed loading preloaded module %s (%T)\n",
 		     name, res);
 	i++;
     }
@@ -648,7 +648,7 @@ void erts_usage(void)
     erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n");
     erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n");
     erts_fprintf(stderr, "\n\n");
-    erl_exit(-1, "");
+    erts_exit(1, "");
 }
 
 #ifdef USE_THREADS
@@ -1445,7 +1445,7 @@ erl_start(int argc, char **argv)
 		}
 		erts_fprintf(stderr, "(" EMULATOR ") emulator version "
 			   ERLANG_VERSION "\n");
-		erl_exit(0, "");
+		erts_exit(0, "");
 	    }
 	    break;
 
@@ -2247,23 +2247,17 @@ system_cleanup(int flush_async)
 }
 
 static __decl_noreturn void __noreturn
-erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
+erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
 {
-    unsigned int an;
-
     system_cleanup(flush_async);
 
     save_statistics();
-    if (n < 0)
-        an = -(unsigned int)n;
-    else
-        an = n;
 
     if (erts_mtrace_enabled)
-	erts_mtrace_exit((Uint32) an);
+	erts_mtrace_exit((Uint32) n);
 
     /* Produce an Erlang core dump if error */
-    if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT)
+    if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT)
 	&& erts_initialized) {
 	erl_crash_dump_v((char*) NULL, 0, fmt, args1);
     }
@@ -2276,29 +2270,29 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
 	exit(0);
     else if (n == ERTS_DUMP_EXIT)
 	ERTS_EXIT_AFTER_DUMP(1);
-    else if (n > 0 || n == ERTS_ABORT_EXIT)
+    else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT)
         abort();
-    exit(an);
+    exit(n);
 }
 
 /* Exit without flushing async threads */
-__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...)
+__decl_noreturn void __noreturn erts_exit(int n, char *fmt, ...)
 {
     va_list args1, args2;
     va_start(args1, fmt);
     va_start(args2, fmt);
-    erl_exit_vv(n, 0, fmt, args1, args2);
+    erts_exit_vv(n, 0, fmt, args1, args2);
     va_end(args2);
     va_end(args1);
 }
 
 /* Exit after flushing async threads */
-__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...)
+__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...)
 {
     va_list args1, args2;
     va_start(args1, fmt);
     va_start(args2, fmt);
-    erl_exit_vv(n, 1, fmt, args1, args2);
+    erts_exit_vv(n, 1, fmt, args1, args2);
     va_end(args2);
     va_end(args1);
 }
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index ff2a355309..3c066cea7b 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1274,7 +1274,7 @@ recurse:
                 break;
             }
             default:
-                erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA);
+                erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA);
             }
         }
 
@@ -1301,7 +1301,7 @@ recurse:
                 break;
             }
             default:
-                erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB);
+                erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB);
             }
         }
     }
@@ -1391,7 +1391,7 @@ resume_from_trap:
             res = make_boxed(nhp);
             break;
         default:
-            erl_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix);
+            erts_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix);
         }
     }
 
@@ -1886,7 +1886,7 @@ void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) {
         sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
 	break;
     default:
-	erl_exit(ERTS_ABORT_EXIT, "bad header");
+	erts_exit(ERTS_ABORT_EXIT, "bad header");
     }
 
     WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE,  /* end marker */
@@ -1923,7 +1923,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
 		ASSERT(sz < 17);
 		break;
 	    default:
-		erl_exit(ERTS_ABORT_EXIT, "bad header");
+		erts_exit(ERTS_ABORT_EXIT, "bad header");
 	    }
 
 	    idx++;
@@ -1973,7 +1973,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
 		ASSERT(sz < 17);
 		break;
 	    default:
-		erl_exit(1, "bad header");
+		erts_exit(ERTS_ERROR_EXIT, "bad header");
 	    }
 
             if (idx > sz)
@@ -2161,12 +2161,12 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
 			size += HAMT_HEAD_BITMAP_SZ(n+1);
 			goto unroll;
 		    default:
-			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
 			break;
 		}
 		break;
 	    default:
-		erl_exit(1, "bad primary tag %p\r\n", node);
+		erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node);
 		break;
 	}
     }
@@ -2281,12 +2281,12 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
 			res = make_hashmap(nhp);
 			break;
 		    default:
-			erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
 			break;
 		}
 		break;
 	    default:
-		erl_exit(1, "bad primary tag %x\r\n", primary_tag(node));
+		erts_exit(ERTS_ERROR_EXIT, "bad primary tag %x\r\n", primary_tag(node));
 		break;
 	}
 
@@ -2404,12 +2404,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
 			/* not occupied */
 			goto not_found;
 		    default:
-			erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+			erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
 			break;
 		}
 		break;
 	    default:
-		erl_exit(1, "bad primary tag %p\r\n", node);
+		erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node);
 		break;
 	}
     }
@@ -2586,7 +2586,7 @@ unroll:
 		res = make_hashmap(nhp);
 		break;
 	    default:
-		erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
+		erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
 		break;
 	}
     } while(!ESTACK_ISEMPTY(stack));
@@ -2727,7 +2727,7 @@ BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) {
             case HAMT_SUBTAG_NODE_BITMAP:
                 BIF_RET(AM_hashmap_node);
             default:
-                erl_exit(1, "bad header");
+                erts_exit(ERTS_ERROR_EXIT, "bad header");
         }
     }
     BIF_P->fvalue = BIF_ARG_1;
@@ -2763,7 +2763,7 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) {
                 ptr += 2;
                 break;
             default:
-                erl_exit(1, "bad header\r\n");
+                erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
                 break;
         }
         ASSERT(sz < 17);
@@ -2841,7 +2841,7 @@ static Eterm hashmap_info(Process *p, Eterm node) {
 			}
 			break;
 		    default:
-			erl_exit(1, "bad header\r\n");
+			erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
 			break;
 		}
 	}
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 7dfa01c8ac..bd899fa2f9 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -356,7 +356,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid,
 	    tstack[tpos++] = this;
 	    this = &((*this)->right);
 	} else { /* Equal key is an error for monitors */
-	    erl_exit(1,"Insertion of already present monitor!");
+	    erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!");
 	    break;
 	}
     }
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d7a2076d85..3e93f9e514 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -330,7 +330,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 #ifdef ERTS_SMP
 	c_p = NULL;
 #else
-	erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
+	erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
 #endif
     }    
 
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 707de39556..e69bd49c42 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1050,7 +1050,7 @@ insert_dist_entry(DistEntry *dist, int type, Eterm id, Uint creation)
     }
 
     if(!rdp)
-	erl_exit(1,
+	erts_exit(ERTS_ERROR_EXIT,
 		 "Reference to non-existing distribution table entry found!\n");
 
     insert_dist_referrer(rdp, type, id, creation);
@@ -1111,7 +1111,7 @@ insert_node(ErlNode *node, int type, Eterm id)
     }
 
     if (!rnp)
-	erl_exit(1, "Reference to non-existing node table entry found!\n");
+	erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing node table entry found!\n");
 
     insert_node_referrer(rnp, type, id);
 }
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 5c38db1cbc..ddcca06b0a 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1784,7 +1784,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds);
 	    break;
 	default:
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Invalid port task type: %d\n",
 		     (int) ptp->type);
 	    break;
@@ -2048,7 +2048,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 		break;
 	    }
 	    default:
-		erl_exit(ERTS_ABORT_EXIT,
+		erts_exit(ERTS_ABORT_EXIT,
 			 "Invalid port task type: %d\n",
 			 (int) ptp->type);
 	    }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d583118e7b..12687deaeb 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -515,7 +515,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
 
     if (~valid & value)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Invalid aux_work value found: 0x%x\n",
 		 ~valid & value);
 }
@@ -1211,7 +1211,7 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
     case 0:
 	break;
     default:
-	erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n",
+	erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n",
 		 __FILE__, __LINE__);
 	break;
     }
@@ -2195,7 +2195,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 	    erts_port_release(prt);
 	}
 	if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) {
-	    erl_exit_flush_async(erts_halt_code, "");
+	    erts_flush_async_exit(erts_halt_code, "");
 	}
     }
     return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS;
@@ -3717,7 +3717,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 	    prio = ERTS_PORT_PRIO_LEVEL;
 	    break;
 	default:
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "%s:%d:%s(): Invalid immigrate queue mask",
 		     __FILE__, __LINE__, __func__);
 	    prio = 0;
@@ -3741,7 +3741,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 		    rq = erts_port_runq(prt);
 		    if (rq) {
 			if (rq != c_rq)
-			    erl_exit(ERTS_ABORT_EXIT,
+			    erts_exit(ERTS_ABORT_EXIT,
 				     "%s:%d:%s(): Internal error",
 				     __FILE__, __LINE__, __func__);
 			erts_enqueue_port(c_rq, prt);
@@ -3932,7 +3932,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	    prt_rq = erts_port_runq(prt);
 	    if (prt_rq) {
 		if (prt_rq != to_rq)
-		    erl_exit(ERTS_ABORT_EXIT,
+		    erts_exit(ERTS_ABORT_EXIT,
 			     "%s:%d:%s() internal error\n",
 			     __FILE__, __LINE__, __func__);
 		erts_enqueue_port(to_rq, prt);
@@ -4133,7 +4133,7 @@ no_procs:
 	    return 0;
 	else {
 	    if (prt_rq != rq)
-		erl_exit(ERTS_ABORT_EXIT,
+		erts_exit(ERTS_ABORT_EXIT,
 			 "%s:%d:%s() internal error\n",
 			 __FILE__, __LINE__, __func__);
 	    *rq_lockedp = 1;
@@ -8020,7 +8020,7 @@ sched_thread_func(void *vesdp)
 
     process_main();
     /* No schedulers should *ever* terminate */
-    erl_exit(ERTS_ABORT_EXIT,
+    erts_exit(ERTS_ABORT_EXIT,
 	     "Scheduler thread number %beu terminated\n",
 	     no);
     return NULL;
@@ -8085,7 +8085,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
 
     process_main();
     /* No schedulers should *ever* terminate */
-    erl_exit(ERTS_ABORT_EXIT,
+    erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty CPU scheduler thread number %beu terminated\n",
 	     no);
     return NULL;
@@ -8148,7 +8148,7 @@ sched_dirty_io_thread_func(void *vesdp)
 
     process_main();
     /* No schedulers should *ever* terminate */
-    erl_exit(ERTS_ABORT_EXIT,
+    erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty I/O scheduler thread number %beu terminated\n",
 	     no);
     return NULL;
@@ -8178,12 +8178,12 @@ erts_start_schedulers(void)
         erts_snprintf(opts.name, 16, "runq_supervisor");
 	erts_atomic_init_nob(&runq_supervisor_sleeping, 0);
 	if (0 != ethr_event_init(&runq_supervision_event))
-	    erl_exit(1, "Failed to create run-queue supervision event\n");
+	    erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n");
 	if (0 != ethr_thr_create(&runq_supervisor_tid,
 				 runq_supervisor,
 				 NULL,
 				 &opts))
-	    erl_exit(1, "Failed to create run-queue supervision thread\n");
+	    erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n");
 
     }
 #endif
@@ -8227,14 +8227,14 @@ erts_start_schedulers(void)
 	    erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1);
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
-		erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix);
+		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix);
 	}
 	for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
 	    erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1);
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
-		erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix);
+		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix);
 	}
     }
 #endif
@@ -8250,10 +8250,10 @@ erts_start_schedulers(void)
 
     res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
     if (res != 0)
-	erl_exit(1, "Failed to create aux thread\n");
+	erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n");
 
     if (actual < 1)
-	erl_exit(1,
+	erts_exit(ERTS_ERROR_EXIT,
 		 "Failed to create any scheduler-threads: %s (%d)\n",
 		 erl_errno_id(res),
 		 res);
@@ -12092,7 +12092,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 	break;
 	
     default:
-	erl_exit(1, "bad type in link list\n");
+	erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n");
 	break;
     }
     erts_destroy_link(lnk);
@@ -12133,7 +12133,7 @@ erts_do_exit_process(Process* p, Eterm reason)
 #endif
 
     if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC)
-	erl_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n",
+	erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n",
                  p->common.id, reason);
 
 #ifdef ERTS_SMP
@@ -12251,7 +12251,7 @@ erts_continue_exit_process(Process *p)
 	    break;
 	case ERTS_SCHDLR_SSPND_EINVAL:
 	default:
-	    erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n",
+	    erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n",
 		     __FILE__, __LINE__, (int) ssr);
 	}
     }
@@ -12748,10 +12748,10 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
  *    The same global atomic is used as refcount.
  *
  * A BIF that calls this should make sure to schedule out to never come back:
- *    erl_halt((int)(- code));
+ *    erts_halt(code);
  *    ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL);
  */
-void erl_halt(int code)
+void erts_halt(int code)
 {
     if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress,
 					     erts_no_schedulers,
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 10c6fa4a67..a938e1d884 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -2303,6 +2303,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
 #endif
 
 
-void erl_halt(int code);
+void erts_halt(int code);
 extern erts_smp_atomic32_t erts_halt_progress;
 extern int erts_halt_code;
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 8606371bdf..3253ee6b70 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -374,7 +374,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret)
 		     "display term found in line %d:\n"
 		     "%T\n", p->common.id, __LINE__, old);
 #endif
-	erl_exit(1, "Damaged process dictionary found during erase/1.");
+	erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during erase/1.");
     }
     if ((range = HASH_RANGE(p->dictionary)) > INITIAL_SIZE && 
 	range / 2  > (p->dictionary->numElements)) {
@@ -419,7 +419,7 @@ Eterm erts_pd_hash_get(Process *p, Eterm id)
 		     "display term found in line %d:\n"
 		     "%T\n", p->common.id, __LINE__, tmp);
 #endif
-	erl_exit(1, "Damaged process dictionary found during get/1.");
+	erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during get/1.");
     }
     return am_undefined;
 }
@@ -670,7 +670,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 		     "%T\n", p->common.id, __LINE__, old);
 #endif
 
-	erl_exit(1, "Damaged process dictionary found during put/2.");
+	erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2.");
     }
     if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) {
 	grow(p);
@@ -1024,7 +1024,7 @@ static void pd_check(ProcDict *pd)
 	    }
 	    continue;
 	} else {
-	    erl_exit(1, 
+	    erts_exit(ERTS_ERROR_EXIT,
 		     "Found tag 0x%08x in process dictionary at position %d",
 		     (unsigned long) t, (int) i);
 	}
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index f7997df051..9ed175fe01 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -1255,7 +1255,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp)
 	    return 1;
 
 	default:
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "%s:%d:ptab_list_bif_engine(): Invalid state: %d\n",
 		     __FILE__, __LINE__, (int) ptlbdp->state);
 	}
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index e5050bfaa5..bae3385d3f 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -41,7 +41,7 @@ et_abort(const char *expr, const char *file, unsigned line)
 	 * Prevent infinite loop.
 	 */
 	have_been_called = 1;
-	erl_exit(1, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
+	erts_exit(ERTS_ERROR_EXIT, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
     }
 #else
     erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 7148b756e7..7b06fd840f 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -502,7 +502,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks)
 
     if (tpd) {
 	if (!tpd->is_temporary)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "%s:%d:%s(): Double register of thread\n",
 		     __FILE__, __LINE__, __func__);
 	is_blocking = tpd->is_blocking;
@@ -524,7 +524,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks)
 #endif
     ASSERT(tpd->id >= 0);
     if (tpd->id >= intrnl->unmanaged.no)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "%s:%d:%s(): Too many unmanaged registered threads\n",
 		 __FILE__, __LINE__, __func__);
 
@@ -547,7 +547,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
 
     if (tpd) {
 	if (!tpd->is_temporary)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "%s:%d:%s(): Double register of thread\n",
 		     __FILE__, __LINE__, __func__);
 	is_blocking = tpd->is_blocking;
@@ -568,7 +568,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
 	tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id);
     ASSERT(tpd->id >= 0);
     if (tpd->id >= intrnl->managed.no)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "%s:%d:%s(): Too many managed registered threads\n",
 		 __FILE__, __LINE__, __func__);
 
@@ -1033,7 +1033,7 @@ has_reached_wakeup(ErtsThrPrgrVal wakeup)
 	    limit += 1;
 
 	if (!erts_thr_progress_has_passed__(limit, wakeup))
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "Invalid wakeup request value found:"
 		     " current=%b64u, wakeup=%b64u, limit=%b64u",
 		     current, wakeup, limit);
@@ -1102,7 +1102,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value)
     ix = erts_atomic32_inc_read_nob(&mwd->len) - 1;
 #if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE
     if (ix >= intrnl->managed.no)
-	erl_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n");
+	erts_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n");
 #endif
     mwd->id[ix] = tpd->id;
 
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index 3a91ca9dbe..7ff456b915 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -224,7 +224,7 @@ ErtsThrQCleanState_t
 erts_thr_q_destroy(ErtsThrQ_t *q)
 {
     if (!q->q.blk)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Trying to destroy not created thread queue\n");
     return erts_thr_q_finalize(q);
 }
@@ -589,7 +589,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
 
 #if ERTS_THR_Q_DBG_CHK_DATA
     if (!data)
-	erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n");
+	erts_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n");
 #endif
 
     ASSERT(!q->q.finalizing);
@@ -771,7 +771,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q)
 #if ERTS_THR_Q_DBG_CHK_DATA
     head->data.ptr = NULL;
     if (!res)
-	erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n");
+	erts_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n");
 #endif
     clean(q,
 	  (q->head.deq_fini.automatic
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 7327e0b48c..b88a0a813d 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -286,7 +286,7 @@ read_corrected_time(int os_drift_corrected)
 	ci = time_sup.inf.c.parmon.cdata.insts.curr;
     else {
 	if (os_mtime < time_sup.inf.c.parmon.cdata.insts.prev.os_mtime)
-	    erl_exit(ERTS_ABORT_EXIT,
+	    erts_exit(ERTS_ABORT_EXIT,
 		     "OS monotonic time stepped backwards\n");
 	ci = time_sup.inf.c.parmon.cdata.insts.prev;
     }
@@ -378,7 +378,7 @@ check_time_correction(void *vesdp)
     erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
 
     if (os_mtime < ci.os_mtime)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "OS monotonic time stepped backwards\n");
 
     erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, &mdiff,
@@ -795,7 +795,7 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep)
     erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
 
     if (os_mtime < ci.os_mtime)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "OS monotonic time stepped backwards\n");
 
     return calc_corrected_erl_mtime(os_mtime, &ci, NULL,
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index e1b03a057f..214d8b641e 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -420,7 +420,7 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
 
     erts_encode_ext(message, &ptr);
     if (!(ptr <= buffer+size)) {
-	erl_exit(1, "Internal error in do_send_to_port: %d\n", ptr-buffer);
+	erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer);
     }
 
 #ifndef ERTS_SMP
@@ -1098,7 +1098,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
     case SEQ_TRACE_PRINT:   type_atom = am_print; break;
     case SEQ_TRACE_RECEIVE: type_atom = am_receive; break;
     default:
-	erl_exit(1, "invalid type in seq_trace_output_generic: %d:\n", type);
+	erts_exit(ERTS_ERROR_EXIT, "invalid type in seq_trace_output_generic: %d:\n", type);
 	return;			/* To avoid warning */
     }
 
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 551717139d..36d85b0a22 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2507,7 +2507,7 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
 		((Uint) (bytes[3] & ((byte) 0x3F)));
 	    bytes += 4;
 	} else {
-	    erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1");
+	    erts_exit(ERTS_ERROR_EXIT,"Internal unicode error in prim_file:internal_name2native/1");
 	}
 	*target++ = (byte) (unipoint & 0xFF);
 	*target++ = (byte) ((unipoint >> 8) & 0xFF);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index a85aa15403..ffe3303796 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -570,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext)
     *ep++ = VERSION_MAGIC;
     ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL);
     if (!ep)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "%s:%d:erts_encode_ext(): Internal data structure error\n",
 		 __FILE__, __LINE__);
     *ext = ep;
@@ -1559,7 +1559,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
             b2t_destroy_context(ctx);
 
             if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) {
-                erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n",
+                erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n",
                          __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end);
             }
 	    erts_factory_close(&ctx->u.dc.factory);
@@ -1708,12 +1708,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl
 
 	if ((endp = enc_term(NULL, Term, bytes, flags, NULL))
 	    == NULL) {
-	    erl_exit(1, "%s, line %d: bad term: %x\n",
+	    erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n",
 		     __FILE__, __LINE__, Term);
 	}
 	real_size = endp - bytes;
 	if (real_size > size) {
-	    erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
+	    erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n",
 		     __FILE__, __LINE__, real_size - size);
 	}
 
@@ -1753,12 +1753,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl
 	bytes[0] = VERSION_MAGIC;
 	if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL))
 	    == NULL) {
-	    erl_exit(1, "%s, line %d: bad term: %x\n",
+	    erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n",
 		     __FILE__, __LINE__, Term);
 	}
 	real_size = endp - bytes;
 	if (real_size > size) {
-	    erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
+	    erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n",
 		     __FILE__, __LINE__, endp - (bytes + size));
 	}
 	return erts_realloc_binary(bin, real_size);
@@ -2650,7 +2650,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    ASSERT(node_sz < 17);
 		    break;
 		default:
-		    erl_exit(1, "bad header\r\n");
+		    erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
 		}
 
 		ptr++;
@@ -4110,7 +4110,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 		    ASSERT(node_sz < 17);
 		    break;
 		default:
-		    erl_exit(1, "bad header\r\n");
+		    erts_exit(ERTS_ERROR_EXIT, "bad header\r\n");
 		}
 
 		ptr++;
@@ -4202,7 +4202,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    break;
 
 	default:
-	    erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n",
+	    erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n",
 		     obj);
 	}
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ec9296d034..1b8595fe57 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -428,7 +428,7 @@ void erl_grow_estack(ErtsEStack*, Uint need);
 #define ESTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
     if ((s).start != ESTK_DEF_STACK(s)) {				\
-	erl_exit(1, "Internal error - trying to change allocator "	\
+	erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator "	\
 		 "type of active estack\n");				\
     }									\
     (s).alloc_type = (t);						\
@@ -589,7 +589,7 @@ do {	 	                                                  \
 #define WSTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
     if (s.wstart != WSTK_DEF_STACK(s)) {				\
-	erl_exit(1, "Internal error - trying to change allocator "	\
+	erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator "	\
 		 "type of active wstack\n");				\
     }									\
     s.alloc_type = (t);							\
@@ -781,7 +781,7 @@ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */                    \
 #define PSTACK_CHANGE_ALLOCATOR(s,t)					\
 do {									\
     if (s.pstart != (byte*)PSTK_DEF_STACK(s)) {				\
-	erl_exit(1, "Internal error - trying to change allocator "	\
+	erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator "	\
 		 "type of active pstack\n");				\
     }									\
     s.alloc_type = (t);							\
@@ -952,8 +952,8 @@ double erts_get_positive_zero_float(void);
 
 /* config.c */
 
-__decl_noreturn void __noreturn erl_exit(int n, char*, ...);
-__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...);
+__decl_noreturn void __noreturn erts_exit(int n, char*, ...);
+__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...);
 void erl_error(char*, va_list);
 
 /* copy.c */
@@ -1398,7 +1398,7 @@ erts_alloc_message_heap_state(Uint size,
 #endif
 
     if (size > (Uint) INT_MAX)
-	erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);
+	erts_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);
 
     if (
 #if defined(ERTS_SMP)
diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index e0fde337f2..895fe657d1 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -133,7 +133,7 @@ Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions
     while (h_size_table[ix] != -1 && h_size_table[ix] < size)
 	ix++;
     if (h_size_table[ix] == -1)
-	erl_exit(1, "panic: too large hash table size (%d)\n", size);
+	erts_exit(ERTS_ERROR_EXIT, "panic: too large hash table size (%d)\n", size);
 
     size = h_size_table[ix];
     sz = size*sizeof(HashBucket*);
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index 06d0b5123d..5f6ea14732 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -84,7 +84,7 @@ index_put_entry(IndexTable* t, void* tmpl)
 	Uint sz;
 	if (ix >= t->limit) {
 	    /* A core dump is unnecessary */
-	    erl_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n",
+	    erts_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n",
 		     t->htable.name, t->limit);
 	}
 	sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*);
@@ -123,7 +123,7 @@ void erts_index_merge(Hash* src, IndexTable* dst)
 	    ix = dst->entries++;
 	    if (ix >= dst->size) {
 		if (ix >= dst->limit) {
-		    erl_exit(1, "no more index entries in %s (max=%d)\n",
+		    erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n",
 			     dst->htable.name, dst->limit);
 		}
 		sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c64c8802b9..e8b25d42a2 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3466,7 +3466,7 @@ terminate_port(Port *prt)
     if ((state & ERTS_PORT_SFLG_HALT)
 	&& (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
 	erts_port_release(prt); /* We will exit and never return */
-	erl_exit_flush_async(erts_halt_code, "");
+	erts_flush_async_exit(erts_halt_code, "");
     }
     if (is_internal_port(send_closed_port_id))
 	deliver_result(send_closed_port_id, connected_id, am_closed);
@@ -4881,7 +4881,7 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
 
     if (len > (Uint) INT_MAX)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "Absurdly large data buffer (%beu bytes) passed to"
 		 "output callback of %s driver.\n",
 		 len,
@@ -7289,7 +7289,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
      * of ErlDrvSysInfo (introduced in driver version 1.0).
      */
     if (!sip || si_size < ERL_DRV_SYS_INFO_SIZE(smp_support)) 
-	erl_exit(1,
+	erts_exit(ERTS_ERROR_EXIT,
 		 "driver_system_info(%p, %ld) called with invalid arguments\n",
 		 sip, si_size);
 
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 7ade8bca0f..020e61d136 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -125,7 +125,7 @@ static RegProc* reg_alloc(RegProc *tmpl)
 {
     RegProc* obj = (RegProc*) erts_alloc(ERTS_ALC_T_REG_PROC, sizeof(RegProc));
     if (!obj) {
-	erl_exit(1, "Can't allocate %d bytes of memory\n", sizeof(RegProc));
+	erts_exit(ERTS_ERROR_EXIT, "Can't allocate %d bytes of memory\n", sizeof(RegProc));
     }
     obj->name = tmpl->name;
     obj->p = tmpl->p;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index ec94e3a596..ae8617f3a0 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -611,15 +611,16 @@ static unsigned long zero_value = 0, one_value = 1;
 #  endif /* !__WIN32__ */
 #endif /* WANT_NONBLOCKING */
 
-__decl_noreturn void __noreturn erl_exit(int n, char*, ...);
+__decl_noreturn void __noreturn erts_exit(int n, char*, ...);
 
-/* Some special erl_exit() codes: */
-#define ERTS_INTR_EXIT	INT_MIN		/* called from signal handler */
-#define ERTS_ABORT_EXIT	(INT_MIN + 1)	/* no crash dump; only abort() */
-#define ERTS_DUMP_EXIT	(INT_MIN + 2)	/* crash dump; then exit() */
+/* Some special erts_exit() codes: */
+#define ERTS_INTR_EXIT	-1		/* called from signal handler */
+#define ERTS_ABORT_EXIT	-2	        /* no crash dump; only abort() */
+#define ERTS_DUMP_EXIT	-3              /* crash dump; then exit() */
+#define ERTS_ERROR_EXIT	-4              /* crash dump; then abort() */
 
 #define ERTS_INTERNAL_ERROR(What) \
-    erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
+    erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
 	     __FILE__, __LINE__, __func__, What)
 
 Eterm erts_check_io_info(void *p);
@@ -929,7 +930,7 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
 #ifdef ERTS_REFC_DEBUG
     erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
     if (val < min_val)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #else
@@ -943,7 +944,7 @@ erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
     erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #endif
@@ -956,7 +957,7 @@ erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
 #ifdef ERTS_REFC_DEBUG
     erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
     if (val < min_val)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #else
@@ -970,7 +971,7 @@ erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
     erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #endif
@@ -983,7 +984,7 @@ erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
 #ifdef ERTS_REFC_DEBUG
     erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
     if (val < min_val)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
 		 diff, val, min_val);
 #else
@@ -997,7 +998,7 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
     erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
-	erl_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #endif
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 0ab6661c9f..988338ed81 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -555,7 +555,7 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
     itime = erts_init_time_sup(time_correction, time_warp_mode);
 #ifdef TIW_ITIME_IS_CONSTANT 
     if (itime != TIW_ITIME) {
-	erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME);
+	erts_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME);
     }
 #else
     tiw_itime = itime;
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 5286391746..b9ce70e364 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1055,7 +1055,7 @@ tail_recur:
 	}    
 	
     default:
-	erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
+	erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
 	return 0;
       }
       if (WSTACK_ISEMPTY(stack)) break;
@@ -1328,7 +1328,7 @@ make_hash2(Eterm term)
                     i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
                     break;
                 default:
-                    erl_exit(1, "bad header");
+                    erts_exit(ERTS_ERROR_EXIT, "bad header");
                 }
                 while (i) {
                     if (is_list(*ptr)) {
@@ -1491,7 +1491,7 @@ make_hash2(Eterm term)
 	    break;
 		    
 	    default:
-		erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 	    }
 	}
 	break;
@@ -1522,7 +1522,7 @@ make_hash2(Eterm term)
 			UINT32_HASH(NIL_DEF, HCONST_2);
 		    goto hash2_common;
 		default:
-		    erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+		    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 		}
 	    case _TAG_IMMED1_SMALL:
 	      {
@@ -1538,7 +1538,7 @@ make_hash2(Eterm term)
 	    }
 	    break;
 	default:
-	    erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 	hash2_common:
 
 	    /* Uint32 hash always has the hash value of the previous term,
@@ -1733,7 +1733,7 @@ make_internal_hash(Eterm term)
                     i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
                     break;
                 default:
-                    erl_exit(1, "bad header");
+                    erts_exit(ERTS_ERROR_EXIT, "bad header");
                 }
                 while (i) {
                     if (is_list(*ptr)) {
@@ -1909,7 +1909,7 @@ make_internal_hash(Eterm term)
 		goto pop_next;
 	    }
 	    default:
-		erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 	    }
 	}
 	break;
@@ -1922,7 +1922,7 @@ make_internal_hash(Eterm term)
             goto pop_next;
 
 	default:
-	    erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 
 	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
@@ -2205,7 +2205,7 @@ tail_recur:
 	}
 
     default:
-	erl_exit(1, "Invalid tag in make_broken_hash\n");
+	erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n");
 	return 0;
       }
       if (WSTACK_ISEMPTY(stack)) break;
@@ -2879,7 +2879,7 @@ tailrecur_ne:
 			ASSERT(sz > 0 && sz < 17);
 			break;
 		    default:
-			erl_exit(1, "Unknown hashmap subsubtag\n");
+			erts_exit(ERTS_ERROR_EXIT, "Unknown hashmap subsubtag\n");
 		    }
 		    goto term_array;
 		}
-- 
cgit v1.2.3


From 1b094d72ffc56069c72f17c7edd673dbbfe47e39 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 23 Feb 2016 21:05:03 +0100
Subject: erts: Make erlang:halt() accept bignums as Status

Just mask away the high bits to get a more tolerant erlang:halt
that behaves the same on 32 and 64 bit architectures.
---
 erts/emulator/beam/bif.c | 12 ++++++------
 erts/emulator/beam/big.c | 26 ++++++++++++++++++++++++++
 erts/emulator/beam/big.h |  1 +
 3 files changed, 33 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index a61025d19a..66c2853534 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3938,10 +3938,10 @@ static char halt_msg[HALT_MSG_SIZE];
 /* ARGSUSED */
 BIF_RETTYPE halt_1(BIF_ALIST_1)
 {
-    Sint code;
+    Uint code;
     
-    if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
-	int pos_int_code = (int)code & INT_MAX;
+    if (term_to_Uint_mask(BIF_ARG_1, &code)) {
+	int pos_int_code = (int) (code & INT_MAX);
 	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
 	erts_halt(pos_int_code);
 	ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
@@ -3975,7 +3975,7 @@ BIF_RETTYPE halt_1(BIF_ALIST_1)
 /* ARGSUSED */
 BIF_RETTYPE halt_2(BIF_ALIST_2)
 {
-    Sint code;
+    Uint code;
     Eterm optlist = BIF_ARG_2;
     int flush = 1;
 
@@ -4002,8 +4002,8 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
     if (is_not_nil(optlist))
 	goto error;
 
-    if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
-	int pos_int_code = (int)code & INT_MAX;
+    if (term_to_Uint_mask(BIF_ARG_1, &code)) {
+	int pos_int_code = (int) (code & INT_MAX);
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
 	if (flush) {
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 15bcd44fb9..87d3be2b0f 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2028,6 +2028,32 @@ term_to_Uint(Eterm term, Uint *up)
     }
 }
 
+/* same as term_to_Uint()
+   but also accept larger bignums by masking
+ */
+int
+term_to_Uint_mask(Eterm term, Uint *up)
+{
+    if (is_small(term)) {
+	Sint i = signed_val(term);
+	if (i < 0) {
+	    *up = BADARG;
+	    return 0;
+	}
+	*up = (Uint) i;
+	return 1;
+    } else if (is_big(term) && !big_sign(term)) {
+	ErtsDigit* xr = big_v(term);
+
+	ERTS_CT_ASSERT(sizeof(ErtsDigit) == sizeof(Uint));
+	*up = (Uint)*xr;  /* just pick first word */
+	return 1;
+    } else {
+	*up = BADARG;
+	return 0;
+    }
+}
+
 int
 term_to_UWord(Eterm term, UWord *up)
 {
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 4aa9724ae3..3c1f1e6b23 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -161,6 +161,7 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*);
 byte* big_to_bytes(Eterm, byte*);
 
 int term_to_Uint(Eterm, Uint*);
+int term_to_Uint_mask(Eterm, Uint*);
 int term_to_UWord(Eterm, UWord*);
 int term_to_Sint(Eterm, Sint*);
 #if HAVE_INT64
-- 
cgit v1.2.3


From 8f4c278b69fe4d613a0b865a2edac43231cad913 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 30 Nov 2015 15:35:47 +0100
Subject: Allow erlang:finish_loading/1 to load more than one module
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The BIFs prepare_loading/2 and finish_loading/1 have been
designed to allow fast loading in parallel of many modules.

Because of the complications with on_load functions,
the initial implementation of finish_loading/1 only allowed
a single element in the list of prepared modules.

finish_loading/1 does not suspend other processes, but it must wait
for all schedulers to pass a write barrier ("thread progress"). The
time for all schedulers to pass the write barrier is highly variable,
depending on what kind of code they are executing. Therefore, allowing
finish_loading/1 to finish the loading for more than one module before
passing the write barrier could potentially be much faster than
calling finish_loading/1 multiple times.

The test case many/1 run on my computer shows that with "heavy load",
finish loading of 100 modules in parallel is almost 50 times faster
than loading them sequentially. With "light load", the gain is still
almost 10 times.

Here follows an actual sample of the output from the test case on
my computer (an 2012 iMac):

Light load
==========
Sequential:     22361 µs
Parallel:        2586 µs
Ratio:              9

Heavy load
==========
Sequential:    254512 µs
Parallel:        5246 µs
Ratio:             49
---
 erts/emulator/beam/atom.names      |   1 +
 erts/emulator/beam/beam_bif_load.c |  54 +++++++++----
 erts/emulator/beam/beam_ranges.c   | 152 +++++++++++++++++--------------------
 erts/emulator/beam/code_ix.c       |   4 +-
 erts/emulator/beam/code_ix.h       |   2 +-
 erts/emulator/beam/global.h        |   2 +-
 erts/emulator/beam/module.h        |   1 +
 7 files changed, 112 insertions(+), 104 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 6fb08ee896..a00b22eac0 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -211,6 +211,7 @@ atom dsend
 atom dsend_continue_trap
 atom dunlink
 atom duplicate_bag
+atom duplicated
 atom dupnames
 atom einval
 atom elib_malloc
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 1b4c022370..d426b6eebb 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -86,7 +86,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
 	ASSERT(modp->curr.num_breakpoints == 0);
     }
 
-    erts_start_staging_code_ix();
+    erts_start_staging_code_ix(1);
 
     res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 
@@ -163,14 +163,13 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions)
     Eterm* hp = HAlloc(p, 3 + 2*exceptions);
     Eterm res = NIL;
 
-    mp += exceptions - 1;
     while (exceptions > 0) {
 	if (mp->exception) {
 	    res = CONS(hp, mp->module, res);
 	    hp += 2;
 	    exceptions--;
 	}
-	mp--;
+	mp++;
     }
     return TUPLE2(hp, tag, res);
 }
@@ -202,8 +201,12 @@ finish_loading_1(BIF_ALIST_1)
 
     n = erts_list_length(BIF_ARG_1);
     if (n < 0) {
-	ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
-	goto done;
+    badarg:
+	if (p) {
+	    erts_free(ERTS_ALC_T_LOADER_TMP, p);
+	}
+	erts_release_code_write_permission();
+	BIF_ERROR(BIF_P, BADARG);
     }
     p = erts_alloc(ERTS_ALC_T_LOADER_TMP, n*sizeof(struct m));
 
@@ -218,29 +221,32 @@ finish_loading_1(BIF_ALIST_1)
 	ProcBin* pb;
 
 	if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
-	    ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
-	    goto done;
+	    goto badarg;
 	}
 	pb = (ProcBin*) binary_val(term);
 	p[i].code = pb->val;
 	p[i].module = erts_module_for_prepared_code(p[i].code);
 	if (p[i].module == NIL) {
-	    ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
-	    goto done;
+	    goto badarg;
 	}
 	BIF_ARG_1 = CDR(cons);
     }
 
     /*
      * Since we cannot handle atomic loading of a group of modules
-     * if one or more of them uses on_load, we will only allow one
-     * element in the list. This limitation is intended to be
-     * lifted in the future.
+     * if one or more of them uses on_load, we will only allow
+     * more than one element in the list if none of the modules
+     * have an on_load function.
      */
 
     if (n > 1) {
-	ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT);
-	goto done;
+	for (i = 0; i < n; i++) {
+	    if (erts_has_code_on_load(p[i].code) == am_true) {
+		erts_free(ERTS_ALC_T_LOADER_TMP, p);
+		erts_release_code_write_permission();
+		BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+	    }
+	}
     }
 
     /*
@@ -252,11 +258,27 @@ finish_loading_1(BIF_ALIST_1)
      */
 
     res = am_ok;
-    erts_start_staging_code_ix();
+    erts_start_staging_code_ix(n);
 
     for (i = 0; i < n; i++) {
 	p[i].modp = erts_put_module(p[i].module);
+	p[i].modp->seen = 0;
+    }
+
+    exceptions = 0;
+    for (i = 0; i < n; i++) {
+	p[i].exception = 0;
+	if (p[i].modp->seen) {
+	    p[i].exception = 1;
+	    exceptions++;
+	}
+	p[i].modp->seen = 1;
     }
+    if (exceptions) {
+	res = exception_list(BIF_P, am_duplicated, p, exceptions);
+	goto done;
+    }
+
     for (i = 0; i < n; i++) {
 	if (p[i].modp->curr.num_breakpoints > 0 ||
 	    p[i].modp->curr.num_traced_exports > 0 ||
@@ -492,7 +514,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
     }
 
     {
-	erts_start_staging_code_ix();
+	erts_start_staging_code_ix(0);
 	code_ix = erts_staging_code_ix();
 	modp = erts_get_module(BIF_ARG_1, code_ix);
 	if (!modp) {
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 5a2b66727a..54c337ee72 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -53,6 +53,7 @@ struct ranges {
 };
 static struct ranges r[ERTS_NUM_CODE_IX];
 static erts_smp_atomic_t mem_used;
+static Range* write_ptr;
 
 #ifdef HARD_DEBUG
 static void check_consistency(struct ranges* p)
@@ -72,6 +73,17 @@ static void check_consistency(struct ranges* p)
 #  define CHECK(r)
 #endif /* HARD_DEBUG */
 
+static int
+rangecompare(Range* a, Range* b)
+{
+    if (a->start < b->start) {
+	return -1;
+    } else if (a->start == b->start) {
+	return 0;
+    } else {
+	return 1;
+    }
+}
 
 void
 erts_init_ranges(void)
@@ -88,45 +100,70 @@ erts_init_ranges(void)
 }
 
 void
-erts_start_staging_ranges(void)
+erts_start_staging_ranges(int num_new)
 {
+    ErtsCodeIndex src = erts_active_code_ix();
     ErtsCodeIndex dst = erts_staging_code_ix();
+    Sint need;
 
     if (r[dst].modules) {
 	erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated);
 	erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules);
-	r[dst].modules = NULL;
     }
+
+    need = r[dst].allocated = r[src].n + num_new;
+    erts_smp_atomic_add_nob(&mem_used, need);
+    write_ptr = erts_alloc(ERTS_ALC_T_MODULE_REFS,
+			   need * sizeof(Range));
+    r[dst].modules = write_ptr;
 }
 
 void
 erts_end_staging_ranges(int commit)
 {
-    ErtsCodeIndex dst = erts_staging_code_ix();
-
-    if (commit && r[dst].modules == NULL) {
+    if (commit) {
 	Sint i;
-	Sint n;
-
-	/* No modules added, just clone src and remove purged code. */
 	ErtsCodeIndex src = erts_active_code_ix();
+	ErtsCodeIndex dst = erts_staging_code_ix();
+	Range* mp;
+	Sint num_inserted;
 
-	erts_smp_atomic_add_nob(&mem_used, r[src].n);
-	r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS,
-				    r[src].n * sizeof(Range));
-	r[dst].allocated = r[src].n;
-	n = 0;
+	mp = r[dst].modules;
+	num_inserted = write_ptr - mp;
 	for (i = 0; i < r[src].n; i++) {
 	    Range* rp = r[src].modules+i;
 	    if (rp->start < RANGE_END(rp)) {
 		/* Only insert a module that has not been purged. */
-		r[dst].modules[n] = *rp;
-		n++;
+		write_ptr->start = rp->start;
+		erts_smp_atomic_init_nob(&write_ptr->end,
+					 (erts_aint_t)(RANGE_END(rp)));
+		write_ptr++;
+	    }
+	}
+
+	/*
+	 * There are num_inserted new range entries (unsorted) at the
+	 * beginning of the modules array, followed by the old entries
+	 * (sorted). We must now sort the entire array.
+	 */
+
+	r[dst].n = write_ptr - mp;
+	if (num_inserted > 1) {
+	    qsort(mp, r[dst].n, sizeof(Range),
+		  (int (*)(const void *, const void *)) rangecompare);
+	} else if (num_inserted == 1) {
+	    /* Sift the new range into place. This is faster than qsort(). */
+	    Range t = mp[0];
+	    for (i = 0; i < r[dst].n-1 && t.start > mp[i+1].start; i++) {
+		mp[i] = mp[i+1];
 	    }
+	    mp[i] = t;
 	}
-	r[dst].n = n;
+	r[dst].modules = mp;
+	CHECK(&r[dst]);
 	erts_smp_atomic_set_nob(&r[dst].mid,
-				(erts_aint_t) (r[dst].modules + n / 2));
+				(erts_aint_t) (r[dst].modules +
+					       r[dst].n / 2));
     }
 }
 
@@ -135,82 +172,29 @@ erts_update_ranges(BeamInstr* code, Uint size)
 {
     ErtsCodeIndex dst = erts_staging_code_ix();
     ErtsCodeIndex src = erts_active_code_ix();
-    Sint i;
-    Sint n;
-    Sint need;
 
     if (src == dst) {
 	ASSERT(!erts_initialized);
 
 	/*
-	 * During start-up of system, the indices are the same.
-	 * Handle this by faking a source area.
+	 * During start-up of system, the indices are the same
+	 * and erts_start_staging_ranges() has not been called.
 	 */
-	src = (src+1) % ERTS_NUM_CODE_IX;
-	if (r[src].modules) {
-	    erts_smp_atomic_add_nob(&mem_used, -r[src].allocated);
-	    erts_free(ERTS_ALC_T_MODULE_REFS, r[src].modules);
+	if (r[dst].modules == NULL) {
+	    Sint need = 128;
+	    erts_smp_atomic_add_nob(&mem_used, need);
+	    r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS,
+					need * sizeof(Range));
+	    r[dst].allocated = need;
+	    write_ptr = r[dst].modules;
 	}
-	r[src] = r[dst];
-	r[dst].modules = 0;
     }
 
-    CHECK(&r[src]);
-
-    ASSERT(r[dst].modules == NULL);
-    need = r[dst].allocated = r[src].n + 1;
-    erts_smp_atomic_add_nob(&mem_used, need);
-    r[dst].modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS,
-					  need * sizeof(Range));
-    n = 0;
-    for (i = 0; i < r[src].n; i++) {
-	Range* rp = r[src].modules+i;
-	if (code < rp->start) {
-	    r[dst].modules[n].start = code;
-	    erts_smp_atomic_init_nob(&r[dst].modules[n].end,
-				     (erts_aint_t)(((byte *)code) + size));
-	    ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code);
-	    n++;
-	    break;
-	}
-	if (rp->start < RANGE_END(rp)) {
-	    /* Only insert a module that has not been purged. */
-	    r[dst].modules[n].start = rp->start;
-	    erts_smp_atomic_init_nob(&r[dst].modules[n].end,
-				     (erts_aint_t)(RANGE_END(rp)));
-	    ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start);
-	    n++;
-	}
-    }
-
-    while (i < r[src].n) {
-	Range* rp = r[src].modules+i;
-	if (rp->start < RANGE_END(rp)) {
-	    /* Only insert a module that has not been purged. */
-	    r[dst].modules[n].start = rp->start;
-	    erts_smp_atomic_init_nob(&r[dst].modules[n].end,
-				     (erts_aint_t)(RANGE_END(rp)));
-	    ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start);
-	    n++;
-	}
-	i++;
-    }
-
-    if (n == 0 || code > r[dst].modules[n-1].start) {
-	r[dst].modules[n].start = code;
-	erts_smp_atomic_init_nob(&r[dst].modules[n].end,
-				 (erts_aint_t)(((byte *)code) + size));
-	ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code);
-	n++;
-    }
-
-    ASSERT(n <= r[src].n+1);
-    r[dst].n = n;
-    erts_smp_atomic_set_nob(&r[dst].mid,
-			    (erts_aint_t) (r[dst].modules + n / 2));
-
-    CHECK(&r[dst]);
-    CHECK(&r[src]);
+    ASSERT(r[dst].modules);
+    write_ptr->start = code;
+    erts_smp_atomic_init_nob(&(write_ptr->end),
+			     (erts_aint_t)(((byte *)code) + size));
+    write_ptr++;
 }
 
 void
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 209d008d18..1b0968c55c 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -65,12 +65,12 @@ void erts_code_ix_init(void)
     CIX_TRACE("init");
 }
 
-void erts_start_staging_code_ix(void)
+void erts_start_staging_code_ix(int num_new)
 {
     beam_catches_start_staging();
     export_start_staging();
     module_start_staging();
-    erts_start_staging_ranges();
+    erts_start_staging_ranges(num_new);
     CIX_TRACE("start");
 }
 
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 5f00b409ef..7a66211a5b 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -100,7 +100,7 @@ void erts_release_code_write_permission(void);
  * Must be followed by calls to either "end" and "commit" or "abort" before
  * code write permission can be released.
  */
-void erts_start_staging_code_ix(void);
+void erts_start_staging_code_ix(int num_new);
 
 /* End the staging.
  * Preceded by "start" and must be followed by "commit".
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b174fa3e8a..956efa5e36 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1027,7 +1027,7 @@ Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
 
 /* beam_ranges.c */
 void erts_init_ranges(void);
-void erts_start_staging_ranges(void);
+void erts_start_staging_ranges(int num_new);
 void erts_end_staging_ranges(int commit);
 void erts_update_ranges(BeamInstr* code, Uint size);
 void erts_remove_from_ranges(BeamInstr* code);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index e66d628ca9..b7468b0926 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -37,6 +37,7 @@ struct erl_module_instance {
 typedef struct erl_module {
     IndexSlot slot;		/* Must be located at top of struct! */
     int module;			/* Atom index for module (not tagged). */
+    int seen;			/* Used by finish_loading() */
 
     struct erl_module_instance curr;
     struct erl_module_instance old; /* protected by "old_code" rwlock */
-- 
cgit v1.2.3


From 272985d3d1b36833ebd586e5f051ac3a4ab0cd91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 3 Dec 2015 15:42:02 +0100
Subject: Add has_prepared_code_on_load/1 BIF

---
 erts/emulator/beam/beam_bif_load.c | 19 +++++++++++++++++++
 erts/emulator/beam/bif.tab         |  1 +
 2 files changed, 20 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index d426b6eebb..0b47fc3586 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -139,6 +139,25 @@ prepare_loading_2(BIF_ALIST_2)
     BIF_RET(res);
 }
 
+BIF_RETTYPE
+has_prepared_code_on_load_1(BIF_ALIST_1)
+{
+    Eterm res;
+    ProcBin* pb;
+
+    if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1)) {
+    error:
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    pb = (ProcBin*) binary_val(BIF_ARG_1);
+    res = erts_has_code_on_load(pb->val);
+    if (res == NIL) {
+	goto error;
+    }
+    BIF_RET(res);
+}
+
 struct m {
     Binary* code;
     Eterm module;
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 4efc055aaf..5c84b9c0eb 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -649,6 +649,7 @@ bif binary:split/2
 bif binary:split/3
 bif erts_debug:size_shared/1
 bif erts_debug:copy_shared/1
+bif erlang:has_prepared_code_on_load/1
 
 #
 # Obsolete
-- 
cgit v1.2.3


From e4fd76bdd9dd15547531947aaecf487385a6d796 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 8 Mar 2016 17:15:18 +0100
Subject: erts: Fix alloc_SUITE:rbtree and migration for win64

One little (unsigned long) left behind.
---
 erts/emulator/beam/erl_alloc_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 18312eacde..d9eaa7b32c 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -6049,7 +6049,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
     case 0x019:	return (UWord) PREV_BLK((Block_t *) a1);
     case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2);
     case 0x01b: return (UWord) sizeof(Unit_t);
-    case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1);
+    case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1);
     case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
     case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
 #ifdef ERTS_SMP
-- 
cgit v1.2.3


From 207a569d4caf80b10a54eec2220ac672a3377f00 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 22 Jan 2016 17:48:49 +0100
Subject: Improved scheduler suspend functionality

- The calling process is now suspended while synchronizing
  scheduler suspends via erlang:system_flag(schedulers_online, _)
  and erlang:system_flag(multi_scheduling, _), instead of blocking
  the scheduler thread in the BIF call waiting for the operation
  to synchronize. Besides releasing the scheduler for other work
  (or immediate suspend) it also makes it possible to abort the
  operation by killing the process.
- erlang:system_flag(schedulers_online, _) now only wait for normal
  schedulers to complete before it returns. This since it may take
  a very long time before all dirty schedulers suspends.
- erlang:system_flag(multi_scheduling, block_normal|unblock_normal)
  which only operate on normal schedulers has been introduced. This
  since there are use cases where suspend of dirty schedulers are
  not of interest (hipe loader).
- erlang:system_flag(multi_scheduling, block) still blocks all
  dirty schedulers as well as all normal schedulers except one since
  it is hard to redefine what multi scheduling block means.
- The three operations:
  - changing amount of schedulers online
  - blocking/unblocking normal multi scheduling
  - blocking/unblocking full multi scheduling
  can now be done in parallel. This is important since otherwise
  a full multi scheduling block would potentially delay the other
  operations for a very long time.
---
 erts/emulator/beam/atom.names     |    6 +
 erts/emulator/beam/bif.c          |   66 +-
 erts/emulator/beam/bif.h          |   25 +-
 erts/emulator/beam/erl_bif_info.c |  172 ++-
 erts/emulator/beam/erl_process.c  | 2162 +++++++++++++++++--------------------
 erts/emulator/beam/erl_process.h  |   26 +-
 6 files changed, 1171 insertions(+), 1286 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index dca8b503bf..169b071cd7 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -125,7 +125,9 @@ atom binary_longest_suffix_trap
 atom binary_to_list_continue
 atom binary_to_term_trap
 atom block
+atom block_normal
 atom blocked
+atom blocked_normal
 atom bm
 atom bnot
 atom bor
@@ -190,7 +192,9 @@ atom dexit
 atom depth
 atom dgroup_leader
 atom dictionary
+atom dirty_cpu
 atom dirty_cpu_schedulers_online
+atom dirty_io
 atom disable_trace
 atom disabled
 atom display_items
@@ -305,6 +309,7 @@ atom index
 atom infinity
 atom info
 atom info_msg
+atom init
 atom initial_call
 atom input
 atom internal
@@ -612,6 +617,7 @@ atom use_stdio
 atom used
 atom utf8
 atom unblock
+atom unblock_normal
 atom uniq
 atom unless_suspending
 atom unloaded
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 6ab75496a0..37bb28c6f8 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4197,22 +4197,33 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
     Sint n;
 
     if (BIF_ARG_1 == am_multi_scheduling) {
-	if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock) {
+	if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock
+	    || BIF_ARG_2 == am_block_normal || BIF_ARG_2 == am_unblock_normal) {
 #ifndef ERTS_SMP
 	    BIF_RET(am_disabled);
 #else
+	    int block = (BIF_ARG_2 == am_block
+			 || BIF_ARG_2 == am_block_normal);
+	    int normal = (BIF_ARG_2 == am_block_normal
+			  || BIF_ARG_2 == am_unblock_normal);
 	    if (erts_no_schedulers == 1)
 		BIF_RET(am_disabled);
 	    else {
 		switch (erts_block_multi_scheduling(BIF_P,
 						    ERTS_PROC_LOCK_MAIN,
-						    BIF_ARG_2 == am_block,
+						    block,
+						    normal,
 						    0)) {
 		case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
 		    BIF_RET(am_blocked);
+		case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
+		    BIF_RET(am_blocked_normal);
 		case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
 		    ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked,
 					    am_multi_scheduling);
+		case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
+		    ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal,
+					    am_multi_scheduling);
 		case ERTS_SCHDLR_SSPND_DONE:
 		    BIF_RET(am_enabled);
 		case ERTS_SCHDLR_SSPND_YIELD_RESTART:
@@ -4245,11 +4256,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	switch (erts_set_schedulers_online(BIF_P,
 					   ERTS_PROC_LOCK_MAIN,
 					   signed_val(BIF_ARG_2),
-					   &old_no
-#ifdef ERTS_DIRTY_SCHEDULERS
-					   , 0
-#endif
-					   )) {
+					   &old_no, 0)) {
 	case ERTS_SCHDLR_SSPND_DONE:
 	    BIF_RET(make_small(old_no));
 	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
@@ -4619,29 +4626,27 @@ BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) {
 /*
  * Processes doing yield on return in a bif ends up in bif_return_trap().
  */
-static BIF_RETTYPE bif_return_trap(
-#ifdef DEBUG
-    BIF_ALIST_2
-#else
-    BIF_ALIST_1
-#endif
-    )
+static BIF_RETTYPE bif_return_trap(BIF_ALIST_2)
 {
-#ifdef DEBUG
+    Eterm res = BIF_ARG_1;
+
     switch (BIF_ARG_2) {
-    case am_multi_scheduling:
 #ifdef ERTS_SMP
-	erts_dbg_multi_scheduling_return_trap(BIF_P, BIF_ARG_1);
-#endif
-	break;
-    case am_schedulers_online:
+    case am_multi_scheduling: {
+	int msb = erts_is_multi_scheduling_blocked();
+	if (msb > 0)
+	    res = am_blocked;
+	else if (msb < 0)
+	    res = am_blocked_normal;
+	else
+	    ERTS_INTERNAL_ERROR("Unexpected multi scheduling block state");
 	break;
+    }
+#endif
     default:
 	break;
     }
-#endif
-
-    BIF_RET(BIF_ARG_1);
+    BIF_RET(res);
 }
 
 /*
@@ -4746,17 +4751,12 @@ void erts_init_bif(void)
     erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL);
 
     /*
-     * bif_return_trap/1 is a hidden BIF that bifs that need to
-     * yield the calling process traps to. The only thing it does:
-     * return the value passed as argument.
+     * bif_return_trap/2 is a hidden BIF that bifs that need to
+     * yield the calling process traps to.
      */
-    erts_init_trap_export(&bif_return_trap_export, am_erlang, am_bif_return_trap,
-#ifdef DEBUG
-		     2
-#else
-		     1
-#endif
-		     , &bif_return_trap);
+    erts_init_trap_export(&bif_return_trap_export,
+			  am_erlang, am_bif_return_trap, 2,
+			  &bif_return_trap);
 
     erts_await_result = erts_export_put(am_erts_internal,
 					am_await_result,
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index a62eddf36b..46af552b39 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -338,37 +338,20 @@ do {							\
  } while(0)
 
 extern Export bif_return_trap_export;
-#ifdef DEBUG
-#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL)		\
+#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, OP)			\
 do {									\
     ERTS_VBUMP_ALL_REDS(P);						\
-    ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL),	\
-			(DEBUG_VAL));					\
+    ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL), (OP));\
 } while (0)
-#else
-#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL)		\
-do {									\
-    ERTS_VBUMP_ALL_REDS(P);						\
-    ERTS_BIF_PREP_TRAP1(RET, &bif_return_trap_export, (P), (VAL));	\
-} while (0)
-#endif
 
 #define ERTS_BIF_PREP_YIELD_RETURN(RET, P, VAL) \
   ERTS_BIF_PREP_YIELD_RETURN_X(RET, (P), (VAL), am_undefined)
 
-#ifdef DEBUG
-#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL)			\
+#define ERTS_BIF_YIELD_RETURN_X(P, VAL, OP)				\
 do {									\
     ERTS_VBUMP_ALL_REDS(P);						\
-    BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (DEBUG_VAL));	\
+    BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (OP));		\
 } while (0)
-#else
-#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL)			\
-do {									\
-    ERTS_VBUMP_ALL_REDS(P);						\
-    BIF_TRAP1(&bif_return_trap_export, (P), (VAL));			\
-} while (0)
-#endif
 
 #define ERTS_BIF_RETURN_YIELD(P) ERTS_VBUMP_ALL_REDS((P))
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 39eb257c2c..8c748c9bf7 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -320,13 +320,11 @@ erts_print_system_version(int to, void *arg, Process *c_p)
     char *ov = otp_version;
 #ifdef ERTS_SMP
     Uint total, online, active;
-#ifdef ERTS_DIRTY_SCHEDULERS
     Uint dirty_cpu, dirty_cpu_onln, dirty_io;
 
-    (void) erts_schedulers_state(&total, &online, &active, &dirty_cpu, &dirty_cpu_onln, &dirty_io, 0);
-#else
-    (void) erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 0);
-#endif
+    erts_schedulers_state(&total, &online, &active,
+			  &dirty_cpu, &dirty_cpu_onln, NULL,
+			  &dirty_io, NULL);
 #endif
     for (i = 0; i < sizeof(otp_version)-4; i++) {
 	if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c')
@@ -2044,12 +2042,18 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 #ifndef ERTS_SMP
 	BIF_RET(am_disabled);
 #else
+#ifndef ERTS_DIRTY_SCHEDULERS
 	if (erts_no_schedulers == 1)
 	    BIF_RET(am_disabled);
-	else {
-	    BIF_RET(erts_is_multi_scheduling_blocked()
-		    ? am_blocked
-		    : am_enabled);
+	else
+#endif
+	{
+	    int msb = erts_is_multi_scheduling_blocked();
+	    BIF_RET(!msb
+		    ? am_enabled
+		    : (msb > 0
+		       ? am_blocked
+		       : am_blocked_normal));
 	}
 #endif
     } else if (BIF_ARG_1 == am_build_type) {
@@ -2518,77 +2522,120 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	res = TUPLE3(hp, make_small(1), make_small(1), make_small(1));
 	BIF_RET(res);
 #else
+	Eterm *hp;
 	Uint total, online, active;
-	switch (erts_schedulers_state(&total,
-				      &online,
-				      &active,
-				      NULL,
-				      NULL,
-				      NULL,
-				      1)) {
-	case ERTS_SCHDLR_SSPND_DONE: {
-	    Eterm *hp = HAlloc(BIF_P, 4);
-	    res = TUPLE3(hp,
-			 make_small(total),
-			 make_small(online),
-			 make_small(active));
-	    BIF_RET(res);
+	erts_schedulers_state(&total, &online, &active,
+			      NULL, NULL, NULL, NULL, NULL);
+	hp = HAlloc(BIF_P, 4);
+	res = TUPLE3(hp,
+		     make_small(total),
+		     make_small(online),
+		     make_small(active));
+	BIF_RET(res);
+#endif
+    } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) {
+#ifndef ERTS_SMP
+	Eterm *hp = HAlloc(BIF_P, 4);
+	res = TUPLE3(hp, make_small(1), make_small(1), make_small(1));
+	BIF_RET(res);
+#else
+	Eterm *hp;
+	Uint total, online, active;
+	erts_schedulers_state(&total, &online, &active,
+			      NULL, NULL, NULL, NULL, NULL);
+	hp = HAlloc(BIF_P, 4);
+	res = TUPLE3(hp,
+		     make_small(total),
+		     make_small(online),
+		     make_small(active));
+	BIF_RET(res);
+#endif
+    } else if (ERTS_IS_ATOM_STR("all_schedulers_state", BIF_ARG_1)) {
+#ifndef ERTS_SMP
+	Eterm *hp = HAlloc(BIF_P, 2+5);
+	res = CONS(hp+5,
+		   TUPLE4(hp,
+			  am_normal,
+			  make_small(1),
+			  make_small(1),
+			  make_small(1)),
+		   NIL);
+	BIF_RET(res);
+#else
+	Eterm *hp, tpl;
+	Uint sz, total, online, active,
+	    dirty_cpu_total, dirty_cpu_online, dirty_cpu_active,
+	    dirty_io_total, dirty_io_active;
+	erts_schedulers_state(&total, &online, &active,
+			      &dirty_cpu_total, &dirty_cpu_online, &dirty_cpu_active,
+			      &dirty_io_total, &dirty_io_active);
+
+	sz = 2+5;
+	if (dirty_cpu_total)
+	    sz += 2+5;
+	if (dirty_io_total)
+	    sz += 2+5;
+
+	hp = HAlloc(BIF_P, sz);
+
+	res = NIL;
+	if (dirty_io_total) {
+	    tpl = TUPLE4(hp,
+			 am_dirty_io,
+			 make_small(dirty_io_total),
+			 make_small(dirty_io_total),
+			 make_small(dirty_io_active));
+	    hp += 5;
+	    res = CONS(hp, tpl, res);
+	    hp += 2;
 	}
-	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
-	    ERTS_VBUMP_ALL_REDS(BIF_P);
-	    BIF_TRAP1(bif_export[BIF_system_info_1],
-		      BIF_P, BIF_ARG_1);
-	default:
-	    ASSERT(0);
-	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+	if (dirty_cpu_total) {
+	    tpl = TUPLE4(hp,
+			 am_dirty_cpu,
+			 make_small(dirty_cpu_total),
+			 make_small(dirty_cpu_online),
+			 make_small(dirty_cpu_active));
+	    hp += 5;
+	    res = CONS(hp, tpl, res);
+	    hp += 2;
 	}
+	tpl = TUPLE4(hp,
+		     am_normal,
+		     make_small(total),
+		     make_small(online),
+		     make_small(active));
+	hp += 5;
+	res = CONS(hp, tpl, res);
+	BIF_RET(res);
 #endif
     } else if (ERTS_IS_ATOM_STR("schedulers_online", BIF_ARG_1)) {
 #ifndef ERTS_SMP
 	BIF_RET(make_small(1));
 #else
-	Uint total, online, active;
-	switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) {
-	case ERTS_SCHDLR_SSPND_DONE:
-	    BIF_RET(make_small(online));
-	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
-	    ERTS_VBUMP_ALL_REDS(BIF_P);
-	    BIF_TRAP1(bif_export[BIF_system_info_1],
-		      BIF_P, BIF_ARG_1);
-	default:
-	    ASSERT(0);
-	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
-	}
+	Uint online;
+	erts_schedulers_state(NULL, &online, NULL, NULL, NULL, NULL, NULL, NULL);
+	BIF_RET(make_small(online));
 #endif
     } else if (ERTS_IS_ATOM_STR("schedulers_active", BIF_ARG_1)) {
 #ifndef ERTS_SMP
 	BIF_RET(make_small(1));
 #else
-	Uint total, online, active;
-	switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) {
-	case ERTS_SCHDLR_SSPND_DONE:
-	    BIF_RET(make_small(active));
-	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
-	    ERTS_VBUMP_ALL_REDS(BIF_P);
-	    BIF_TRAP1(bif_export[BIF_system_info_1],
-		      BIF_P, BIF_ARG_1);
-	default:
-	    ASSERT(0);
-	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
-	}
+	Uint active;
+	erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL);
+	BIF_RET(make_small(active));
 #endif
 #if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) {
 	Uint dirty_cpu;
-	erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, 1);
+	erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL);
 	BIF_RET(make_small(dirty_cpu));
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) {
 	Uint dirty_cpu_onln;
-	erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, 1);
+	erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL);
 	BIF_RET(make_small(dirty_cpu_onln));
     } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) {
 	Uint dirty_io;
-	erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, &dirty_io, 1);
+	erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL);
 	BIF_RET(make_small(dirty_io));
 #endif
     } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
@@ -2642,7 +2689,16 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	if (erts_no_schedulers == 1)
 	    BIF_RET(NIL);
 	else
-	    BIF_RET(erts_multi_scheduling_blockers(BIF_P));
+	    BIF_RET(erts_multi_scheduling_blockers(BIF_P, 0));
+#endif
+    } else if (ERTS_IS_ATOM_STR("normal_multi_scheduling_blockers", BIF_ARG_1)) {
+#ifndef ERTS_SMP
+	BIF_RET(NIL);
+#else
+	if (erts_no_schedulers == 1)
+	    BIF_RET(NIL);
+	else
+	    BIF_RET(erts_multi_scheduling_blockers(BIF_P, 1));
 #endif
     } else if (ERTS_IS_ATOM_STR("modified_timing_level", BIF_ARG_1)) {
 	BIF_RET(ERTS_USE_MODIFIED_TIMING()
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index c69cbc023c..41e471a96a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -153,10 +153,8 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
 int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
 int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
-#ifdef ERTS_DIRTY_SCHEDULERS
-Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
-Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
-#endif
+Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0;
+Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0;
 
 static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
 int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS;
@@ -188,84 +186,176 @@ int erts_disable_proc_not_running_opt;
 
 static ErtsAuxWorkData *aux_thread_aux_work_data;
 
-#define ERTS_SCHDLR_SSPND_CHNG_WAITER		(((erts_aint32_t) 1) << 0)
+#define ERTS_SCHDLR_SSPND_CHNG_NMSB		(((erts_aint32_t) 1) << 0)
 #define ERTS_SCHDLR_SSPND_CHNG_MSB		(((erts_aint32_t) 1) << 1)
 #define ERTS_SCHDLR_SSPND_CHNG_ONLN		(((erts_aint32_t) 1) << 2)
+#define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN	(((erts_aint32_t) 1) << 3)
 
-#ifndef DEBUG
+typedef enum {
+    ERTS_SCHED_NORMAL,
+    ERTS_SCHED_DIRTY_CPU,
+    ERTS_SCHED_DIRTY_IO
+} ErtsSchedType;
 
-#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
-  erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL))
+typedef struct {
+    int ongoing;
+    ErtsProcList *blckrs;
+    ErtsProcList *chngq;
+} ErtsMultiSchedulingBlock;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \
-  erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_cpu_changing, (VAL))
-#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \
-  erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_io_changing, (VAL))
-#endif
+static struct {
+    erts_smp_mtx_t mtx;
+    Uint32 online;
+    Uint32 curr_online;
+    Uint32 active;
+    erts_smp_atomic32_t changing;
+    ErtsProcList *chngq;
+    Eterm changer;
+    ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */
+    ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */
+} schdlr_sspnd;
 
-#else
+#define ERTS_SCHDLR_SSPND_S_BITS	10
+#define ERTS_SCHDLR_SSPND_DCS_BITS	11
+#define ERTS_SCHDLR_SSPND_DIS_BITS	11
 
-#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL)			\
-do {									\
-    erts_aint32_t old_val__;						\
-    old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing,     	\
-					   (VAL));			\
-    ASSERT(old_val__ == (OLD_VAL));					\
-} while (0)
+#define ERTS_SCHDLR_SSPND_S_MASK	((1 << ERTS_SCHDLR_SSPND_S_BITS)-1)
+#define ERTS_SCHDLR_SSPND_DCS_MASK	((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1)
+#define ERTS_SCHDLR_SSPND_DIS_MASK	((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1)
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL)		\
-do {									\
-    erts_aint32_t old_val__;						\
-    old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_cpu_changing, \
-					   (VAL));			\
-    ASSERT(old_val__ == (OLD_VAL));					\
-} while (0)
-#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL)		\
-do {									\
-    erts_aint32_t old_val__;						\
-    old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_io_changing, \
-					   (VAL));			\
-    ASSERT(old_val__ == (OLD_VAL));					\
-} while (0)
-#endif
+#define ERTS_SCHDLR_SSPND_S_SHIFT	0
+#define ERTS_SCHDLR_SSPND_DCS_SHIFT	(ERTS_SCHDLR_SSPND_S_SHIFT \
+					 + ERTS_SCHDLR_SSPND_S_BITS)
+#define ERTS_SCHDLR_SSPND_DIS_SHIFT	(ERTS_SCHDLR_SSPND_DCS_SHIFT \
+					 + ERTS_SCHDLR_SSPND_DCS_BITS)
 
+#if (ERTS_SCHDLR_SSPND_S_BITS		\
+     + ERTS_SCHDLR_SSPND_DCS_BITS	\
+     + ERTS_SCHDLR_SSPND_DIS_BITS) > 32
+#  error Wont fit in Uint32
 #endif
 
-
-static struct {
-    erts_smp_mtx_t mtx;
-    erts_smp_cnd_t cnd;
-    int online;
-    int curr_online;
-    int wait_curr_online;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    int dirty_cpu_online;
-    int dirty_cpu_curr_online;
-    int dirty_cpu_wait_curr_online;
-    int dirty_io_online;
-    int dirty_io_curr_online;
-    int dirty_io_wait_curr_online;
+#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK
+#  error Max no schedulers wont fit in its bit-field
 #endif
-    erts_smp_atomic32_t changing;
-    erts_smp_atomic32_t active;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_atomic32_t dirty_cpu_changing;
-    erts_smp_atomic32_t dirty_cpu_active;
-    erts_smp_atomic32_t dirty_io_changing;
-    erts_smp_atomic32_t dirty_io_active;
+#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK
+#  error Max no dirty cpu schedulers wont fit in its bit-field
 #endif
-    struct {
-	int ongoing;
-	long wait_active;
-#ifdef ERTS_DIRTY_SCHEDULERS
-	long dirty_cpu_wait_active;
-	long dirty_io_wait_active;
+#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK
+#  error Max no dirty io schedulers wont fit in its bit-field
 #endif
-	ErtsProcList *procs;
-    } msb; /* Multi Scheduling Block */
-} schdlr_sspnd;
+
+#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS)			\
+    ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1)			\
+      << ERTS_SCHDLR_SSPND_S_SHIFT)					\
+     | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK))		\
+	 << ERTS_SCHDLR_SSPND_DCS_SHIFT))				\
+     | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK))		\
+	 << ERTS_SCHDLR_SSPND_DIS_SHIFT)))
+
+static void init_scheduler_suspend(void);
+
+static ERTS_INLINE Uint32
+schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type)
+{
+    Uint32 res = (Uint32) (*valp);
+    switch (type) {
+    case ERTS_SCHED_NORMAL:
+	res >>= ERTS_SCHDLR_SSPND_S_SHIFT;
+	res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK;
+	res++;
+	break;
+    case ERTS_SCHED_DIRTY_CPU:
+	res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT;
+	res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK;
+	break;
+    case ERTS_SCHED_DIRTY_IO:
+	res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT;
+	res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK;
+	break;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid scheduler type");
+	return 0;
+    }
+
+    return res;
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type)
+{
+    ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0);
+
+    switch (type) {
+    case ERTS_SCHED_NORMAL:
+	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+	break;
+    case ERTS_SCHED_DIRTY_CPU:
+	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+	break;
+    case ERTS_SCHED_DIRTY_IO:
+	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+	break;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid scheduler type");
+    }
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type)
+{
+    switch (type) {
+    case ERTS_SCHED_NORMAL:
+	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
+	       < ERTS_MAX_NO_OF_SCHEDULERS-1);
+	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+	break;
+    case ERTS_SCHED_DIRTY_CPU:
+	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
+	       < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS);
+	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+	break;
+    case ERTS_SCHED_DIRTY_IO:
+	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
+	       < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS);
+	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+	break;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid scheduler type");
+    }
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no)
+{
+    Uint32 val = *valp;
+
+    switch (type) {
+    case ERTS_SCHED_NORMAL:
+	ASSERT(no > 0);
+	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK)
+		 << ERTS_SCHDLR_SSPND_S_SHIFT);
+	val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK))
+		<< ERTS_SCHDLR_SSPND_S_SHIFT);
+	break;
+    case ERTS_SCHED_DIRTY_CPU:
+	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)
+		 << ERTS_SCHDLR_SSPND_DCS_SHIFT);
+	val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK))
+		 << ERTS_SCHDLR_SSPND_DCS_SHIFT);
+	break;
+    case ERTS_SCHED_DIRTY_IO:
+	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)
+		 << ERTS_SCHDLR_SSPND_DIS_SHIFT);
+	val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK))
+		<< ERTS_SCHDLR_SSPND_DIS_SHIFT);
+	break;
+    default:
+	ERTS_INTERNAL_ERROR("Invalid scheduler type");
+    }
+
+    *valp = val;
+}
 
 static struct {
     erts_smp_mtx_t update_mtx;
@@ -426,8 +516,10 @@ do {									\
 do {									\
     ErtsRunQueue *RQVAR;						\
     int ix__;								\
+    int online__ = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,	\
+						  ERTS_SCHED_NORMAL);	\
     ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx));	\
-    for (ix__ = 0; ix__ < schdlr_sspnd.online; ix__++) {		\
+    for (ix__ = 0; ix__ < online__; ix__++) {				\
 	RQVAR = ERTS_RUNQ_IX(ix__);					\
 	erts_smp_runq_lock(RQVAR);					\
 	{ DO; }								\
@@ -1188,12 +1280,27 @@ proclist_create(Process *p)
     return plp;
 }
 
+static ERTS_INLINE ErtsProcList *
+proclist_copy(ErtsProcList *plp0)
+{
+    ErtsProcList *plp1 = proclist_alloc();
+    plp1->pid = plp0->pid;
+    plp1->started_interval = plp0->started_interval;
+    return plp1;
+}
+
 static ERTS_INLINE void
 proclist_destroy(ErtsProcList *plp)
 {
     proclist_free(plp);
 }
 
+ErtsProcList *
+erts_proclist_copy(ErtsProcList *plp)
+{
+    return proclist_copy(plp);
+}
+
 ErtsProcList *
 erts_proclist_create(Process *p)
 {
@@ -2632,13 +2739,6 @@ sched_active(Uint no, ErtsRunQueue *rq)
 	profile_scheduler(make_small(no), am_active);
 }
 
-static int ERTS_INLINE
-ongoing_multi_scheduling_block(void)
-{
-    ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx));
-    return schdlr_sspnd.msb.ongoing;
-}
-
 static ERTS_INLINE void
 empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags)
 {
@@ -3025,8 +3125,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    timeout_time = erts_check_next_timeout_time(esdp);
 		    current_time = erts_get_monotonic_time(esdp);
 		    do_timeout = (current_time >= timeout_time);
-		} else
+		} else {
+		    current_time = 0;
 		    timeout_time = ERTS_MONOTONIC_TIME_MAX;
+		}
 		if (do_timeout) {
 		    if (!thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -3836,24 +3938,33 @@ static ERTS_INLINE void
 resume_run_queue(ErtsRunQueue *rq)
 {
     int pix;
+    Uint32 oflgs;
 
     erts_smp_runq_lock(rq);
 
-    (void) ERTS_RUNQ_FLGS_READ_BSET(rq,
-				    (ERTS_RUNQ_FLG_OUT_OF_WORK
-				     | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
-				     | ERTS_RUNQ_FLG_SUSPENDED),
-				    (ERTS_RUNQ_FLG_OUT_OF_WORK
-				     | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
+    oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq,
+				     (ERTS_RUNQ_FLG_OUT_OF_WORK
+				      | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
+				      | ERTS_RUNQ_FLG_SUSPENDED),
+				     (ERTS_RUNQ_FLG_OUT_OF_WORK
+				      | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
+
+    if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) {
+	erts_aint32_t len;
+
+	rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
+	for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
+	    len = erts_smp_atomic32_read_dirty(&rq->procs.prio_info[pix].len);
+	    rq->procs.prio_info[pix].max_len = len;
+	    rq->procs.prio_info[pix].reds = 0;
+	}
+	len = erts_smp_atomic32_read_dirty(&rq->ports.info.len);
+	rq->ports.info.max_len = len;
+	rq->ports.info.reds = 0;
+	len = erts_smp_atomic32_read_dirty(&rq->len);
+	rq->max_len = len;
 
-    rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
-    for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
-	rq->procs.prio_info[pix].max_len = 0;
-	rq->procs.prio_info[pix].reds = 0;
     }
-    rq->ports.info.max_len = 0;
-    rq->ports.info.reds = 0;
-    rq->max_len = 0;
 
     erts_smp_runq_unlock(rq);
 
@@ -5632,6 +5743,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     char *daww_ptr;
     size_t daww_sz;
     size_t size_runqs;
+#ifdef ERTS_SMP
+    erts_aint32_t set_schdlr_sspnd_change_flags;
+#endif
 
     init_misc_op_list_alloc();
     init_proc_sys_task_queues_alloc();
@@ -5869,25 +5983,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 					   sizeof(ErtsAuxWorkData));
 
-    erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
-    erts_smp_cnd_init(&schdlr_sspnd.cnd);
-
-    erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
-    schdlr_sspnd.online = no_schedulers_online;
-    schdlr_sspnd.curr_online = no_schedulers;
-    schdlr_sspnd.msb.ongoing = 0;
-    erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_changing, 0);
-    schdlr_sspnd.dirty_cpu_online = no_dirty_cpu_schedulers_online;
-    schdlr_sspnd.dirty_cpu_curr_online = no_dirty_cpu_schedulers;
-    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_active, no_dirty_cpu_schedulers);
-    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_changing, 0);
-    schdlr_sspnd.dirty_io_online = no_dirty_io_schedulers;
-    schdlr_sspnd.dirty_io_curr_online = no_dirty_io_schedulers;
-    erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_active, no_dirty_io_schedulers);
-#endif
-    schdlr_sspnd.msb.procs = NULL;
     init_no_runqs(no_schedulers_online, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
     erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update");
@@ -5902,32 +5997,66 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     init_migration_paths();
 
-    if (no_schedulers_online < no_schedulers) {
+    init_scheduler_suspend();
+
+    set_schdlr_sspnd_change_flags = 0;
+
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+			     ERTS_SCHED_NORMAL,
+			     no_schedulers_online);
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online,
+			     ERTS_SCHED_NORMAL,
+			     no_schedulers);
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.active,
+			     ERTS_SCHED_NORMAL,
+			     no_schedulers);
+
+    if (no_schedulers_online != no_schedulers) {
+	ASSERT(no_schedulers_online < no_schedulers);
+	set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN;
+	schdlr_sspnd.changer = am_init;
 	change_no_used_runqs(no_schedulers_online);
 	for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++)
 	    suspend_run_queue(ERTS_RUNQ_IX(ix));
     }
 
-    schdlr_sspnd.wait_curr_online = no_schedulers_online;
-    schdlr_sspnd.curr_online *= 2; /* Boot strapping... */
-    ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
-				| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
 #ifdef ERTS_DIRTY_SCHEDULERS
-    schdlr_sspnd.dirty_cpu_wait_curr_online = no_dirty_cpu_schedulers_online;
-    schdlr_sspnd.dirty_cpu_curr_online *= 2;
-    ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
-					  | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-    for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) {
-	ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
-	erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+			     ERTS_SCHED_DIRTY_CPU,
+			     no_dirty_cpu_schedulers_online);
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online,
+			     ERTS_SCHED_DIRTY_CPU,
+			     no_dirty_cpu_schedulers);
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.active,
+			     ERTS_SCHED_DIRTY_CPU,
+			     no_dirty_cpu_schedulers);
+
+    if (no_dirty_cpu_schedulers_online != no_dirty_cpu_schedulers) {
+	ASSERT(no_dirty_cpu_schedulers_online < no_dirty_cpu_schedulers);
+	set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+	for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) {
+	    ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+	    erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+	}
     }
 
-    schdlr_sspnd.dirty_io_wait_curr_online = no_dirty_io_schedulers;
-    schdlr_sspnd.dirty_io_curr_online *= 2;
-    ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
-					 | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+			     ERTS_SCHED_DIRTY_IO,
+			     no_dirty_io_schedulers);
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online,
+			     ERTS_SCHED_DIRTY_IO,
+			     no_dirty_io_schedulers);
+    schdlr_sspnd_set_nscheds(&schdlr_sspnd.active,
+			     ERTS_SCHED_DIRTY_IO,
+			     no_dirty_io_schedulers);
+
 #endif
 
+    if (set_schdlr_sspnd_change_flags)
+	erts_smp_atomic32_set_nob(&schdlr_sspnd.changing,
+				  set_schdlr_sspnd_change_flags);
+
     erts_smp_atomic32_init_nob(&doing_sys_schedule, 0);
 
     init_misc_aux_work();
@@ -5942,10 +6071,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 #endif
     }
     erts_no_schedulers = 1;
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = 0;
     erts_no_dirty_io_schedulers = 0;
-#endif
 #endif
 
     erts_smp_atomic32_init_nob(&function_calls, 0);
@@ -6705,19 +6832,6 @@ resume_process(Process *p, ErtsProcLocks locks)
     add2runq(enqueue, enq_prio, p, state, NULL);
 }
 
-int
-erts_get_max_no_executing_schedulers(void)
-{
-#ifdef ERTS_SMP
-    if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
-	return (int) erts_no_schedulers;
-    ERTS_THR_MEMORY_BARRIER;
-    return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
-#else
-    return 1;
-#endif
-}
-
 #ifdef ERTS_SMP
 
 static void
@@ -6820,42 +6934,82 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
     }
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
+static void
+init_scheduler_suspend(void)
+{
+    erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
+    schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+    schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+    schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+    erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
+    schdlr_sspnd.chngq = NULL;
+    schdlr_sspnd.changer = am_false;
+    schdlr_sspnd.nmsb.ongoing = 0;
+    schdlr_sspnd.nmsb.blckrs = NULL;
+    schdlr_sspnd.nmsb.chngq = NULL;
+    schdlr_sspnd.msb.ongoing = 0;
+    schdlr_sspnd.msb.blckrs = NULL;
+    schdlr_sspnd.msb.chngq = NULL;
+}
+
+typedef struct {
+    struct {
+	Eterm chngr;
+	Eterm nxt;
+    } onln;
+    struct {
+	ErtsProcList *chngrs;
+    } msb;
+} ErtsSchdlrSspndResume;
+
+static void
+schdlr_sspnd_resume_proc(Eterm pid)
+{
+    Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS);
+    if (p) {
+	resume_process(p, ERTS_PROC_LOCK_STATUS);
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    }
+}
+
+static ERTS_INLINE void
+schdlr_sspnd_resume_procs(ErtsSchedType sched_type,
+			  ErtsSchdlrSspndResume *resume)
+{
+    if (is_internal_pid(resume->onln.chngr)) {
+	schdlr_sspnd_resume_proc(resume->onln.chngr);
+	resume->onln.chngr = NIL;
+    }
+    if (is_internal_pid(resume->onln.nxt)) {
+	schdlr_sspnd_resume_proc(resume->onln.nxt);
+	resume->onln.nxt = NIL;
+    }
+    while (resume->msb.chngrs) {
+	ErtsProcList *plp = resume->msb.chngrs;
+	resume->msb.chngrs = plp->next;
+	schdlr_sspnd_resume_proc(plp->pid);
+	proclist_destroy(plp);
+    }
+}
 
 static void
 suspend_scheduler(ErtsSchedulerData *esdp)
 {
     erts_aint32_t flgs;
     erts_aint32_t changing;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    long no = (long) (ERTS_SCHEDULER_IS_DIRTY(esdp)
-		      ? ERTS_DIRTY_SCHEDULER_NO(esdp)
-		      : esdp->no);
-#else
-    long no = (long) esdp->no;
-#endif
+    long no;
     ErtsSchedulerSleepInfo *ssi = esdp->ssi;
-    long active_schedulers;
     int curr_online = 1;
-    int wake = 0;
+    ErtsSchdlrSspndResume resume = {{NIL, NIL}, {NULL}};
     erts_aint32_t aux_work;
     int thr_prgr_active = 1;
     ErtsStuckBoundProcesses sbp = {NULL, NULL};
-    int* ss_onlinep;
-    int* ss_curr_onlinep;
-    int* ss_wait_curr_onlinep;
-    long* ss_wait_activep;
-    long ss_wait_active_target;
-    erts_smp_atomic32_t* ss_changingp;
-    erts_smp_atomic32_t* ss_activep;
+    ErtsSchedType sched_type;
+    erts_aint32_t online_flag;
 
     /*
      * Schedulers may be suspended in two different ways:
      * - A scheduler may be suspended since it is not online.
-     *   All schedulers with scheduler ids greater than
-     *   schdlr_sspnd.online are suspended; same for dirty
-     *   schedulers and schdlr_sspnd.dirty_cpu_online and
-     *   schdlr_sspnd.dirty_io_online.
      * - Multi scheduling is blocked. All schedulers except the
      *   scheduler with scheduler id 1 are suspended, and all
      *   dirty CPU and dirty I/O schedulers are suspended.
@@ -6863,27 +7017,43 @@ suspend_scheduler(ErtsSchedulerData *esdp)
      * Regardless of why a scheduler is suspended, it ends up here.
      */
 
-    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) || no != 1);
-
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+	if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
+	    online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+	    sched_type = ERTS_SCHED_DIRTY_CPU;
+	}
+	else {
+	    online_flag = 0;
+	    sched_type = ERTS_SCHED_DIRTY_IO;
+	}
+    }
+    else
+#endif
+    {
+	online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
+	no = esdp->no;
+	sched_type = ERTS_SCHED_NORMAL;
+    }
+
+    ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1);
+
+    if (sched_type != ERTS_SCHED_NORMAL) {
 	if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) {
 	    erts_smp_runq_unlock(esdp->run_queue);
 	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 	    erts_smp_runq_lock(esdp->run_queue);
 	}
-	if (ongoing_multi_scheduling_block())
+	if (schdlr_sspnd.msb.ongoing)
 	    evacuate_run_queue(esdp->run_queue, &sbp);
-    } else
-#endif
+	erts_smp_runq_unlock(esdp->run_queue);
+    }
+    else {
 	evacuate_run_queue(esdp->run_queue, &sbp);
 
-    erts_smp_runq_unlock(esdp->run_queue);
+	erts_smp_runq_unlock(esdp->run_queue);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
-    {
 	erts_sched_check_cpu_bind_prep_suspend(esdp);
 
 	if (erts_system_profile_flags.scheduler)
@@ -6897,99 +7067,139 @@ suspend_scheduler(ErtsSchedulerData *esdp)
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	    if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
-		active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_cpu_active);
-		ASSERT(active_schedulers >= 0);
-		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing);
-		ss_onlinep = &schdlr_sspnd.dirty_cpu_online;
-		ss_curr_onlinep = &schdlr_sspnd.dirty_cpu_curr_online;
-		ss_wait_curr_onlinep = &schdlr_sspnd.dirty_cpu_wait_curr_online;
-		ss_changingp = &schdlr_sspnd.dirty_cpu_changing;
-		ss_wait_activep = &schdlr_sspnd.msb.dirty_cpu_wait_active;
-		ss_activep = &schdlr_sspnd.dirty_cpu_active;
-	    } else {
-		active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_io_active);
-		ASSERT(active_schedulers >= 0);
-		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing);
-		ss_onlinep = &schdlr_sspnd.dirty_io_online;
-		ss_curr_onlinep = &schdlr_sspnd.dirty_io_curr_online;
-		ss_wait_curr_onlinep = &schdlr_sspnd.dirty_io_wait_curr_online;
-		ss_changingp = &schdlr_sspnd.dirty_io_changing;
-		ss_wait_activep = &schdlr_sspnd.msb.dirty_io_wait_active;
-		ss_activep = &schdlr_sspnd.dirty_io_active;
-	    }
-	    ss_wait_active_target = 0;
-	}
-	else
-#endif
-	{
-	    active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
-	    ASSERT(active_schedulers >= 1);
-	    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-	    ss_onlinep = &schdlr_sspnd.online;
-	    ss_curr_onlinep = &schdlr_sspnd.curr_online;
-	    ss_wait_curr_onlinep = &schdlr_sspnd.wait_curr_online;
-	    ss_changingp = &schdlr_sspnd.changing;
-	    ss_wait_activep = &schdlr_sspnd.msb.wait_active;
-	    ss_activep = &schdlr_sspnd.active;
-	    ss_wait_active_target = 1;
-	}
-	if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
-	    if (active_schedulers == *ss_wait_activep)
-		wake = 1;
-	    if (active_schedulers == ss_wait_active_target) {
-		changing = erts_smp_atomic32_read_band_nob(ss_changingp,
-							   ~ERTS_SCHDLR_SSPND_CHNG_MSB);
-		changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
-	    }
-	}
+	schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type);
+
+	ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+					ERTS_SCHED_NORMAL) >= 1);
+
+	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
 
 	while (1) {
-	    if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+
+	    if (changing & (ERTS_SCHDLR_SSPND_CHNG_NMSB
+			    | ERTS_SCHDLR_SSPND_CHNG_MSB)) {
+		int i = 0;
+		ErtsMultiSchedulingBlock *msb[3] = {0};
+		if (changing & ERTS_SCHDLR_SSPND_CHNG_NMSB)
+		    msb[i++] = &schdlr_sspnd.nmsb;
+		if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
+		    msb[i++] = &schdlr_sspnd.msb;
+
+		for (i = 0; msb[i]; i++) {
+		    erts_aint32_t clr_flg = 0;
+
+		    if (msb[i] == &schdlr_sspnd.nmsb
+			&& schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+						    ERTS_SCHED_NORMAL) == 1) {
+			clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB;
+		    }
+		    else if (schdlr_sspnd.active
+			     == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) {
+			clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
+		    }
+
+		    if (clr_flg) {
+			ErtsProcList *plp, *end_plp;
+			changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+								   ~clr_flg);
+			changing &= ~clr_flg;
+			(void) erts_proclist_fetch(&msb[i]->chngq, &end_plp);
+			/* resume processes that initiated the multi scheduling block... */
+			plp = msb[i]->chngq;
+			while (plp) {
+			    erts_proclist_store_last(&msb[i]->blckrs,
+						     proclist_copy(plp));
+			    plp = plp->next;
+			}
+			if (end_plp)
+			    end_plp->next = resume.msb.chngrs;
+			resume.msb.chngrs = msb[i]->chngq;
+			msb[i]->chngq = NULL;
+		    }
+		}
+	    }
+
+	    if (changing & online_flag) {
 		int changed = 0;
-		if (no > *ss_onlinep && curr_online) {
-		    (*ss_curr_onlinep)--;
+		Uint32 st_online;
+
+		st_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+						     sched_type);
+		if (no > st_online && curr_online) {
+		    schdlr_sspnd_dec_nscheds(&schdlr_sspnd.curr_online,
+					     sched_type);
 		    curr_online = 0;
 		    changed = 1;
 		}
-		else if (no <= *ss_onlinep && !curr_online) {
-		    (*ss_curr_onlinep)++;
+		else if (no <= st_online && !curr_online) {
+		    schdlr_sspnd_inc_nscheds(&schdlr_sspnd.curr_online,
+					     sched_type);
 		    curr_online = 1;
 		    changed = 1;
 		}
 		if (changed
-		    && *ss_curr_onlinep == *ss_wait_curr_onlinep)
-		    wake = 1;
-		if (*ss_onlinep == *ss_curr_onlinep) {
-		    changing = erts_smp_atomic32_read_band_nob(ss_changingp,
-							       ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
-		    changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
+		    && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+						 sched_type)
+			== schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online,
+						    sched_type))) {
+		    ErtsProcList *plp;
+		    changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+							       ~online_flag);
+		    changing &= ~online_flag;
+		    if (sched_type == ERTS_SCHED_NORMAL) {
+			ASSERT(is_internal_pid(schdlr_sspnd.changer)
+			       || schdlr_sspnd.changer == am_init);
+			/* resume process that initiated this change... */
+			resume.onln.chngr = schdlr_sspnd.changer;
+			plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+			if (!plp)
+			    schdlr_sspnd.changer = am_false;
+			else {
+			    schdlr_sspnd.changer = am_true; /* change right in transit */
+			    /* resume process that is queued for next change... */
+			    resume.onln.nxt = plp->pid;
+			    ASSERT(is_internal_pid(resume.onln.nxt));
+			}
+		    }
 		}
 	    }
 
-	    if (wake) {
-		erts_smp_cnd_signal(&schdlr_sspnd.cnd);
-		wake = 0;
-	    }
-
-	    if (curr_online && !ongoing_multi_scheduling_block()) {
+	    if (curr_online
+		&& (sched_type == ERTS_SCHED_NORMAL
+		    ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
+		    : !schdlr_sspnd.msb.ongoing)) {
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		    break;
 	    }
 	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
+	    schdlr_sspnd_resume_procs(sched_type, &resume);
+
 	    while (1) {
+		ErtsMonotonicTime current_time;
 		erts_aint32_t qmask;
 		erts_aint32_t flgs;
 
 		qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
 			 & ERTS_RUNQ_FLGS_QMASK);
-		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
-		if (aux_work|qmask) {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+
+		if (sched_type != ERTS_SCHED_NORMAL) {
+		    if (qmask) {
+			erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+			erts_smp_runq_lock(esdp->run_queue);
+			if (schdlr_sspnd.msb.ongoing)
+			    evacuate_run_queue(esdp->run_queue, &sbp);
+			erts_smp_runq_unlock(esdp->run_queue);
+			erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+		    }
+		    aux_work = 0;
+		}
+		else {
+
+		    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+
+		    if (aux_work|qmask) {
 			if (!thr_prgr_active) {
 			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			    sched_wall_time_change(esdp, 1);
@@ -7001,271 +7211,50 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 			if (aux_work && erts_thr_progress_update(esdp))
 			    erts_thr_progress_leader_update(esdp);
-		    }
-		    if (qmask) {
-#ifdef ERTS_DIRTY_SCHEDULERS
-			if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-			    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-			    erts_smp_runq_lock(esdp->run_queue);
-			    if (ongoing_multi_scheduling_block())
-				evacuate_run_queue(esdp->run_queue, &sbp);
-			    erts_smp_runq_unlock(esdp->run_queue);
-			    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-			} else
-#endif
-			{
+			if (qmask) {
 			    erts_smp_runq_lock(esdp->run_queue);
 			    evacuate_run_queue(esdp->run_queue, &sbp);
 			    erts_smp_runq_unlock(esdp->run_queue);
 			}
 		    }
-		}
 
-		if (!aux_work) {
-#ifdef ERTS_DIRTY_SCHEDULERS
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
-		    {
-			if (thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			    sched_wall_time_change(esdp, 0);
-			}
-			erts_thr_progress_prepare_wait(esdp);
-		    }
-		    flgs = sched_spin_suspended(ssi,
-						ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
-		    if (flgs == (ERTS_SSI_FLG_SLEEPING
-				 | ERTS_SSI_FLG_WAITING
-				 | ERTS_SSI_FLG_SUSPENDED)) {
-			flgs = sched_set_suspended_sleeptype(ssi);
-			if (flgs == (ERTS_SSI_FLG_SLEEPING
-				     | ERTS_SSI_FLG_TSE_SLEEPING
-				     | ERTS_SSI_FLG_WAITING
-				     | ERTS_SSI_FLG_SUSPENDED)) {
-			    int res;
+		}
 
-			    do {
-				res = erts_tse_twait(ssi->event, -1);
-			    } while (res == EINTR);
+		if (aux_work) {
+		    ASSERT(sched_type == ERTS_SCHED_NORMAL);
+		    current_time = erts_get_monotonic_time(esdp);
+		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
 			}
+			erts_bump_timers(esdp->timer_wheel, current_time);
 		    }
-#ifdef ERTS_DIRTY_SCHEDULERS
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
-			erts_thr_progress_finalize_wait(esdp);
 		}
+		else {
+		    ErtsMonotonicTime timeout_time;
+		    int do_timeout;
 
-		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
-						       | ERTS_SSI_FLG_SUSPENDED));
-		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
-		    break;
-		changing = erts_smp_atomic32_read_nob(ss_changingp);
-		if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
-		    break;
-	    }
-
-	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	    changing = erts_smp_atomic32_read_nob(ss_changingp);
-	}
+		    if (sched_type == ERTS_SCHED_NORMAL) {
+			timeout_time = erts_check_next_timeout_time(esdp);
+			current_time = erts_get_monotonic_time(esdp);
+			do_timeout = (current_time >= timeout_time);
+		    }
+		    else {
+			timeout_time = ERTS_MONOTONIC_TIME_MAX;
+			current_time = 0;
+			do_timeout = 0;
+		    }
 
-	active_schedulers = erts_smp_atomic32_inc_read_nob(ss_activep);
-	changing = erts_smp_atomic32_read_nob(ss_changingp);
-	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
-	    && *ss_onlinep == active_schedulers) {
-	    erts_smp_atomic32_read_band_nob(ss_changingp,
-					    ~ERTS_SCHDLR_SSPND_CHNG_MSB);
-	}
-
-#ifdef ERTS_DIRTY_SCHEDULERS
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
-	    ASSERT(no <= *ss_onlinep);
-	ASSERT(!ongoing_multi_scheduling_block());
-
-    }
-
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
-    ASSERT(curr_online);
-
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
-    {
-	if (erts_system_profile_flags.scheduler)
-	    profile_scheduler(make_small(esdp->no), am_active);
-
-	if (!thr_prgr_active) {
-	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	    sched_wall_time_change(esdp, 1);
-	}
-    }
-
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-	(void) erts_get_monotonic_time(esdp);
-    erts_smp_runq_lock(esdp->run_queue);
-    non_empty_runq(esdp->run_queue);
-
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-#endif
-    {
-	schedule_bound_processes(esdp->run_queue, &sbp);
-
-	erts_sched_check_cpu_bind_post_suspend(esdp);
-    }
-}
-
-#else /* !ERTS_DIRTY_SCHEDULERS */
-
-static void
-suspend_scheduler(ErtsSchedulerData *esdp)
-{
-    erts_aint32_t flgs;
-    erts_aint32_t changing;
-    long no = (long) esdp->no;
-    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
-    long active_schedulers;
-    int curr_online = 1;
-    int wake = 0;
-    erts_aint32_t aux_work;
-    int thr_prgr_active = 1;
-    ErtsStuckBoundProcesses sbp = {NULL, NULL};
-
-    /*
-     * Schedulers may be suspended in two different ways:
-     * - A scheduler may be suspended since it is not online.
-     *   All schedulers with scheduler ids greater than
-     *   schdlr_sspnd.online are suspended.
-     * - Multi scheduling is blocked. All schedulers except the
-     *   scheduler with scheduler id 1 are suspended.
-     *
-     * Regardless of why a scheduler is suspended, it ends up here.
-     */
-
-    ASSERT(no != 1);
-
-    evacuate_run_queue(esdp->run_queue, &sbp);
-
-    erts_smp_runq_unlock(esdp->run_queue);
-
-    erts_sched_check_cpu_bind_prep_suspend(esdp);
-
-    if (erts_system_profile_flags.scheduler)
-    	profile_scheduler(make_small(esdp->no), am_inactive);
-
-    sched_wall_time_change(esdp, 0);
-
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
-    flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
-    if (flgs & ERTS_SSI_FLG_SUSPENDED) {
-
-	active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
-	ASSERT(active_schedulers >= 1);
-	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-	if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
-	    if (active_schedulers == schdlr_sspnd.msb.wait_active)
-		wake = 1;
-	    if (active_schedulers == 1) {
-		changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-							   ~ERTS_SCHDLR_SSPND_CHNG_MSB);
-		changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
-	    }
-	}
-
-	while (1) {
-	    if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
-		int changed = 0;
-		if (no > schdlr_sspnd.online && curr_online) {
-		    schdlr_sspnd.curr_online--;
-		    curr_online = 0;
-		    changed = 1;
-		}
-		else if (no <= schdlr_sspnd.online && !curr_online) {
-		    schdlr_sspnd.curr_online++;
-		    curr_online = 1;
-		    changed = 1;
-		}
-		if (changed
-		    && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
-		    wake = 1;
-		if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
-		    changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-							       ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
-		    changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
-		}
-	    }
-
-	    if (wake) {
-		erts_smp_cnd_signal(&schdlr_sspnd.cnd);
-		wake = 0;
-	    }
-
-	    if (curr_online && !ongoing_multi_scheduling_block()) {
-		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
-		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
-		    break;
-	    }
-	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
-	    while (1) {
-		ErtsMonotonicTime current_time;
-		erts_aint32_t qmask;
-		erts_aint32_t flgs;
-
-		qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
-			 & ERTS_RUNQ_FLGS_QMASK);
-		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
-		if (aux_work|qmask) {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-			if (!thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			    sched_wall_time_change(esdp, 1);
-			}
-			if (aux_work)
-			    aux_work = handle_aux_work(&esdp->aux_work_data,
-						       aux_work,
-						       1);
-			if (aux_work && erts_thr_progress_update(esdp))
-			    erts_thr_progress_leader_update(esdp);
-		    }
-		    if (qmask) {
-			erts_smp_runq_lock(esdp->run_queue);
-			evacuate_run_queue(esdp->run_queue, &sbp);
-			erts_smp_runq_unlock(esdp->run_queue);
-		    }
-		}
-
-		if (aux_work) {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-			current_time = erts_get_monotonic_time(esdp);
-			if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-			    if (!thr_prgr_active) {
-				erts_thr_progress_active(esdp, thr_prgr_active = 1);
-				sched_wall_time_change(esdp, 1);
-			    }
-			    erts_bump_timers(esdp->timer_wheel, current_time);
-			}
-		    }
-		}
-		else {
-		    ErtsMonotonicTime timeout_time;
-		    int do_timeout = 0;
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-			timeout_time = erts_check_next_timeout_time(esdp);
-			current_time = erts_get_monotonic_time(esdp);
-			do_timeout = (current_time >= timeout_time);
-		    } else
-			timeout_time = ERTS_MONOTONIC_TIME_MAX;
 		    if (do_timeout) {
+			ASSERT(sched_type == ERTS_SCHED_NORMAL);
 			if (!thr_prgr_active) {
 			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			    sched_wall_time_change(esdp, 1);
 			}
 		    }
 		    else {
-			if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			if (sched_type == ERTS_SCHED_NORMAL) {
 			    if (thr_prgr_active) {
 				erts_thr_progress_active(esdp, thr_prgr_active = 0);
 				sched_wall_time_change(esdp, 0);
@@ -7284,30 +7273,39 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 					 | ERTS_SSI_FLG_SUSPENDED)) {
 				int res;
 
-				current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
-				    erts_get_monotonic_time(esdp);
+				if (sched_type == ERTS_SCHED_NORMAL)
+				    current_time = erts_get_monotonic_time(esdp);
+				else
+				    current_time = 0;
+
 				do {
 				    Sint64 timeout;
 				    if (current_time >= timeout_time)
 					break;
-				    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+				    if (sched_type != ERTS_SCHED_NORMAL)
+					timeout = -1;
+				    else
 					timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
 									 - current_time
 									 - 1) + 1;
-				    } else
-					timeout = -1;
 				    res = erts_tse_twait(ssi->event, timeout);
-				    current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
-					erts_get_monotonic_time(esdp);
+
+				    if (sched_type == ERTS_SCHED_NORMAL)
+					current_time = erts_get_monotonic_time(esdp);
+				    else
+					current_time = 0;
+
 				} while (res == EINTR);
 			    }
 			}
-			if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+			if (sched_type == ERTS_SCHED_NORMAL)
 			    erts_thr_progress_finalize_wait(esdp);
 		    }
 
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time)
+		    if (current_time >= timeout_time) {
+			ASSERT(sched_type == ERTS_SCHED_NORMAL);
 			erts_bump_timers(esdp->timer_wheel, current_time);
+		    }
 		}
 
 		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
@@ -7315,7 +7313,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		    break;
 		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-		if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
+		if (changing)
 		    break;
 	    }
 
@@ -7323,612 +7321,494 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
 	}
 
-	active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active);
+	schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type);
 	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
 	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
-	    && schdlr_sspnd.online == active_schedulers) {
+	    && schdlr_sspnd.online == schdlr_sspnd.active) {
 	    erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
 					    ~ERTS_SCHDLR_SSPND_CHNG_MSB);
 	}
 
-	ASSERT(no <= schdlr_sspnd.online);
-	ASSERT(!ongoing_multi_scheduling_block());
-
+	ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type));
+	ASSERT((sched_type == ERTS_SCHED_NORMAL
+		? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
+		: !schdlr_sspnd.msb.ongoing));
     }
 
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
+    ASSERT(!resume.msb.chngrs);
+    schdlr_sspnd_resume_procs(sched_type, &resume);
+
     ASSERT(curr_online);
 
-    if (erts_system_profile_flags.scheduler)
-    	profile_scheduler(make_small(esdp->no), am_active);
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	if (erts_system_profile_flags.scheduler)
+	    profile_scheduler(make_small(esdp->no), am_active);
 
-    if (!thr_prgr_active) {
-	erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	sched_wall_time_change(esdp, 1);
+	if (!thr_prgr_active) {
+	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	    sched_wall_time_change(esdp, 1);
+	}
     }
 
+    if (sched_type == ERTS_SCHED_NORMAL)
+	(void) erts_get_monotonic_time(esdp);
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
-    schedule_bound_processes(esdp->run_queue, &sbp);
+    if (sched_type == ERTS_SCHED_NORMAL) {
+	schedule_bound_processes(esdp->run_queue, &sbp);
 
-    erts_sched_check_cpu_bind_post_suspend(esdp);
+	erts_sched_check_cpu_bind_post_suspend(esdp);
+    }
 }
 
-#endif
-
-ErtsSchedSuspendResult
+void
 erts_schedulers_state(Uint *total,
 		      Uint *online,
 		      Uint *active,
 		      Uint *dirty_cpu,
 		      Uint *dirty_cpu_online,
+		      Uint *dirty_cpu_active,
 		      Uint *dirty_io,
-		      int yield_allowed)
+		      Uint *dirty_io_active)
 {
-    int res = ERTS_SCHDLR_SSPND_EINVAL;
-    erts_aint32_t changing;
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
-		 | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing));
-#endif
-    if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
-	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
-    else {
+    if (active || online || dirty_cpu_online
+	|| dirty_cpu_active || dirty_io_active) {
+	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 	if (active)
-	    *active = schdlr_sspnd.online;
+	    *active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+					       ERTS_SCHED_NORMAL);
 	if (online)
-	    *online = schdlr_sspnd.online;
-	if (ongoing_multi_scheduling_block() && active)
-	    *active = 1;
-#ifdef ERTS_DIRTY_SCHEDULERS
+	    *online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online,
+					       ERTS_SCHED_NORMAL);
+	if (dirty_cpu_active)
+	    *dirty_cpu_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+							 ERTS_SCHED_DIRTY_CPU);
 	if (dirty_cpu_online)
-	    *dirty_cpu_online = schdlr_sspnd.dirty_cpu_online;
-#endif
-	res = ERTS_SCHDLR_SSPND_DONE;
+	    *dirty_cpu_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online,
+							 ERTS_SCHED_DIRTY_CPU);
+	if (dirty_io_active)
+	    *dirty_io_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+							ERTS_SCHED_DIRTY_IO);
+	erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
     }
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
     if (total)
 	*total = erts_no_schedulers;
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (dirty_cpu)
 	*dirty_cpu = erts_no_dirty_cpu_schedulers;
     if (dirty_io)
 	*dirty_io = erts_no_dirty_io_schedulers;
-#endif
-    return res;
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-
-ErtsSchedSuspendResult
-erts_set_schedulers_online(Process *p,
-			   ErtsProcLocks plocks,
-			   Sint new_no,
-			   Sint *old_no
-#ifdef ERTS_DIRTY_SCHEDULERS
-			   , int dirty_only
-#endif
-			   )
+static void
+abort_sched_onln_chng_waitq(Process *p)
 {
-    ErtsSchedulerData *esdp;
-    int ix, res = -1, no, have_unlocked_plocks, end_wait;
-    erts_aint32_t changing = 0;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ErtsSchedulerSleepInfo* ssi;
-    int dirty_no, change_dirty;
-#endif
-
-    if (new_no < 1)
-	return ERTS_SCHDLR_SSPND_EINVAL;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no)
-	return ERTS_SCHDLR_SSPND_EINVAL;
-#endif
-    else if (erts_no_schedulers < new_no)
-	return ERTS_SCHDLR_SSPND_EINVAL;
-
-    esdp = ERTS_PROC_GET_SCHDATA(p);
-    end_wait = 0;
+    Eterm resume = NIL;
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
-    have_unlocked_plocks = 0;
-    no = (int) new_no;
-
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ASSERT(schdlr_sspnd.dirty_cpu_online <= erts_no_dirty_cpu_schedulers);
-    if (dirty_only) {
-	if (no > schdlr_sspnd.online) {
-	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-	    return ERTS_SCHDLR_SSPND_EINVAL;
+#ifdef DEBUG
+    {
+	int found_it = 0;
+	ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+	while (plp) {
+	    if (erts_proclist_same(plp, p))
+		found_it++;
+	    plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp);
 	}
-	dirty_no = no;
-    } else {
-	/*
-	 * Adjust the number of dirty CPU schedulers online relative to the
-	 * adjustment made to the number of normal schedulers online.
-	 */
-	int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers;
-	int onln_pct = no*total_pct/schdlr_sspnd.online;
-	dirty_no = schdlr_sspnd.dirty_cpu_online*onln_pct/100;
-	if (dirty_no == 0)
-	    dirty_no = 1;
-	ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers);
+	ASSERT(found_it == !!(p->flags & F_SCHDLR_ONLN_WAITQ));
     }
 #endif
-    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    changing |= erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing);
-#endif
-    if (changing) {
-	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
-    }
-    else {
-	int online = *old_no = schdlr_sspnd.online;
-#ifdef ERTS_DIRTY_SCHEDULERS
-	int dirty_online = schdlr_sspnd.dirty_cpu_online;
-
-	if (dirty_only) {
-	    *old_no = schdlr_sspnd.dirty_cpu_online;
-	    if (dirty_no == schdlr_sspnd.dirty_cpu_online) {
-		res = ERTS_SCHDLR_SSPND_DONE;
-	    }
-	    change_dirty = 1;
-	} else {
-#endif
-	    if (no == schdlr_sspnd.online) {
-#ifdef ERTS_DIRTY_SCHEDULERS
-		dirty_only = 1;
-		if (dirty_no == schdlr_sspnd.dirty_cpu_online)
-#endif
-		    res = ERTS_SCHDLR_SSPND_DONE;
-#ifdef ERTS_DIRTY_SCHEDULERS
-		else
-		    change_dirty = 1;
-#endif
-	    }
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    else
-		change_dirty = (dirty_no != schdlr_sspnd.dirty_cpu_online);
-	}
-#endif
-	if (res == -1)
-	{
-	    int increase = (no > online);
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    if (!dirty_only) {
-#endif
-		ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
-					    | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-		schdlr_sspnd.online = no;
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    } else
-		increase = (dirty_no > dirty_online);
-	    if (change_dirty) {
-		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
-						      | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-		schdlr_sspnd.dirty_cpu_online = dirty_no;
-	    }
-#endif
-	    if (increase) {
-		int ix;
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (!dirty_only) {
-#endif
-		    schdlr_sspnd.wait_curr_online = no;
-		    if (ongoing_multi_scheduling_block()) {
-			for (ix = online; ix < no; ix++)
-			    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
-		    }
-		    else {
-			if (plocks) {
-			    have_unlocked_plocks = 1;
-			    erts_smp_proc_unlock(p, plocks);
-			}
-			change_no_used_runqs(no);
 
-			for (ix = online; ix < no; ix++)
-			    resume_run_queue(ERTS_RUNQ_IX(ix));
+    if (p->flags & F_SCHDLR_ONLN_WAITQ) {
+	ErtsProcList *plp = NULL;
 
-			for (ix = no; ix < erts_no_run_queues; ix++)
-			    suspend_run_queue(ERTS_RUNQ_IX(ix));
-		    }
-#ifdef ERTS_DIRTY_SCHEDULERS
-		}
-		if (change_dirty) {
-		    schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no;
-		    ASSERT(schdlr_sspnd.dirty_cpu_curr_online !=
-			   schdlr_sspnd.dirty_cpu_wait_curr_online);
-		    if (ongoing_multi_scheduling_block()) {
-			for (ix = dirty_online; ix < dirty_no; ix++) {
-			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-			    erts_sched_poke(ssi);
-			}
-		    } else {
-			for (ix = dirty_online; ix < dirty_no; ix++) {
-			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-			    scheduler_ssi_resume_wake(ssi);
-			    erts_smp_atomic32_read_band_nob(&ssi->flags,
-							    ~ERTS_SSI_FLG_SUSPENDED);
-			}
-			wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
-		    }
-		}
-#endif
-		res = ERTS_SCHDLR_SSPND_DONE;
-	    }
-	    else /* if (no < online) */ {
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (change_dirty) {
-		    schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no;
-		    ASSERT(schdlr_sspnd.dirty_cpu_curr_online !=
-			   schdlr_sspnd.dirty_cpu_wait_curr_online);
-		    if (ongoing_multi_scheduling_block()) {
-			for (ix = dirty_no; ix < dirty_online; ix++) {
-			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-			    erts_sched_poke(ssi);
-			}
-		    } else {
-			for (ix = dirty_no; ix < dirty_online; ix++) {
-			    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-			    erts_smp_atomic32_read_bor_nob(&ssi->flags,
-							   ERTS_SSI_FLG_SUSPENDED);
-			}
-			wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
-		    }
-		}
-		if (dirty_only) {
-		    res = ERTS_SCHDLR_SSPND_DONE;
-		}
+	plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+	if (plp) {
+	    if (erts_proclist_same(plp, p)
+		&& schdlr_sspnd.changer == am_true) {
+		p->flags &= ~F_SCHDLR_ONLN_WAITQ;
+		/*
+		 * Change right was in transit to us;
+		 * transfer it to the next process by
+		 * resuming it...
+		 */
+		erts_proclist_remove(&schdlr_sspnd.chngq, plp);
+		proclist_destroy(plp);
+		plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+		if (plp)
+		    resume = plp->pid;
 		else
-#endif
-		{
-		    if (p->scheduler_data->no <= no) {
-			res = ERTS_SCHDLR_SSPND_DONE;
-			schdlr_sspnd.wait_curr_online = no;
-		    }
-		    else {
-			/*
-			 * Yield! Current process needs to migrate
-			 * before bif returns.
-			 */
-			res = ERTS_SCHDLR_SSPND_YIELD_DONE;
-			schdlr_sspnd.wait_curr_online = no+1;
-		    }
-
-		    if (ongoing_multi_scheduling_block()) {
-			for (ix = no; ix < online; ix++)
-			    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
-		    }
-		    else {
-			if (plocks) {
-			    have_unlocked_plocks = 1;
-			    erts_smp_proc_unlock(p, plocks);
-			}
-
-			change_no_used_runqs(no);
-			for (ix = no; ix < erts_no_run_queues; ix++)
-			    suspend_run_queue(ERTS_RUNQ_IX(ix));
-
-			for (ix = no; ix < online; ix++) {
-			    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-			    wake_scheduler(rq);
-			}
-		    }
-		}
+		    schdlr_sspnd.changer = am_false;
 	    }
-
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    if (change_dirty) {
-		while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online)
-		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing,
-						~ERTS_SCHDLR_SSPND_CHNG_WAITER);
-	    }
-	    if (!dirty_only)
-#endif
-	    {
-		if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) {
-		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-		    if (plocks && !have_unlocked_plocks) {
-			have_unlocked_plocks = 1;
-			erts_smp_proc_unlock(p, plocks);
+	    else {
+		do {
+		    if (erts_proclist_same(plp, p)) {
+			p->flags &= ~F_SCHDLR_ONLN_WAITQ;
+			erts_proclist_remove(&schdlr_sspnd.chngq, plp);
+			proclist_destroy(plp);
+			break;
 		    }
-		    erts_thr_progress_active(esdp, 0);
-		    erts_thr_progress_prepare_wait(esdp);
-		    end_wait = 1;
-		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-		}
-
-		while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
-		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-
-		ASSERT(res != ERTS_SCHDLR_SSPND_DONE
-		       ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
-			  & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
-		       : (ERTS_SCHDLR_SSPND_CHNG_WAITER
-			  == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
-		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-						~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+		    plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp);
+		} while (plp);
 	    }
 	}
     }
 
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ASSERT(schdlr_sspnd.dirty_cpu_online <= schdlr_sspnd.online);
-    if (!dirty_only)
-#endif
-    {
-	if (end_wait) {
-	    erts_thr_progress_finalize_wait(esdp);
-	    erts_thr_progress_active(esdp, 1);
-	}
-	if (have_unlocked_plocks)
-	    erts_smp_proc_lock(p, plocks);
-    }
 
-    return res;
+    if (is_internal_pid(resume))
+	schdlr_sspnd_resume_proc(resume);
 }
 
-#else /* !ERTS_DIRTY_SCHEDULERS */
-
 ErtsSchedSuspendResult
 erts_set_schedulers_online(Process *p,
 			   ErtsProcLocks plocks,
 			   Sint new_no,
-			   Sint *old_no)
+			   Sint *old_no,
+			   int dirty_only)
 {
-    ErtsSchedulerData *esdp;
-    int ix, res, no, have_unlocked_plocks, end_wait;
-    erts_aint32_t changing;
+    int resume_proc, ix, res = -1, no, have_unlocked_plocks;
+    erts_aint32_t changing = 0, change_flags;
+    int online, increase;
+    ErtsProcList *plp;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    int dirty_no, change_dirty, dirty_online;
+#else
+    ASSERT(!dirty_only);
+#endif
 
-    if (new_no < 1 || erts_no_schedulers < new_no)
+    if (new_no < 1)
+	return ERTS_SCHDLR_SSPND_EINVAL;
+    else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no)
+	return ERTS_SCHDLR_SSPND_EINVAL;
+    else if (erts_no_schedulers < new_no)
 	return ERTS_SCHDLR_SSPND_EINVAL;
 
-    esdp = ERTS_PROC_GET_SCHDATA(p);
-    end_wait = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (dirty_only)
+	resume_proc = 0;
+    else
+#endif
+    {
+	resume_proc = 1;
+	/*
+	 * If we suspend current process we need to suspend before
+	 * requesting the change; otherwise, we got a resume/suspend
+	 * race...
+	 */
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	suspend_process(p, p);
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    }
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
+    change_flags = 0;
     have_unlocked_plocks = 0;
     no = (int) new_no;
 
-    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-    if (changing) {
-	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!dirty_only)
+#endif
+    {
+	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+	enqueue_wait:
+	    p->flags |= F_SCHDLR_ONLN_WAITQ;
+	    plp = proclist_create(p);
+	    erts_proclist_store_last(&schdlr_sspnd.chngq, plp);
+	    resume_proc = 0;
+	    res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
+	    goto done;
+	}
+	plp = erts_proclist_peek_first(schdlr_sspnd.chngq);
+	if (!plp) {
+	    ASSERT(schdlr_sspnd.changer == am_false);
+	}
+	else {
+	    ASSERT(schdlr_sspnd.changer == am_true);
+	    if (!erts_proclist_same(plp, p))
+		goto enqueue_wait;
+	    p->flags &= ~F_SCHDLR_ONLN_WAITQ;
+	    erts_proclist_remove(&schdlr_sspnd.chngq, plp);
+	    proclist_destroy(plp);
+	}
+    }
+
+    *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+						ERTS_SCHED_NORMAL);
+#ifndef ERTS_DIRTY_SCHEDULERS
+    if (no == online) {
+	res = ERTS_SCHDLR_SSPND_DONE;
+	goto done;
     }
-    else {
-	int online = *old_no = schdlr_sspnd.online;
-	if (no == schdlr_sspnd.online) {
+#else /* ERTS_DIRTY_SCHEDULERS */
+    dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+					    ERTS_SCHED_DIRTY_CPU);
+    if (dirty_only)
+	*old_no = dirty_online;
+
+    ASSERT(dirty_online <= erts_no_dirty_cpu_schedulers);
+
+    if (dirty_only) {
+	if (no > online) {
+	    res = ERTS_SCHDLR_SSPND_EINVAL;
+	    goto done;
+	}
+	dirty_no = no;
+	if (dirty_no == dirty_online) {
 	    res = ERTS_SCHDLR_SSPND_DONE;
+	    goto done;
 	}
-	else {
-	    ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
-					| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-	    schdlr_sspnd.online = no;
-	    if (no > online) {
-		int ix;
-		schdlr_sspnd.wait_curr_online = no;
-		if (ongoing_multi_scheduling_block()) {
-		    for (ix = online; ix < no; ix++)
-			erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
-		}
-		else {
-		    if (plocks) {
-			have_unlocked_plocks = 1;
-			erts_smp_proc_unlock(p, plocks);
-		    }
-		    change_no_used_runqs(no);
-
-		    for (ix = online; ix < no; ix++)
-			resume_run_queue(ERTS_RUNQ_IX(ix));
+	change_dirty = 1;
+    } else {
+	/*
+	 * Adjust the number of dirty CPU schedulers online relative to the
+	 * adjustment made to the number of normal schedulers online.
+	 */
+	int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers;
+	int onln_pct = no*total_pct/online;
+	dirty_no = dirty_online*onln_pct/100;
+	if (dirty_no == 0)
+	    dirty_no = 1;
+	ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers);
 
-		    for (ix = no; ix < erts_no_run_queues; ix++)
-			suspend_run_queue(ERTS_RUNQ_IX(ix));
-		}
+	if (no != online)
+	    change_dirty = (dirty_no != dirty_online);
+	else {
+	    dirty_only = 1;
+	    if (dirty_no == dirty_online) {
 		res = ERTS_SCHDLR_SSPND_DONE;
+		goto done;
 	    }
-	    else /* if (no < online) */ {
-		if (p->scheduler_data->no <= no) {
-		    res = ERTS_SCHDLR_SSPND_DONE;
-		    schdlr_sspnd.wait_curr_online = no;
+	    change_dirty = 1;
+	}
+    }
+    if (change_dirty) {
+	change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+	schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+				 ERTS_SCHED_DIRTY_CPU,
+				 dirty_no);
+    }
+
+    if (dirty_only)
+	increase = (dirty_no > dirty_online);
+    else
+#endif /* ERTS_DIRTY_SCHEDULERS */
+    {
+	change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN;
+	schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
+				 ERTS_SCHED_NORMAL,
+				 no);
+	increase = (no > online);
+    }
+
+    erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags);
+
+    res = ERTS_SCHDLR_SSPND_DONE;
+    if (increase) {
+	int ix;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (change_dirty) {
+	    ErtsSchedulerSleepInfo* ssi;
+	    if (schdlr_sspnd.msb.ongoing) {
+		for (ix = dirty_online; ix < dirty_no; ix++) {
+		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+		    erts_sched_poke(ssi);
 		}
-		else {
-		    /*
-		     * Yield! Current process needs to migrate
-		     * before bif returns.
-		     */
-		    res = ERTS_SCHDLR_SSPND_YIELD_DONE;
-		    schdlr_sspnd.wait_curr_online = no+1;
+	    } else {
+		for (ix = dirty_online; ix < dirty_no; ix++) {
+		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+		    scheduler_ssi_resume_wake(ssi);
 		}
-
-		if (ongoing_multi_scheduling_block()) {
-		    for (ix = no; ix < online; ix++)
-			erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+	    }
+	}
+	if (!dirty_only)
+#endif
+	{
+	    if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) {
+		for (ix = online; ix < no; ix++)
+		    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+	    }
+	    else {
+		if (plocks) {
+		    have_unlocked_plocks = 1;
+		    erts_smp_proc_unlock(p, plocks);
 		}
-		else {
-		    if (plocks) {
-			have_unlocked_plocks = 1;
-			erts_smp_proc_unlock(p, plocks);
-		    }
+		change_no_used_runqs(no);
 
-		    change_no_used_runqs(no);
-		    for (ix = no; ix < erts_no_run_queues; ix++)
-			suspend_run_queue(ERTS_RUNQ_IX(ix));
+		for (ix = online; ix < no; ix++)
+		    resume_run_queue(ERTS_RUNQ_IX(ix));
 
-		    for (ix = no; ix < online; ix++) {
-			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-			wake_scheduler(rq);
-		    }
+		for (ix = no; ix < erts_no_run_queues; ix++)
+		    suspend_run_queue(ERTS_RUNQ_IX(ix));
+	    }
+	}
+    }
+    else /* if decrease */ {
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (change_dirty) {
+	    ErtsSchedulerSleepInfo* ssi;
+	    if (schdlr_sspnd.msb.ongoing) {
+		for (ix = dirty_no; ix < dirty_online; ix++) {
+		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+		    erts_sched_poke(ssi);
 		}
+	    } else {
+		for (ix = dirty_no; ix < dirty_online; ix++) {
+		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
+						   ERTS_SSI_FLG_SUSPENDED);
+		}
+		wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
 	    }
-
-	    if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) {
-		erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-		if (plocks && !have_unlocked_plocks) {
+	}
+	if (!dirty_only)
+#endif
+	{
+	    if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) {
+		for (ix = no; ix < online; ix++)
+		    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
+	    }
+	    else {
+		if (plocks) {
 		    have_unlocked_plocks = 1;
 		    erts_smp_proc_unlock(p, plocks);
 		}
-		erts_thr_progress_active(esdp, 0);
-		erts_thr_progress_prepare_wait(esdp);
-		end_wait = 1;
-		erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	    }
 
-	    while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
-		erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-
-	    ASSERT(res != ERTS_SCHDLR_SSPND_DONE
-		   ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
-		      & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
-		   : (ERTS_SCHDLR_SSPND_CHNG_WAITER
-		      == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
-	    erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-					    ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+		change_no_used_runqs(no);
+		for (ix = no; ix < erts_no_run_queues; ix++)
+		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 
+		for (ix = no; ix < online; ix++) {
+		    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+		    wake_scheduler(rq);
+		}
+	    }
 	}
     }
 
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-    if (end_wait) {
-	erts_thr_progress_finalize_wait(esdp);
-	erts_thr_progress_active(esdp, 1);
+    if (change_flags & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+	/* Suspend and wait for requested change to complete... */
+	schdlr_sspnd.changer = p->common.id;
+	resume_proc = 0;
+	res = ERTS_SCHDLR_SSPND_YIELD_DONE;
     }
+
+done:
+
+    ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+				    ERTS_SCHED_DIRTY_CPU)
+	   <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+				       ERTS_SCHED_NORMAL));
+
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
     if (have_unlocked_plocks)
 	erts_smp_proc_lock(p, plocks);
 
+    if (resume_proc) {
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	resume_process(p, plocks|ERTS_PROC_LOCK_STATUS);
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    }
+
     return res;
 }
 
-#endif
-
 ErtsSchedSuspendResult
-erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
+erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal, int all)
 {
-    int ix, res, have_unlocked_plocks = 0, online;
-    erts_aint32_t changing;
+    int resume_proc, ix, res, have_unlocked_plocks = 0;
     ErtsProcList *plp;
 #ifdef ERTS_DIRTY_SCHEDULERS
     ErtsSchedulerSleepInfo* ssi;
 #endif
+    ErtsMultiSchedulingBlock *msbp;
+    erts_aint32_t chng_flg;
+    int have_blckd_flg;
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
-		 | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing));
-#endif
-    if (changing) {
-	res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */
+    if (normal) {
+	chng_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB;
+	have_blckd_flg = F_HAVE_BLCKD_NMSCHED;
+	msbp = &schdlr_sspnd.nmsb;
+    }
+    else {
+	chng_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
+	have_blckd_flg = F_HAVE_BLCKD_MSCHED;
+	msbp = &schdlr_sspnd.msb;
+    }
+
+    /*
+     * If we suspend current process we need to suspend before
+     * requesting the change; otherwise, we got a resume/suspend
+     * race...
+     */
+    if (!on) {
+	/* We never suspend current process when unblocking... */
+	resume_proc = 0;
+    }
+    else {
+	resume_proc = 1;
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	suspend_process(p, p);
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
-    else if (on) { /* ------ BLOCK ------ */
-	if (schdlr_sspnd.msb.procs) {
+
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    if (on) { /* ------ BLOCK ------ */
+	if (msbp->chngq) {
+	    ASSERT(msbp->ongoing);
+	    p->flags |= have_blckd_flg;
+	    goto wait_until_msb;
+	}
+	else if (msbp->blckrs) {
+	    ASSERT(msbp->ongoing);
 	    plp = proclist_create(p);
-	    erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp);
-	    p->flags |= F_HAVE_BLCKD_MSCHED;
-	    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 0);
-	    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) == 0);
-#endif
+	    erts_proclist_store_last(&msbp->blckrs, plp);
+	    p->flags |= have_blckd_flg;
+	    ASSERT(schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0));
 	    ASSERT(p->scheduler_data->no == 1);
-	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-	} else {
-	    int online = schdlr_sspnd.online;
-	    p->flags |= F_HAVE_BLCKD_MSCHED;
+	    if (schdlr_sspnd.msb.ongoing)
+		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
+	    else
+		res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
+	}
+	else {
+	    int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+							ERTS_SCHED_NORMAL);
+	    ASSERT(!msbp->ongoing);
+	    p->flags |= have_blckd_flg;
 	    if (plocks) {
 		have_unlocked_plocks = 1;
 		erts_smp_proc_unlock(p, plocks);
 	    }
-	    ASSERT(!ongoing_multi_scheduling_block());
-	    schdlr_sspnd.msb.ongoing = 1;
-	    if (online == 1) {
-		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
-#ifdef ERTS_DIRTY_SCHEDULERS
-		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 1);
-		ASSERT(!(erts_smp_atomic32_read_nob(&ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0)->flags)
-			 & ERTS_SSI_FLG_SUSPENDED));
-		schdlr_sspnd.msb.dirty_cpu_wait_active = 0;
-		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
-						      | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-		ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0);
-		erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
-		wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
-		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active)
-		       != schdlr_sspnd.msb.dirty_cpu_wait_active)
-		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-
-		schdlr_sspnd.msb.dirty_io_wait_active = 0;
-		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
-						     | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
-		    ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						   ERTS_SSI_FLG_SUSPENDED);
-		}
-		wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
-		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active)
-		       != schdlr_sspnd.msb.dirty_io_wait_active)
-		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-#endif
+	    ASSERT(!msbp->ongoing);
+	    msbp->ongoing = 1;
+	    if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)
+		|| (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+						       ERTS_SCHED_NORMAL) == 1)) {
 		ASSERT(p->scheduler_data->no == 1);
+		plp = proclist_create(p);
+		erts_proclist_store_last(&msbp->blckrs, plp);
+		if (schdlr_sspnd.msb.ongoing)
+		    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
+		else
+		    res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
 	    }
 	    else {
-		ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
-					    | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-		if (p->scheduler_data->no == 1) {
-		    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-		    schdlr_sspnd.msb.wait_active = 1;
-		}
-		else {
-		    /*
-		     * Yield! Current process needs to migrate
-		     * before bif returns.
-		     */
-		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
-		    schdlr_sspnd.msb.wait_active = 2;
-		}
-
-#ifdef ERTS_DIRTY_SCHEDULERS
-		schdlr_sspnd.msb.dirty_cpu_wait_active = 0;
-		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
-						      | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-		for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						   ERTS_SSI_FLG_SUSPENDED);
-		}
-		wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
-		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active)
-		       != schdlr_sspnd.msb.dirty_cpu_wait_active)
-		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-		ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-		ASSERT(schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_online);
-
-		schdlr_sspnd.msb.dirty_io_wait_active = 0;
-		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
-						     | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
-		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
-		    ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						   ERTS_SSI_FLG_SUSPENDED);
-		}
-		wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
-		while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active)
-		       != schdlr_sspnd.msb.dirty_io_wait_active)
-		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-		ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-		ASSERT(schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_online);
-#endif
+		erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+					       chng_flg);
 		change_no_used_runqs(1);
 		for (ix = 1; ix < erts_no_run_queues; ix++)
 		    suspend_run_queue(ERTS_RUNQ_IX(ix));
@@ -7938,84 +7818,84 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    wake_scheduler(rq);
 		}
 
-		if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
-		    != schdlr_sspnd.msb.wait_active) {
-		    ErtsSchedulerData *esdp;
-
-		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
-		    if (plocks && !have_unlocked_plocks) {
-			have_unlocked_plocks = 1;
-			erts_smp_proc_unlock(p, plocks);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (!normal) {
+		    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+			ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+			erts_smp_atomic32_read_bor_nob(&ssi->flags,
+						       ERTS_SSI_FLG_SUSPENDED);
 		    }
+		    wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
 
-		    esdp = ERTS_PROC_GET_SCHDATA(p);
-
-		    erts_thr_progress_active(esdp, 0);
-		    erts_thr_progress_prepare_wait(esdp);
-
-		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
-		    while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
-			   != schdlr_sspnd.msb.wait_active)
-			erts_smp_cnd_wait(&schdlr_sspnd.cnd,
-					  &schdlr_sspnd.mtx);
-		    
-		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-		    
-		    erts_thr_progress_active(esdp, 1);
-		    erts_thr_progress_finalize_wait(esdp);
+		    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+			ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+			erts_smp_atomic32_read_bor_nob(&ssi->flags,
+						       ERTS_SSI_FLG_SUSPENDED);
+		    }
+		    wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
+		}
+#endif
 
-		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	    wait_until_msb:
 
-		}
+		ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing));
 
-		ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED
-		       ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
-			  & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
-		       : (ERTS_SCHDLR_SSPND_CHNG_WAITER
-			  == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
-		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-						~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+		plp = proclist_create(p);
+		erts_proclist_store_last(&msbp->chngq, plp);
+		resume_proc = 0;
+		if (schdlr_sspnd.msb.ongoing)
+		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
+		else
+		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED;
 	    }
-	    plp = proclist_create(p);
-	    erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp);
 	    ASSERT(p->scheduler_data);
 	}
     }
-    else if (!ongoing_multi_scheduling_block()) {
-	/* unblock not ongoing */
-	ASSERT(!schdlr_sspnd.msb.procs);
-	res = ERTS_SCHDLR_SSPND_DONE;
+    else if (!msbp->ongoing) {
+	ASSERT(!msbp->blckrs);
+	goto unblock_res;
     }
     else {  /* ------ UNBLOCK ------ */
-	if (p->flags & F_HAVE_BLCKD_MSCHED) {
-	    ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
-
-	    while (plp) {
-		ErtsProcList *tmp_plp = plp;
-		plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp);
-		if (erts_proclist_same(tmp_plp, p)) {
-		    erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp);
-		    proclist_destroy(tmp_plp);
-		    if (!all)
-			break;
+	if (p->flags & have_blckd_flg) {
+	    ErtsProcList *plps[2];
+	    ErtsProcList *plp;
+	    int limit = 0;
+
+	    plps[limit++] = erts_proclist_peek_first(msbp->blckrs);
+	    if (all)
+		plps[limit++] = erts_proclist_peek_first(msbp->chngq);
+
+	    for (ix = 0; ix < limit; ix++) {
+		plp = plps[ix];
+		while (plp) {
+		    ErtsProcList *tmp_plp = plp;
+		    plp = erts_proclist_peek_next(msbp->blckrs, plp);
+		    if (erts_proclist_same(tmp_plp, p)) {
+			erts_proclist_remove(&msbp->blckrs, tmp_plp);
+			proclist_destroy(tmp_plp);
+			if (!all)
+			    break;
+		    }
 		}
 	    }
 	}
-	if (schdlr_sspnd.msb.procs)
-	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-	else {
-	    ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
-	    p->flags &= ~F_HAVE_BLCKD_MSCHED;
-	    schdlr_sspnd.msb.ongoing = 0;
-	    if (schdlr_sspnd.online == 1) {
+	if (!msbp->blckrs && !msbp->chngq) {
+	    int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+							ERTS_SCHED_NORMAL);
+	    erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+					   chng_flg);
+	    p->flags &= ~have_blckd_flg;
+	    msbp->ongoing = 0;
+	    if (online == 1) {
 		/* No normal schedulers to resume */
-		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
-		ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB);
+		ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+						ERTS_SCHED_NORMAL) == 1);
+#ifndef ERTS_DIRTY_SCHEDULERS
+		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+						~chng_flg);
+#endif
 	    }
-	    else {
-		online = schdlr_sspnd.online;
+	    else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) {
 		if (plocks) {
 		    have_unlocked_plocks = 1;
 		    erts_smp_proc_unlock(p, plocks);
@@ -8031,83 +7911,91 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 	    }
 #ifdef ERTS_DIRTY_SCHEDULERS
-	    ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
-	    schdlr_sspnd.msb.dirty_cpu_wait_active = schdlr_sspnd.dirty_cpu_online;
-	    for (ix = 0; ix < schdlr_sspnd.dirty_cpu_online; ix++) {
-		ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		scheduler_ssi_resume_wake(ssi);
-		erts_smp_atomic32_read_band_nob(&ssi->flags,
-						~ERTS_SSI_FLG_SUSPENDED);
-	    }
-	    wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
-
-	    ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
-	    schdlr_sspnd.msb.dirty_io_wait_active = erts_no_dirty_io_schedulers;
-	    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
-		ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-		scheduler_ssi_resume_wake(ssi);
-		erts_smp_atomic32_read_band_nob(&ssi->flags,
-						~ERTS_SSI_FLG_SUSPENDED);
-	    }
-	    wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
+	    if (!normal) {
+		ASSERT(!schdlr_sspnd.msb.ongoing);
+		online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+							ERTS_SCHED_DIRTY_CPU);
+		for (ix = 0; ix < online; ix++) {
+		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+		    scheduler_ssi_resume_wake(ssi);
+		}
+
+		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+		    ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+		    scheduler_ssi_resume_wake(ssi);
+		}
+	    }
 #endif
-	    res = ERTS_SCHDLR_SSPND_DONE;
 	}
+
+    unblock_res:
+	if (schdlr_sspnd.msb.ongoing)
+	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
+	else if (schdlr_sspnd.nmsb.ongoing)
+	    res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
+	else
+	    res = ERTS_SCHDLR_SSPND_DONE;
     }
 
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+
     if (have_unlocked_plocks)
 	erts_smp_proc_lock(p, plocks);
-    return res;
-}
 
-#ifdef DEBUG
-void
-erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
-{
-    if (return_value == am_blocked) {
-	erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
-	ASSERT(1 <= active && active <= 2);
-	ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1);
+    if (resume_proc) {
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	resume_process(p, plocks|ERTS_PROC_LOCK_STATUS);
+	if (!(plocks & ERTS_PROC_LOCK_STATUS))
+	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
+
+    return res;
 }
-#endif
 
 int
 erts_is_multi_scheduling_blocked(void)
 {
     int res;
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    res = schdlr_sspnd.msb.procs != NULL;
+    if (schdlr_sspnd.msb.blckrs)
+	res = 1;
+    else if (schdlr_sspnd.nmsb.blckrs)
+	res = -1;
+    else
+	res = 0;
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
     return res;
 }
 
 Eterm
-erts_multi_scheduling_blockers(Process *p)
+erts_multi_scheduling_blockers(Process *p, int normal)
 {
     Eterm res = NIL;
+    ErtsMultiSchedulingBlock *msbp;
+
+    msbp = normal ? &schdlr_sspnd.nmsb : &schdlr_sspnd.msb;
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) {
+    if (!erts_proclist_is_empty(msbp->blckrs)) {
 	Eterm *hp, *hp_end;
 	ErtsProcList *plp1, *plp2;
 	Uint max_size = 0;
 
-	for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
+	for (plp1 = erts_proclist_peek_first(msbp->blckrs);
 	     plp1;
-	     plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) {
+	     plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) {
 	    max_size += 2;
 	}
 	ASSERT(max_size);
 	hp = HAlloc(p, max_size);
 	hp_end = hp + max_size;
-	for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
+	for (plp1 = erts_proclist_peek_first(msbp->blckrs);
 	     plp1;
-	     plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) {
-	    for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs);
+	     plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) {
+	    for (plp2 = erts_proclist_peek_first(msbp->blckrs);
 		 plp2->pid != plp1->pid;
-		 plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2));
+		 plp2 = erts_proclist_peek_next(msbp->blckrs, plp2));
 	    if (plp2 == plp1) {
 		res = CONS(hp, plp1->pid, res);
 		hp += 2;
@@ -8176,39 +8064,6 @@ sched_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    if (no == 1) {
-	erts_thr_progress_active(esdp, 0);
-	erts_thr_progress_prepare_wait(esdp);
-    }
-
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
-    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)
-	   & ERTS_SCHDLR_SSPND_CHNG_ONLN);
-
-    if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) {
-	erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-					~ERTS_SCHDLR_SSPND_CHNG_ONLN);
-	if (no != 1)
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-#else
-	    erts_smp_cnd_signal(&schdlr_sspnd.cnd);
-#endif
-    }
-
-    if (no == 1) {
-	while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
-	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-	ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-    }
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
-    if (no == 1) {
-	erts_thr_progress_finalize_wait(esdp);
-	erts_thr_progress_active(esdp, 1);
-    }
-
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
     esdp->verify_unused_temp_alloc
 	= erts_alloc_get_verify_unused_temp_alloc(
@@ -8263,24 +8118,6 @@ sched_dirty_cpu_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing)
-	   & ERTS_SCHDLR_SSPND_CHNG_ONLN);
-
-    if (--schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_wait_curr_online) {
-	erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing,
-					~ERTS_SCHDLR_SSPND_CHNG_ONLN);
-	if (no != 1)
-	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-    }
-
-    if (no == 1) {
-	while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online)
-	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-	ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-    }
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
     process_main();
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
@@ -8326,24 +8163,6 @@ sched_dirty_io_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)
-	   & ERTS_SCHDLR_SSPND_CHNG_ONLN);
-
-    if (--schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_wait_curr_online) {
-	erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_io_changing,
-					~ERTS_SCHDLR_SSPND_CHNG_ONLN);
-	if (no != 1)
-	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-    }
-
-    if (no == 1) {
-	while (schdlr_sspnd.dirty_io_curr_online != schdlr_sspnd.dirty_io_wait_curr_online)
-	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-	ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
-    }
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-
     process_main();
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
@@ -9616,11 +9435,11 @@ Process *schedule(Process *p, int calls)
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
 	    if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
-	    rq->procs.pending_exiters = NULL;
-	    erts_smp_runq_unlock(rq);
-	    handle_pending_exiters(pnd_xtrs);
-	    erts_smp_runq_lock(rq);
-	}
+		rq->procs.pending_exiters = NULL;
+		erts_smp_runq_unlock(rq);
+		handle_pending_exiters(pnd_xtrs);
+		erts_smp_runq_lock(rq);
+	    }
 
 	    if (rq->check_balance_reds <= 0)
 		check_balance(rq);
@@ -12537,14 +12356,39 @@ erts_continue_exit_process(Process *p)
 #endif
 
 #ifdef ERTS_SMP
+    if (p->flags & F_SCHDLR_ONLN_WAITQ)
+	abort_sched_onln_chng_waitq(p);
+
     if (p->flags & F_HAVE_BLCKD_MSCHED) {
 	ErtsSchedSuspendResult ssr;
-	ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1);
+	ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 0, 1);
+	switch (ssr) {
+	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
+	    goto yield;
+	case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
+	case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
+	case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
+	case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
+	case ERTS_SCHDLR_SSPND_DONE:
+	case ERTS_SCHDLR_SSPND_YIELD_DONE:
+	    p->flags &= ~F_HAVE_BLCKD_MSCHED;
+	    break;
+	case ERTS_SCHDLR_SSPND_EINVAL:
+	default:
+	    erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n",
+		      __FILE__, __LINE__, (int) ssr);
+	}
+    }
+    if (p->flags & F_HAVE_BLCKD_NMSCHED) {
+	ErtsSchedSuspendResult ssr;
+	ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1, 1);
 	switch (ssr) {
 	case ERTS_SCHDLR_SSPND_YIELD_RESTART:
 	    goto yield;
 	case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
+	case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
 	case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
+	case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
 	case ERTS_SCHDLR_SSPND_DONE:
 	case ERTS_SCHDLR_SSPND_YIELD_DONE:
 	    p->flags &= ~F_HAVE_BLCKD_MSCHED;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 6b370e630e..c88bd7056c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -79,10 +79,8 @@ struct ErtsNodesMonitor_;
 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0
 
 #define ERTS_MAX_NO_OF_SCHEDULERS 1024
-#ifdef ERTS_DIRTY_SCHEDULERS
 #define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
 #define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
-#endif
 
 #define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)
 
@@ -246,7 +244,9 @@ extern int erts_sched_thread_suggested_stack_size;
 
 typedef enum {
     ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
+    ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED,
     ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
+    ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED,
     ERTS_SCHDLR_SSPND_DONE,
     ERTS_SCHDLR_SSPND_YIELD_RESTART,
     ERTS_SCHDLR_SSPND_YIELD_DONE,
@@ -1339,6 +1339,8 @@ extern int erts_system_profile_ts_type;
 #define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */
 #define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */
 #define F_DELAY_GC           (1 << 16) /* Similar to disable GC (see below) */
+#define F_SCHDLR_ONLN_WAITQ  (1 << 17) /* Process enqueued waiting to change schedulers online */
+#define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1533,6 +1535,7 @@ int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c
 void erts_destroy_nif_export(void *); /* see erl_nif.c */
 
 ErtsProcList *erts_proclist_create(Process *);
+ErtsProcList *erts_proclist_copy(ErtsProcList *);
 void erts_proclist_destroy(ErtsProcList *);
 
 ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
@@ -1718,28 +1721,21 @@ void erts_schedule_complete_off_heap_message_queue_change(Eterm pid);
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_dbg_check_halloc_lock(Process *p);
 #endif
-#ifdef DEBUG
-void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
-#endif
-int erts_get_max_no_executing_schedulers(void);
 #if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
-ErtsSchedSuspendResult
-erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
+void
+erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *);
 #endif
 #ifdef ERTS_SMP
 ErtsSchedSuspendResult
 erts_set_schedulers_online(Process *p,
 			   ErtsProcLocks plocks,
 			   Sint new_no,
-			   Sint *old_no
-#ifdef ERTS_DIRTY_SCHEDULERS
-			   , int dirty_only
-#endif
-			   );
+			   Sint *old_no,
+			   int dirty_only);
 ErtsSchedSuspendResult
-erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
+erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int, int);
 int erts_is_multi_scheduling_blocked(void);
-Eterm erts_multi_scheduling_blockers(Process *);
+Eterm erts_multi_scheduling_blockers(Process *, int);
 void erts_start_schedulers(void);
 void erts_alloc_notify_delayed_dealloc(int);
 void erts_alloc_ensure_handle_delayed_dealloc_call(int);
-- 
cgit v1.2.3


From 6664eed554974336909d3ffe03f20349cc4c38fd Mon Sep 17 00:00:00 2001
From: Henrik Nord <henrik@erlang.org>
Date: Tue, 15 Mar 2016 15:19:56 +0100
Subject: update copyright-year

---
 erts/emulator/beam/atom.c                     | 2 +-
 erts/emulator/beam/atom.h                     | 2 +-
 erts/emulator/beam/atom.names                 | 2 +-
 erts/emulator/beam/beam_bif_load.c            | 2 +-
 erts/emulator/beam/beam_bp.c                  | 2 +-
 erts/emulator/beam/beam_bp.h                  | 2 +-
 erts/emulator/beam/beam_catches.c             | 2 +-
 erts/emulator/beam/beam_catches.h             | 2 +-
 erts/emulator/beam/beam_debug.c               | 2 +-
 erts/emulator/beam/beam_emu.c                 | 2 +-
 erts/emulator/beam/beam_load.c                | 2 +-
 erts/emulator/beam/beam_load.h                | 2 +-
 erts/emulator/beam/beam_ranges.c              | 2 +-
 erts/emulator/beam/benchmark.c                | 2 +-
 erts/emulator/beam/benchmark.h                | 2 +-
 erts/emulator/beam/bif.c                      | 2 +-
 erts/emulator/beam/bif.h                      | 2 +-
 erts/emulator/beam/bif.tab                    | 2 +-
 erts/emulator/beam/big.c                      | 2 +-
 erts/emulator/beam/big.h                      | 2 +-
 erts/emulator/beam/binary.c                   | 2 +-
 erts/emulator/beam/break.c                    | 2 +-
 erts/emulator/beam/code_ix.c                  | 2 +-
 erts/emulator/beam/code_ix.h                  | 2 +-
 erts/emulator/beam/copy.c                     | 2 +-
 erts/emulator/beam/dist.c                     | 2 +-
 erts/emulator/beam/dist.h                     | 2 +-
 erts/emulator/beam/dtrace-wrapper.h           | 2 +-
 erts/emulator/beam/elib_memmove.c             | 2 +-
 erts/emulator/beam/erl_afit_alloc.c           | 2 +-
 erts/emulator/beam/erl_afit_alloc.h           | 2 +-
 erts/emulator/beam/erl_alloc.c                | 2 +-
 erts/emulator/beam/erl_alloc.h                | 2 +-
 erts/emulator/beam/erl_alloc.types            | 2 +-
 erts/emulator/beam/erl_alloc_util.c           | 2 +-
 erts/emulator/beam/erl_alloc_util.h           | 2 +-
 erts/emulator/beam/erl_ao_firstfit_alloc.c    | 2 +-
 erts/emulator/beam/erl_ao_firstfit_alloc.h    | 2 +-
 erts/emulator/beam/erl_arith.c                | 2 +-
 erts/emulator/beam/erl_async.c                | 2 +-
 erts/emulator/beam/erl_async.h                | 2 +-
 erts/emulator/beam/erl_bestfit_alloc.c        | 2 +-
 erts/emulator/beam/erl_bestfit_alloc.h        | 2 +-
 erts/emulator/beam/erl_bif_binary.c           | 2 +-
 erts/emulator/beam/erl_bif_chksum.c           | 2 +-
 erts/emulator/beam/erl_bif_ddll.c             | 2 +-
 erts/emulator/beam/erl_bif_guard.c            | 2 +-
 erts/emulator/beam/erl_bif_info.c             | 2 +-
 erts/emulator/beam/erl_bif_lists.c            | 2 +-
 erts/emulator/beam/erl_bif_op.c               | 2 +-
 erts/emulator/beam/erl_bif_os.c               | 2 +-
 erts/emulator/beam/erl_bif_port.c             | 2 +-
 erts/emulator/beam/erl_bif_re.c               | 2 +-
 erts/emulator/beam/erl_bif_trace.c            | 2 +-
 erts/emulator/beam/erl_bif_unique.c           | 2 +-
 erts/emulator/beam/erl_bif_unique.h           | 2 +-
 erts/emulator/beam/erl_binary.h               | 2 +-
 erts/emulator/beam/erl_bits.c                 | 2 +-
 erts/emulator/beam/erl_bits.h                 | 2 +-
 erts/emulator/beam/erl_cpu_topology.c         | 2 +-
 erts/emulator/beam/erl_cpu_topology.h         | 2 +-
 erts/emulator/beam/erl_db.c                   | 2 +-
 erts/emulator/beam/erl_db.h                   | 2 +-
 erts/emulator/beam/erl_db_hash.c              | 2 +-
 erts/emulator/beam/erl_db_hash.h              | 2 +-
 erts/emulator/beam/erl_db_tree.c              | 2 +-
 erts/emulator/beam/erl_db_tree.h              | 2 +-
 erts/emulator/beam/erl_db_util.c              | 2 +-
 erts/emulator/beam/erl_db_util.h              | 2 +-
 erts/emulator/beam/erl_debug.c                | 2 +-
 erts/emulator/beam/erl_driver.h               | 2 +-
 erts/emulator/beam/erl_drv_nif.h              | 2 +-
 erts/emulator/beam/erl_drv_thread.c           | 2 +-
 erts/emulator/beam/erl_fun.c                  | 2 +-
 erts/emulator/beam/erl_fun.h                  | 2 +-
 erts/emulator/beam/erl_gc.c                   | 2 +-
 erts/emulator/beam/erl_gc.h                   | 2 +-
 erts/emulator/beam/erl_goodfit_alloc.c        | 2 +-
 erts/emulator/beam/erl_goodfit_alloc.h        | 2 +-
 erts/emulator/beam/erl_hl_timer.c             | 2 +-
 erts/emulator/beam/erl_init.c                 | 2 +-
 erts/emulator/beam/erl_instrument.c           | 2 +-
 erts/emulator/beam/erl_instrument.h           | 2 +-
 erts/emulator/beam/erl_lock_check.c           | 2 +-
 erts/emulator/beam/erl_lock_check.h           | 2 +-
 erts/emulator/beam/erl_lock_count.c           | 2 +-
 erts/emulator/beam/erl_lock_count.h           | 2 +-
 erts/emulator/beam/erl_map.c                  | 2 +-
 erts/emulator/beam/erl_map.h                  | 2 +-
 erts/emulator/beam/erl_math.c                 | 2 +-
 erts/emulator/beam/erl_message.c              | 2 +-
 erts/emulator/beam/erl_message.h              | 2 +-
 erts/emulator/beam/erl_monitors.c             | 2 +-
 erts/emulator/beam/erl_monitors.h             | 2 +-
 erts/emulator/beam/erl_mtrace.c               | 2 +-
 erts/emulator/beam/erl_mtrace.h               | 2 +-
 erts/emulator/beam/erl_nif.c                  | 2 +-
 erts/emulator/beam/erl_nif.h                  | 2 +-
 erts/emulator/beam/erl_nif_api_funcs.h        | 2 +-
 erts/emulator/beam/erl_node_container_utils.h | 2 +-
 erts/emulator/beam/erl_node_tables.c          | 2 +-
 erts/emulator/beam/erl_node_tables.h          | 2 +-
 erts/emulator/beam/erl_port.h                 | 2 +-
 erts/emulator/beam/erl_port_task.c            | 2 +-
 erts/emulator/beam/erl_port_task.h            | 2 +-
 erts/emulator/beam/erl_printf_term.c          | 2 +-
 erts/emulator/beam/erl_printf_term.h          | 2 +-
 erts/emulator/beam/erl_process.c              | 2 +-
 erts/emulator/beam/erl_process.h              | 2 +-
 erts/emulator/beam/erl_process_dict.c         | 2 +-
 erts/emulator/beam/erl_process_dict.h         | 2 +-
 erts/emulator/beam/erl_process_dump.c         | 2 +-
 erts/emulator/beam/erl_process_lock.c         | 2 +-
 erts/emulator/beam/erl_process_lock.h         | 2 +-
 erts/emulator/beam/erl_ptab.c                 | 2 +-
 erts/emulator/beam/erl_ptab.h                 | 2 +-
 erts/emulator/beam/erl_sched_spec_pre_alloc.c | 2 +-
 erts/emulator/beam/erl_sched_spec_pre_alloc.h | 2 +-
 erts/emulator/beam/erl_smp.h                  | 2 +-
 erts/emulator/beam/erl_sock.h                 | 2 +-
 erts/emulator/beam/erl_sys_driver.h           | 2 +-
 erts/emulator/beam/erl_term.c                 | 2 +-
 erts/emulator/beam/erl_term.h                 | 2 +-
 erts/emulator/beam/erl_thr_progress.c         | 2 +-
 erts/emulator/beam/erl_thr_progress.h         | 2 +-
 erts/emulator/beam/erl_thr_queue.c            | 2 +-
 erts/emulator/beam/erl_thr_queue.h            | 2 +-
 erts/emulator/beam/erl_threads.h              | 2 +-
 erts/emulator/beam/erl_time.h                 | 2 +-
 erts/emulator/beam/erl_time_sup.c             | 2 +-
 erts/emulator/beam/erl_trace.c                | 2 +-
 erts/emulator/beam/erl_trace.h                | 2 +-
 erts/emulator/beam/erl_unicode.c              | 2 +-
 erts/emulator/beam/erl_unicode.h              | 2 +-
 erts/emulator/beam/erl_unicode_normalize.h    | 2 +-
 erts/emulator/beam/erl_utils.h                | 2 +-
 erts/emulator/beam/erl_vm.h                   | 2 +-
 erts/emulator/beam/erl_zlib.c                 | 2 +-
 erts/emulator/beam/erl_zlib.h                 | 2 +-
 erts/emulator/beam/erlang_dtrace.d            | 2 +-
 erts/emulator/beam/error.h                    | 2 +-
 erts/emulator/beam/export.c                   | 2 +-
 erts/emulator/beam/export.h                   | 2 +-
 erts/emulator/beam/external.c                 | 2 +-
 erts/emulator/beam/external.h                 | 2 +-
 erts/emulator/beam/global.h                   | 2 +-
 erts/emulator/beam/hash.c                     | 2 +-
 erts/emulator/beam/hash.h                     | 2 +-
 erts/emulator/beam/index.c                    | 2 +-
 erts/emulator/beam/index.h                    | 2 +-
 erts/emulator/beam/io.c                       | 2 +-
 erts/emulator/beam/module.c                   | 2 +-
 erts/emulator/beam/module.h                   | 2 +-
 erts/emulator/beam/ops.tab                    | 2 +-
 erts/emulator/beam/packet_parser.c            | 2 +-
 erts/emulator/beam/packet_parser.h            | 2 +-
 erts/emulator/beam/register.c                 | 2 +-
 erts/emulator/beam/register.h                 | 2 +-
 erts/emulator/beam/safe_hash.c                | 2 +-
 erts/emulator/beam/safe_hash.h                | 2 +-
 erts/emulator/beam/sys.h                      | 2 +-
 erts/emulator/beam/time.c                     | 2 +-
 erts/emulator/beam/utils.c                    | 2 +-
 erts/emulator/beam/version.h                  | 2 +-
 164 files changed, 164 insertions(+), 164 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 099c00bcf6..a5e778e4aa 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index 2c002ca92f..fbd0528009 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 169b071cd7..6080f733ec 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2013. All Rights Reserved.
+# Copyright Ericsson AB 1996-2016. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 0b47fc3586..320dee6668 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 5d471d168b..86e1fdc2fb 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 2b89d6fc71..f9eca94e2a 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c
index 7a1f4901aa..cd592c7e5e 100644
--- a/erts/emulator/beam/beam_catches.c
+++ b/erts/emulator/beam/beam_catches.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h
index 59ee64d033..8eb2165ac9 100644
--- a/erts/emulator/beam/beam_catches.h
+++ b/erts/emulator/beam/beam_catches.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index e37bd4d78c..61149721fe 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a390422040..04cb1fbdf9 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index f115df935f..95709db914 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 22ab71c868..6fd9b4ff2a 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 54c337ee72..55342a38c6 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c
index 44f5c760c2..c8409784ef 100644
--- a/erts/emulator/beam/benchmark.c
+++ b/erts/emulator/beam/benchmark.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2002-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h
index 0fb0b93f12..0272896f4f 100644
--- a/erts/emulator/beam/benchmark.h
+++ b/erts/emulator/beam/benchmark.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2002-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 37bb28c6f8..3ca98a4066 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 46af552b39..546e3830b9 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index a6670c10e8..9bb77190d6 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2013. All Rights Reserved.
+# Copyright Ericsson AB 1996-2016. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index c738a9ecf5..4baee7900b 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index ee51e5d313..464acd67f6 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 3fd284b589..cfd8fbe2f5 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 8c039ef132..057ec885e2 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 1b0968c55c..9da750e366 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 7a66211a5b..584a605771 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index b38ebe066b..ccc4cbad43 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index fa385f105d..32a4c9400d 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index fb777d9ac1..3e6e4710c3 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h
index d343dc5ab0..6f70d5961e 100644
--- a/erts/emulator/beam/dtrace-wrapper.h
+++ b/erts/emulator/beam/dtrace-wrapper.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012.
+ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2016.
  * All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/erts/emulator/beam/elib_memmove.c b/erts/emulator/beam/elib_memmove.c
index d4ca30158e..2f45f69026 100644
--- a/erts/emulator/beam/elib_memmove.c
+++ b/erts/emulator/beam/elib_memmove.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index 4b0541c10e..eda3ad870a 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h
index ef050ff50e..74258e284a 100644
--- a/erts/emulator/beam/erl_afit_alloc.h
+++ b/erts/emulator/beam/erl_afit_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 490e0c0915..d58651fa86 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 71e4713624..b2ca0deb02 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 2932adca84..b7fa978896 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2003-2014. All Rights Reserved.
+# Copyright Ericsson AB 2003-2016. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 5e7dd7cce8..6812551075 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index b7d717ed23..b8bdf971c6 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 5cd067ff54..fbe4724047 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index 4200f20622..7349c6ab19 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index 3671025d22..861532f241 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index cdeeb5281b..bf18eef730 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h
index 65538bcef0..473c7686e5 100644
--- a/erts/emulator/beam/erl_async.h
+++ b/erts/emulator/beam/erl_async.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index f39a18ac88..379cee39a1 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h
index b315518b88..3a5f51f5dc 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.h
+++ b/erts/emulator/beam/erl_bestfit_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index aec72bd61a..6e10980b6b 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index e3074d6309..9417803e14 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index f006856b6a..3529a66383 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index 4a9a6a5fcd..b42d2dc28b 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 8c748c9bf7..d4499db291 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index fe64e76575..fcccb5a4a0 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index 0f20ded1d6..aecb8bf0c1 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index 2333ca0851..46777d3aa5 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 86bcf43678..648f33744a 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 7f7cd376ac..f4daecd2a4 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 71a7079b09..a9443ee8df 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index c4a39b8897..08b0baf989 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 37d5d91c39..5eb2e2a619 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index e181b5555d..fdc5efea98 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 11d83686a3..6bf52fb303 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 8b7807fbd9..1c2a090f07 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index c263eebfcc..28aaeeb479 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h
index f3fbfc6da0..45324ac4a0 100644
--- a/erts/emulator/beam/erl_cpu_topology.h
+++ b/erts/emulator/beam/erl_cpu_topology.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 441dbdde4f..615d23402b 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index a589af784c..1d26c49652 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 240b79fc60..74979f984a 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 66d8ec71d9..e654363cd5 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 0f642f0867..02d211a4bb 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h
index 6098387f5d..72749ead1e 100644
--- a/erts/emulator/beam/erl_db_tree.h
+++ b/erts/emulator/beam/erl_db_tree.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 973be58420..be14386b14 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 911ed37aef..5d7946a525 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 2902c98864..3e3bfa03a2 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 0234b9b0e4..97a69140c3 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index f6b946ae82..2700b62854 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 184f8e8931..92edce5176 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index ed555aa92f..6ce1376c81 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 0024b1ff71..8c4deea7a0 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 0d125c5598..62417fa05c 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 2cedd9361f..40b7c5d12c 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index 9b4aad9d91..223ba193da 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h
index ababdbd0a1..76dd558234 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.h
+++ b/erts/emulator/beam/erl_goodfit_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 491d2f8c84..1a621863bb 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 86c4eeb484..2f03a37b79 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index 12a72ad839..f84c63e7a4 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h
index cb3b1920d3..1f04c91d5e 100644
--- a/erts/emulator/beam/erl_instrument.h
+++ b/erts/emulator/beam/erl_instrument.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 598bf84c0b..9231fb1b34 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 66251ef4e8..18296d1fec 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index bd00480ba2..a00e0f0fff 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 4cc6a5c695..3e8dcefe69 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4b6931f848..b74eb04b5b 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 052fa99f03..a40070d00d 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2014. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c
index b46cc37495..fc0aaed18a 100644
--- a/erts/emulator/beam/erl_math.c
+++ b/erts/emulator/beam/erl_math.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 88efb2c59f..a6bef42f77 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 60035d15ae..ad4e65274c 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index bd899fa2f9..910598690d 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h
index 901ed8940b..9e2beedea3 100644
--- a/erts/emulator/beam/erl_monitors.h
+++ b/erts/emulator/beam/erl_monitors.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index fdaf02f37a..e275867928 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_mtrace.h b/erts/emulator/beam/erl_mtrace.h
index b34eab3c4a..776c70a819 100644
--- a/erts/emulator/beam/erl_mtrace.h
+++ b/erts/emulator/beam/erl_mtrace.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8580bc2689..d0ac74d22d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index c9c8b1d0af..dd4f85723d 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 1448a508a2..a444045357 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index b6a3531e28..0c76c6fe7d 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 8617f42d7b..6679393287 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index fb2f2a5407..8ee2708875 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index fa97707a87..7510f94df1 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index e85395e062..fefaede079 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 335f7a77d5..a11fc6c9dc 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 45ba4371dc..1a579704a8 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h
index 4618457f27..8a30286fd8 100644
--- a/erts/emulator/beam/erl_printf_term.h
+++ b/erts/emulator/beam/erl_printf_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a706ebf595..2648ed0cde 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c88bd7056c..15a524a9e6 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 487b944fc0..d8c2eaba94 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index dac214c8a1..387562058c 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 39c36ee7a9..cd62593945 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 0bee2c848c..e1f470d381 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 9c59301086..ac324c6f3e 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2007-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index 9ed175fe01..22830a19c4 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index 8fd961e3ce..97df8bc3fc 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index caec24bc03..cab4bd73db 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
index 4d07b0f674..7808d7d438 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 5fc5e989a6..713ed50b86 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_sock.h b/erts/emulator/beam/erl_sock.h
index 7be6062115..3429a52d7e 100644
--- a/erts/emulator/beam/erl_sock.h
+++ b/erts/emulator/beam/erl_sock.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_sys_driver.h b/erts/emulator/beam/erl_sys_driver.h
index 4031eef0aa..d46e88cb05 100644
--- a/erts/emulator/beam/erl_sys_driver.h
+++ b/erts/emulator/beam/erl_sys_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2001-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index d2343912b4..9cbd156800 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 2b28762db5..811127a1cb 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 7b06fd840f..542541165b 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index b89cc4c267..b2894ba1fe 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index 7ff456b915..b813041d9a 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h
index 27a6d03224..68356582e5 100644
--- a/erts/emulator/beam/erl_thr_queue.h
+++ b/erts/emulator/beam/erl_thr_queue.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index e689547d1d..eccd49f2a9 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 5242063550..a1c4220633 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index f3f528eaf6..947def7398 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2015. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 41876b8c62..2defd0a2eb 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index a0058264d7..1da27ee128 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 36d85b0a22..3c3536c021 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_unicode.h b/erts/emulator/beam/erl_unicode.h
index 4c25d89b7c..e01eaa787e 100644
--- a/erts/emulator/beam/erl_unicode.h
+++ b/erts/emulator/beam/erl_unicode.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h
index 16c62db50e..21e2a52544 100644
--- a/erts/emulator/beam/erl_unicode_normalize.h
+++ b/erts/emulator/beam/erl_unicode_normalize.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 640969daf0..81800752f0 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 98f27a1725..6226522740 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c
index e44e86c39c..944ff2e35f 100644
--- a/erts/emulator/beam/erl_zlib.c
+++ b/erts/emulator/beam/erl_zlib.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h
index 2ca1e3735e..c83c6f291f 100644
--- a/erts/emulator/beam/erl_zlib.h
+++ b/erts/emulator/beam/erl_zlib.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d
index c682f76e3a..73ef5a108a 100644
--- a/erts/emulator/beam/erlang_dtrace.d
+++ b/erts/emulator/beam/erlang_dtrace.d
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012.
+ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2016.
  * All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index cba8672c68..6c33b12dd0 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 581efe6eec..02c24557c1 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index a8bc9d2f66..8c81cbd410 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index aac1490f0c..6bb62d1040 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index d12051c6b4..52a0757a1c 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d7edf451be..1d3704f0af 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index 5a0b93f693..e255b961f1 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h
index e94aaa0a84..9f773d8faa 100644
--- a/erts/emulator/beam/hash.h
+++ b/erts/emulator/beam/hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index 5f6ea14732..26d6c04ea0 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 99b2bdfab0..218779c33b 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 797e4cdadc..6179720f97 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index f5c7b177d3..3eb11f1693 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index b7468b0926..4e12731d85 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 9e53b4bfcc..a1f021d7e0 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2013. All Rights Reserved.
+# Copyright Ericsson AB 1997-2016. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index a737a86f14..f14910bc72 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h
index 717d905fad..358d650804 100644
--- a/erts/emulator/beam/packet_parser.h
+++ b/erts/emulator/beam/packet_parser.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index ebc35b0c4d..071f6aee08 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h
index 144536f34b..88ab7b7bf1 100644
--- a/erts/emulator/beam/register.h
+++ b/erts/emulator/beam/register.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 3f039c8dfd..30b26a7296 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h
index a11370813c..6910b33004 100644
--- a/erts/emulator/beam/safe_hash.h
+++ b/erts/emulator/beam/safe_hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 03b9088adc..52624a76e8 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 0ce4f443f3..6f15082130 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 66fcfcf6ce..384b340a1c 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2014. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/version.h b/erts/emulator/beam/version.h
index 004725eaf5..0fa775fe8c 100644
--- a/erts/emulator/beam/version.h
+++ b/erts/emulator/beam/version.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From 2deb875adb4a921e758223a45fcd2e2df89124bc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Mar 2016 15:51:38 +0100
Subject: erts: Fix erts_debug:set_internal_state(wait,deallocations)

that could hang if concurrent deallocations was initiated.
---
 erts/emulator/beam/erl_alloc.types |  2 ++
 erts/emulator/beam/erl_process.c   | 57 ++++++++++++++++++++++----------------
 2 files changed, 35 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 2932adca84..14067283bd 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -343,6 +343,8 @@ type	SSB		SHORT_LIVED	PROCESSES	ssb
 
 +endif
 
+type	DEBUG	        SHORT_LIVED	SYSTEM          debugging
+
 type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
 type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
 type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a706ebf595..d2185c4fbc 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2211,8 +2211,7 @@ setup_thr_debug_wait_completed(void *vproc)
     if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) {
 	erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
 	wait_flags |= (ERTS_SSI_AUX_WORK_DD
-		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR
-		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR);
 #ifdef ERTS_SMP
 	aux_work_flags |= ERTS_SSI_AUX_WORK_DD;
 #endif
@@ -2220,8 +2219,7 @@ setup_thr_debug_wait_completed(void *vproc)
 
     if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) {
 	wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS
-		       | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR
-		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+		       | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
 #ifdef ERTS_SMP
 	if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp))
 	    aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
@@ -2235,26 +2233,42 @@ setup_thr_debug_wait_completed(void *vproc)
     awdp->debug.wait_completed.arg = vproc;
 }
 
-static void
-prep_setup_thr_debug_wait_completed(void *vproc)
+struct debug_lop {
+    ErtsThrPrgrLaterOp lop;
+    Process *proc;
+};
+
+static void later_thr_debug_wait_completed(void *vlop)
 {
+    struct debug_lop *lop = vlop;
     erts_aint32_t count = (erts_aint32_t) erts_no_schedulers;
 #ifdef ERTS_SMP
     count += 1; /* aux thread */
 #endif
     if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) {
-	/* scheduler threads */
-	erts_schedule_multi_misc_aux_work(0,
-					  erts_no_schedulers,
-					  setup_thr_debug_wait_completed,
-					  vproc);
+        /* scheduler threads */
+        erts_schedule_multi_misc_aux_work(0,
+                                          erts_no_schedulers,
+                                          setup_thr_debug_wait_completed,
+                                          lop->proc);
 #ifdef ERTS_SMP
-	/* aux_thread */
-	erts_schedule_misc_aux_work(0,
-				    setup_thr_debug_wait_completed,
-				    vproc);
+        /* aux_thread */
+        erts_schedule_misc_aux_work(0,
+                                    setup_thr_debug_wait_completed,
+                                    lop->proc);
 #endif
     }
+    erts_free(ERTS_ALC_T_DEBUG, lop);
+}
+
+
+static void
+init_thr_debug_wait_completed(void *vproc)
+{
+    struct debug_lop* lop = erts_alloc(ERTS_ALC_T_DEBUG,
+                                       sizeof(struct debug_lop));
+    lop->proc = vproc;
+    erts_schedule_thr_prgr_later_op(later_thr_debug_wait_completed, lop, &lop->lop);
 }
 
 
@@ -2264,7 +2278,7 @@ erts_debug_wait_completed(Process *c_p, int flags)
     /* Only one process at a time can do this */
     erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers);
 #ifdef ERTS_SMP
-    count += 2; /* aux thread */
+    count += 1; /* aux thread */
 #endif
     if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count,
 				      count,
@@ -2272,17 +2286,12 @@ erts_debug_wait_completed(Process *c_p, int flags)
 	debug_wait_completed_flags = flags;
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	erts_proc_inc_refc(c_p);
-	/* scheduler threads */
+
+        /* First flush later-ops on all scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
-					  prep_setup_thr_debug_wait_completed,
+					  init_thr_debug_wait_completed,
 					  (void *) c_p);
-#ifdef ERTS_SMP
-	/* aux_thread */
-	erts_schedule_misc_aux_work(0,
-				    prep_setup_thr_debug_wait_completed,
-				    (void *) c_p);
-#endif
 	return 1;
     }
     return 0;
-- 
cgit v1.2.3


From aaecaac6a392eb8bc6362d1a523858846ac8d670 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Wed, 16 Mar 2016 15:35:44 +0100
Subject: erts: Create erl_crash.dump when out of memory

This was accidentally removed in commit cd6903be0740db
---
 erts/emulator/beam/erl_alloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 0877c24404..5f8cd2bbf7 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1923,7 +1923,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
 	va_start(argp, n);
 	size = va_arg(argp, Uint);
 	va_end(argp);
-	erts_exit(1,
+	erts_exit(ERTS_DUMP_EXIT,
 		 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
 		 allctr_str, op, size, t_str);
 	break;
-- 
cgit v1.2.3


From 0fe04b07b4d15c9671d6665ac5304bffd0f63d23 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 16 Mar 2016 13:42:04 +0100
Subject: Unbreak process_info(Pid,last_calls)

---
 erts/emulator/beam/erl_bif_info.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 2c232c6c03..ac7a70c642 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1534,7 +1534,7 @@ process_info_aux(Process *BIF_P,
     }
 
     case am_last_calls: {
-	struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(BIF_P);
+	struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp);
 	if (!scb) {
 	    hp = HAlloc(BIF_P, 3);
 	    res = am_false;
-- 
cgit v1.2.3


From da035ea5b65610bd9d19765609ed924c3fe0bdd1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 16 Mar 2016 17:34:05 +0100
Subject: erts: Fix LOW_WRITE section for non llvm os x compilation

---
 erts/emulator/beam/sys.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 03b9088adc..44735c0ec0 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -138,10 +138,10 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #endif
 
 #if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
-#ifndef __llvm__
-#  define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("ERTS_LOW_WRITE") ))
-#else
+#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__)
 #  define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") ))
+#else
+#  define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("ERTS_LOW_WRITE") ))
 #endif
 #else
 #  define ERTS_WRITE_UNLIKELY(X) X
-- 
cgit v1.2.3


From 3c2d5d0dccd34cba89880cd4c1ded616d19696c2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 17 Mar 2016 21:54:40 +0100
Subject: Allow delayed gc while scheduled out

---
 erts/emulator/beam/erl_process.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a706ebf595..7fcf977e08 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9324,8 +9324,6 @@ Process *schedule(Process *p, int calls)
     } else {
     sched_out_proc:
 
-	ASSERT(!(p->flags & F_DELAY_GC));
-
 #ifdef ERTS_SMP
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 	esdp = p->scheduler_data;
@@ -9886,15 +9884,24 @@ Process *schedule(Process *p, int calls)
 #endif
 
 	if (state & ERTS_PSFLG_RUNNING_SYS) {
-	    reds -= execute_sys_tasks(p, &state, reds);
-	    if (reds <= 0
+	    /*
+	     * GC is normally never delayed when a process
+	     * is scheduled out, but might be when executing
+	     * hand written beam assembly in
+	     * prim_eval:'receive'. If GC is delayed we are
+	     * not allowed to execute system tasks.
+	     */
+	    if (!(p->flags & F_DELAY_GC)) {
+		reds -= execute_sys_tasks(p, &state, reds);
+		if (reds <= 0
 #ifdef ERTS_DIRTY_SCHEDULERS
-		|| ERTS_SCHEDULER_IS_DIRTY(esdp)
-		|| (state & ERTS_PSFLGS_DIRTY_WORK)
+		    || ERTS_SCHEDULER_IS_DIRTY(esdp)
+		    || (state & ERTS_PSFLGS_DIRTY_WORK)
 #endif
-		) {
-		p->fcalls = reds;
-		goto sched_out_proc;
+		    ) {
+		    p->fcalls = reds;
+		    goto sched_out_proc;
+		}
 	    }
 
 	    ASSERT(state & ERTS_PSFLG_RUNNING_SYS);
@@ -9924,7 +9931,7 @@ Process *schedule(Process *p, int calls)
 	}
 
 	if (ERTS_IS_GC_DESIRED(p)) {
-	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) {
+	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
 		reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity);
 		if (reds <= 0) {
 		    p->fcalls = reds;
-- 
cgit v1.2.3


From 455cbcf2230715b37da14573cb9ba60a68c24a26 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 18 Mar 2016 11:55:48 +0100
Subject: Fix implementation of dropped signal to port

---
 erts/emulator/beam/io.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 8c67f731f4..9a67e05bfa 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1501,8 +1501,19 @@ erts_schedule_proc2port_signal(Process *c_p,
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
     if (sched_res != 0) {
-	if (refp)
+	if (refp) {
+	    /*
+	     * We need to restore the message queue save
+	     * pointer to the beginning of the message queue
+	     * since the caller now wont wait for a message
+	     * containing the reference created above...
+	     */
+	    ASSERT(c_p);
+	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    JOIN_MESSAGE(c_p);
+	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    *refp = NIL;
+	}
 	return ERTS_PORT_OP_DROPPED;
     }
     return ERTS_PORT_OP_SCHEDULED;
-- 
cgit v1.2.3


From dfa3b2414384f1ee0cdc062a23df5f1094a7069e Mon Sep 17 00:00:00 2001
From: Henrik Nord <henrik@erlang.org>
Date: Thu, 24 Mar 2016 13:28:53 +0100
Subject: erts: std_alloc is not thread safe on non-smp

---
 erts/emulator/beam/erl_alloc.types | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 14067283bd..7a5f821ed8 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -258,7 +258,6 @@ type	PRTSD		STANDARD	SYSTEM		port_specific_data
 type	CPUDATA		LONG_LIVED	SYSTEM		cpu_data
 type	TMP_CPU_IDS	SHORT_LIVED	SYSTEM		tmp_cpu_ids
 type	EXT_TERM_DATA	SHORT_LIVED	PROCESSES	external_term_data
-type	ZLIB		STANDARD	SYSTEM		zlib
 type	CPU_GRPS_MAP	LONG_LIVED	SYSTEM		cpu_groups_map
 type	AUX_WORK_TMO	LONG_LIVED	SYSTEM		aux_work_timeouts
 type	MISC_AUX_WORK_Q	LONG_LIVED	SYSTEM		misc_aux_work_q
@@ -288,8 +287,10 @@ type	THR_Q_LL	LONG_LIVED	SYSTEM		long_lived_thr_queue
 
 +if smp
 type	ASYNC		SHORT_LIVED	SYSTEM		async
+type	ZLIB		STANDARD	SYSTEM		zlib
 +else
-# sl_alloc is not thread safe in non smp build; therefore, we use driver_alloc
+# sl/std_alloc is not thread safe in non smp build; therefore, we use driver_alloc
+type	ZLIB		DRIVER   	SYSTEM		zlib
 type	ASYNC		DRIVER		SYSTEM		async
 +endif
 
-- 
cgit v1.2.3


From ab884c3fdda00479d636de82d68ffbd8628c5c20 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 24 Mar 2016 16:20:51 +0100
Subject: Improve process/port specific data management

---
 erts/emulator/beam/beam_bp.c          |   6 +-
 erts/emulator/beam/bif.c              |   6 +-
 erts/emulator/beam/erl_nif.c          |   2 +-
 erts/emulator/beam/erl_port.h         |  47 +++++++++---
 erts/emulator/beam/erl_process.c      | 132 +++++++++++++---------------------
 erts/emulator/beam/erl_process.h      |  83 +++++++++++----------
 erts/emulator/beam/erl_process_dump.c |   2 +-
 erts/emulator/beam/io.c               |   8 ++-
 8 files changed, 146 insertions(+), 140 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 5d471d168b..74c9d3ee53 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -974,7 +974,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
     if (pbt == 0) {
 	/* First call of process to instrumented function */
 	pbt = Alloc(sizeof(process_breakpoint_time_t));
-	(void) ERTS_PROC_SET_CALL_TIME(c_p, ERTS_PROC_LOCK_MAIN, pbt);
+	(void) ERTS_PROC_SET_CALL_TIME(c_p, pbt);
     } else {
 	ASSERT(pbt->pc);
 	/* add time to previous code */
@@ -1598,9 +1598,7 @@ bp_time_unref(BpDataTime* bdt)
 			h_p = erts_pid2proc(NULL, 0, item->pid,
 					    ERTS_PROC_LOCK_MAIN);
 			if (h_p) {
-			    pbt = ERTS_PROC_SET_CALL_TIME(h_p,
-							  ERTS_PROC_LOCK_MAIN,
-							  NULL);
+			    pbt = ERTS_PROC_SET_CALL_TIME(h_p, NULL);
 			    if (pbt) {
 				Free(pbt);
 			    }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 37bb28c6f8..97d690db9f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1567,7 +1567,7 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
 	   scb->n = 0;
        }
 
-       scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, ERTS_PROC_LOCK_MAIN, scb);
+       scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
 
        if (!scb)
 	   old_value = make_small(0);
@@ -1595,9 +1595,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
       if (is_not_atom(BIF_ARG_2)) {
 	 goto error;
       }
-      old_value = erts_proc_set_error_handler(BIF_P,
-					      ERTS_PROC_LOCK_MAIN,
-					      BIF_ARG_2);
+      old_value = erts_proc_set_error_handler(BIF_P, BIF_ARG_2);
       BIF_RET(old_value);
    }
    else if (BIF_ARG_1 == am_priority) {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8580bc2689..855d5deea1 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1653,7 +1653,7 @@ allocate_nif_sched_data(Process* proc, int argc)
 	ep->exp.addressv[i] = &ep->exp.code[3];
     }
     ep->exp.code[3] = (BeamInstr) em_call_nif;
-    (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, ep);
+    (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep);
     return ep;
 }
 
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index fa97707a87..c25e08545e 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -185,7 +185,7 @@ struct _erl_drv_port {
     int control_flags;		 /* Flags for port_control()  */
     ErlDrvPDL port_data_lock;
 
-    ErtsPrtSD *psd;		 /* Port specific data */
+    erts_smp_atomic_t psd;	 /* Port specific data */
     int reds; /* Only used while executing driver callbacks */
 
     struct {
@@ -252,22 +252,51 @@ ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new);
 ERTS_GLB_INLINE void *
 erts_prtsd_get(Port *prt, int ix)
 {
-    return prt->psd ? prt->psd->data[ix] : NULL;
+    ErtsPrtSD *psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd);
+    if (!psd)
+	return NULL;
+    ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+    return psd->data[ix];
 }
 
 ERTS_GLB_INLINE void *
 erts_prtsd_set(Port *prt, int ix, void *data)
 {
-    if (prt->psd) {
-	void *old = prt->psd->data[ix];
-	prt->psd->data[ix] = data;
+    ErtsPrtSD *psd, *new_psd;
+    void *old;
+    int i;
+
+    psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd);
+
+    if (psd) {
+#ifdef ERTS_SMP
+#ifdef ETHR_ORDERED_READ_DEPEND
+	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+#else
+	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore);
+#endif
+#endif
+	old = psd->data[ix];
+	psd->data[ix] = data;
 	return old;
     }
-    else {
-	prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
-	prt->psd->data[ix] = data;
+
+    if (!data)
 	return NULL;
-    }
+
+    new_psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
+    for (i = 0; i < ERTS_PRTSD_SIZE; i++)
+	new_psd->data[i] = NULL;
+    psd = (ErtsPrtSD *) erts_smp_atomic_cmpxchg_mb(&prt->psd,
+						   (erts_aint_t) new_psd,
+						   (erts_aint_t) NULL);
+    if (psd)
+	erts_free(ERTS_ALC_T_PRTSD, new_psd);
+    else
+	psd = new_psd;
+    old = psd->data[ix];
+    psd->data[ix] = data;
+    return old;
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d2185c4fbc..758f350fa9 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -687,45 +687,36 @@ erts_pre_init_process(void)
 	= "DEBUG_WAIT_COMPLETED";
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
- {
-     int ix;
 
-     erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks
-	 = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks
-	 = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks
+	= ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks
+	= ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS;
 
-     erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks
-	 = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks
-	 = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks
+	= ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks
+	= ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS;
 
-     erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks
-	 = ERTS_PSD_SCHED_ID_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks
-	 = ERTS_PSD_SCHED_ID_SET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks
+	= ERTS_PSD_SCHED_ID_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks
+	= ERTS_PSD_SCHED_ID_SET_LOCKS;
 
-     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks
-	 = ERTS_PSD_CALL_TIME_BP_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
-	 = ERTS_PSD_CALL_TIME_BP_SET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks
+	= ERTS_PSD_CALL_TIME_BP_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
+	= ERTS_PSD_CALL_TIME_BP_SET_LOCKS;
 
-     erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks
-	 = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks
-	 = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks
+	= ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks
+	= ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS;
 
-     erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks
-	 = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS;
-     erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks
-	 = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS;
-
-     /* Check that we have locks for all entries */
-     for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
-	 ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks);
-	 ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks);
-     }
- }
+    erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks
+	= ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks
+	= ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS;
 #endif
 }
 
@@ -1314,46 +1305,25 @@ erts_proclist_destroy(ErtsProcList *plp)
 }
 
 void *
-erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
+erts_psd_set_init(Process *p, int ix, void *data)
 {
     void *old;
-    ErtsProcLocks xplocks;
-    int refc = 0;
-    ErtsPSD *psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD));
+    ErtsPSD *psd, *new_psd;
     int i;
-    for (i = 0; i < ERTS_PSD_SIZE; i++)
-	psd->data[i] = NULL;
 
-    ERTS_SMP_LC_ASSERT(plocks);
-    ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p));
+    new_psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD));
+    for (i = 0; i < ERTS_PSD_SIZE; i++)
+	new_psd->data[i] = NULL;
 
-    xplocks = ERTS_PROC_LOCKS_ALL;
-    xplocks &= ~plocks;
-    if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) {
-	if (xplocks & ERTS_PROC_LOCK_MAIN) {
-	    erts_proc_inc_refc(p);
-	    erts_smp_proc_unlock(p, plocks);
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL);
-	    refc = 1;
-	}
-	else {
-	    if (plocks & ERTS_PROC_LOCKS_ALL_MINOR)
-		erts_smp_proc_unlock(p, plocks & ERTS_PROC_LOCKS_ALL_MINOR);
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-	}
-    }
-    if (!p->psd)
-	p->psd = psd;
-    if (xplocks)
-	erts_smp_proc_unlock(p, xplocks);
-    if (refc)
-	erts_proc_dec_refc(p);
-    ASSERT(p->psd);
-    if (p->psd != psd)
-	erts_free(ERTS_ALC_T_PSD, psd);
-    old = p->psd->data[ix];
-    p->psd->data[ix] = data;
-    ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p));
+    psd = (ErtsPSD *) erts_smp_atomic_cmpxchg_mb(&p->psd,
+						 (erts_aint_t) new_psd,
+						 (erts_aint_t) NULL);
+    if (psd)
+	erts_free(ERTS_ALC_T_PSD, new_psd);
+    else
+	psd = new_psd;
+    old = psd->data[ix];
+    psd->data[ix] = data;
     return old;
 }
 
@@ -9798,10 +9768,7 @@ Process *schedule(Process *p, int calls)
 
 	if (erts_sched_stat.enabled) {
 	    int prio;
-	    UWord old = ERTS_PROC_SCHED_ID(p,
-					  (ERTS_PROC_LOCK_MAIN
-					   | ERTS_PROC_LOCK_STATUS),
-					  (UWord) esdp->no);
+	    UWord old = ERTS_PROC_SCHED_ID(p, (UWord) esdp->no);
 	    int migrated = old && old != esdp->no;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -10465,7 +10432,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
 	qs->q[PRIORITY_NORMAL] = NULL;
 	qs->q[PRIORITY_LOW] = NULL;
 	qs->q[prio] = st;
-	(void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, qs);
+	(void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, qs);
     }
     else {
 	if (!qs->q[prio]) {
@@ -10612,7 +10579,7 @@ erts_set_gc_state(Process *c_p, int enable)
 
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
 
-    (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+    (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, NULL);
 
     if (dgc_tsk_qs)
 	proc_sys_task_queues_free(dgc_tsk_qs);
@@ -11098,7 +11065,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->mbuf = NULL;
     p->msg_frag = NULL;
     p->mbuf_sz = 0;
-    p->psd = NULL;
+    erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
     p->dictionary = NULL;
     p->seq_trace_lastcnt = 0;
     p->seq_trace_clock = 0;
@@ -11267,7 +11234,7 @@ void erts_init_empty_process(Process *p)
     p->mbuf = NULL;
     p->msg_frag = NULL;
     p->mbuf_sz = 0;
-    p->psd = NULL;
+    erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
     ERTS_P_MONITORS(p) = NULL;
     ERTS_P_LINKS(p) = NULL;         /* List of links */
     p->nodes_monitors = NULL;
@@ -11437,14 +11404,17 @@ static void
 delete_process(Process* p)
 {
     Eterm *heap;
+    ErtsPSD *psd;
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
                            HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
 
     /* Cleanup psd */
 
-    if (p->psd)
-	erts_free(ERTS_ALC_T_PSD, p->psd);
+    psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
+
+    if (psd)
+	erts_free(ERTS_ALC_T_PSD, psd);
 
     /* Clean binaries and funs */
     erts_cleanup_offheap(&p->off_heap);
@@ -12523,9 +12493,9 @@ erts_continue_exit_process(Process *p)
     }
     
     dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
-    scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL);
-    pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL);
-    nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, ERTS_PROC_LOCKS_ALL, NULL);
+    scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
+    pbt = ERTS_PROC_SET_CALL_TIME(p, NULL);
+    nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 #ifdef BM_COUNTERS
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c88bd7056c..c07337b325 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -818,8 +818,8 @@ typedef struct {
 #define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
-#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
-#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ((ErtsProcLocks) 0)
+#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ((ErtsProcLocks) 0)
 
 #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
 #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS
@@ -830,8 +830,8 @@ typedef struct {
 #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
-#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
-#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0)
+#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0)
 
 typedef struct {
     ErtsProcLocks get_locks;
@@ -1026,7 +1026,7 @@ struct process {
     ErlHeapFragment* live_hf_end;
     ErtsMessage *msg_frag;	/* Pointer to message fragment list */
     Uint mbuf_sz;		/* Total size of heap fragments and message fragments */
-    ErtsPSD *psd;		/* Rarely used process specific data */
+    erts_smp_atomic_t psd;		/* Rarely used process specific data */
 
     Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
     Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
@@ -1903,18 +1903,19 @@ do {									\
 #define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
 #endif
 
-void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);
+void *erts_psd_set_init(Process *p, int ix, void *data);
 
 ERTS_GLB_INLINE void *
 erts_psd_get(Process *p, int ix);
 ERTS_GLB_INLINE void *
-erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);
+erts_psd_set(Process *p, int ix, void *new);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void *
 erts_psd_get(Process *p, int ix)
 {
+    ErtsPSD *psd;
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
     if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
@@ -1925,17 +1926,19 @@ erts_psd_get(Process *p, int ix)
 			   || erts_thr_progress_is_blocking());
     }
 #endif
+
+    psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
     ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
-    return p->psd ? p->psd->data[ix] : NULL;
+    if (!psd)
+	return NULL;
+    ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+    return psd->data[ix];
 }
 
-
-/*
- * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
- */
 ERTS_GLB_INLINE void *
-erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
+erts_psd_set(Process *p, int ix, void *data)
 {
+    ErtsPSD *psd;
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
     if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
@@ -1946,50 +1949,56 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 			   || erts_thr_progress_is_blocking());
     }
 #endif
+    psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
     ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
-    if (p->psd) {
-	void *old = p->psd->data[ix];
-	p->psd->data[ix] = data;
+    if (psd) {
+	void *old;
+#ifdef ERTS_SMP
+#ifdef ETHR_ORDERED_READ_DEPEND
+	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+#else
+	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore);
+#endif
+#endif
+	old = psd->data[ix];
+	psd->data[ix] = data;
 	return old;
     }
-    else {
-	if (!data)
-	    return NULL;
-	else
-	    return erts_psd_set_init(p, plocks, ix, data);
-    }
+
+    if (!data)
+	return NULL;
+
+    return erts_psd_set_init(p, ix, data);
 }
 
 #endif
 
-#define ERTS_PROC_SCHED_ID(P, L, ID) \
-  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))
+#define ERTS_PROC_SCHED_ID(P, ID) \
+  ((UWord) erts_psd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID)))
 
 #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
   ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
-#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
-  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))
+#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, SCB) \
+  ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))
 
 #define ERTS_PROC_GET_CALL_TIME(P) \
   ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
-#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
-  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))
+#define ERTS_PROC_SET_CALL_TIME(P, PBT) \
+  ((process_breakpoint_time_t *) erts_psd_set((P), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))
 
 #define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
     ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
-#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
-    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))
+#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, PBT) \
+    ((ErtsProcSysTaskQs *) erts_psd_set((P), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))
 
 #define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
     erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
-#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
-    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
+#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \
+    erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
 
 
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
-ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
-						  ErtsProcLocks plocks,
-						  Eterm handler);
+ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 ERTS_GLB_INLINE Eterm
@@ -2005,13 +2014,13 @@ erts_proc_get_error_handler(Process *p)
 }
 
 ERTS_GLB_INLINE Eterm
-erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
+erts_proc_set_error_handler(Process *p, Eterm handler)
 {
     void *old_val;
     void *new_val;
     ASSERT(is_atom(handler));
     new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
-    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
+    old_val = erts_psd_set(p, ERTS_PSD_ERROR_HANDLER, new_val);
     if (!old_val)
 	return am_error_handler;
     else {
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 39c36ee7a9..73552b28de 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -103,7 +103,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
     size += p->arity * sizeof(p->arg_reg[0]);
   }
 
-  if (p->psd)
+  if (erts_smp_atomic_read_nob(&p->psd) != (erts_aint_t) NULL)
     size += sizeof(ErtsPSD);
 
   scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 797e4cdadc..cc58423938 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -397,7 +397,7 @@ static Port *create_port(char *name,
     prt->common.u.alive.reg = NULL;
     ERTS_PTMR_INIT(prt);
     erts_port_task_handle_init(&prt->timeout_task);
-    prt->psd = NULL;
+    erts_smp_atomic_init_nob(&prt->psd, (erts_aint_t) NULL);
     prt->async_open_port = NULL;
     prt->drv_data = (SWord) 0;
     prt->os_pid = -1;
@@ -3520,6 +3520,7 @@ terminate_port(Port *prt)
     Eterm connected_id = NIL /* Initialize to silence compiler */;
     erts_driver_t *drv;
     erts_aint32_t state;
+    ErtsPrtSD *psd;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -3573,8 +3574,9 @@ terminate_port(Port *prt)
 
     erts_cleanup_port_data(prt);
 
-    if (prt->psd)
-	erts_free(ERTS_ALC_T_PRTSD, prt->psd);
+    psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd);
+    if (psd)
+	erts_free(ERTS_ALC_T_PRTSD, psd);
 
     ASSERT(prt->dist_entry == NULL);
 
-- 
cgit v1.2.3


From 303bacbea8e977743c9e6b02c8d9fabe081de669 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 29 Mar 2016 09:57:33 +0200
Subject: erts: Fix incorrect non-smp debug assert

---
 erts/emulator/beam/erl_alloc_util.c | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 5e7dd7cce8..33b6c83bae 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -858,8 +858,9 @@ void*
 erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
 {
     void* res;
-    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
-                   && !allctr->t && allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+                   allctr->t == 0);
+    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
 
     res = erts_alcu_mseg_alloc(allctr, size_p, flags);
     if (res)
@@ -872,8 +873,9 @@ erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg,
                                Uint old_size, Uint *new_size_p)
 {
     void* res;
-    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
-                   && !allctr->t && allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+                   allctr->t == 0);
+    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
 
     if (seg && old_size)
         clear_literal_range(seg, old_size);
@@ -887,8 +889,9 @@ void
 erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
                                Uint flags)
 {
-    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
-                   && !allctr->t && allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+                   allctr->t == 0);
+    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
 
     erts_alcu_mseg_dealloc(allctr, seg, size, flags);
 
@@ -996,8 +999,9 @@ void*
 erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
 {
     void* res;
-    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
-                   && !allctr->t && allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+                   allctr->t == 0);
+    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
 
     res = erts_alcu_sys_alloc(allctr, size, 1);
     if (res)
@@ -1009,8 +1013,9 @@ void*
 erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
 {
     void* res;
-    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
-                   && !allctr->t && allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+                   allctr->t == 0);
+    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
 
     if (ptr && old_size)
         clear_literal_range(ptr, old_size);
@@ -1023,8 +1028,9 @@ erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint ol
 void
 erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
 {
-    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
-                   && !allctr->t && allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+                   allctr->t == 0);
+    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
 
     erts_alcu_sys_dealloc(allctr, ptr, size, 1);
 
-- 
cgit v1.2.3


From da9b3a0825a3243aecfd849e11d1e37e5ccc96af Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Tue, 8 Dec 2015 17:41:12 +0100
Subject: erts: inline tag_val_def

The tag_val_def function was called and multiple switch statements
had to be traversed in term.c, and then a big switch in the calling
code to branch on the term types. By inlining the switches are
merged by the compiler and a lot fewer branches have to be taken.

Benchmarks show that this increases performance of enc_term by as
much as 10%.
---
 erts/emulator/beam/erl_term.c | 90 -------------------------------------------
 erts/emulator/beam/erl_term.h | 81 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 78 insertions(+), 93 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index d2343912b4..72a1e0b6ec 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -59,96 +59,6 @@ erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz)
 #endif
 }
 
-__decl_noreturn static void __noreturn
-et_abort(const char *expr, const char *file, unsigned line)
-{
-#ifdef EXIT_ON_ET_ABORT
-    static int have_been_called = 0;
-
-    if (have_been_called) {
-	abort();
-    } else {
-	/*
-	 * Prevent infinite loop.
-	 */
-	have_been_called = 1;
-	erts_exit(ERTS_ERROR_EXIT, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
-    }
-#else
-    erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr);
-    abort();
-#endif
-}
-
-#if ET_DEBUG
-#define ET_ASSERT(expr,file,line) \
-do { \
-    if (!(expr)) \
-	et_abort(#expr, file, line); \
-} while(0)
-#else
-#define ET_ASSERT(expr,file,line)	do { } while(0)
-#endif
-
-#if ET_DEBUG
-unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line)
-#else
-unsigned tag_val_def(Wterm x)
-#define file __FILE__
-#define line __LINE__
-#endif
-{
-    static char msg[32];
-
-    switch (x & _TAG_PRIMARY_MASK) {
-    case TAG_PRIMARY_LIST:
-	ET_ASSERT(_list_precond(x),file,line);
-	return LIST_DEF;
-      case TAG_PRIMARY_BOXED: {
-	  Eterm hdr = *boxed_val(x);
-	  ET_ASSERT(is_header(hdr),file,line);
-	  switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
-	    case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):	return TUPLE_DEF;
-	    case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):	return BIG_DEF;
-	    case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):	return BIG_DEF;
-	    case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):	return REF_DEF;
-	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):	return FLOAT_DEF;
-	    case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE):     return EXPORT_DEF;
-	    case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):	return FUN_DEF;
-	    case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PID_DEF;
-	    case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PORT_DEF;
-	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):	return EXTERNAL_REF_DEF;
-	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):	return MAP_DEF;
-	    case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
-	    case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
-	    case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
-	    case (_TAG_HEADER_BIN_MATCHSTATE >> _TAG_PRIMARY_SIZE): return MATCHSTATE_DEF;
-	  }
- 
-	  break;
-      }
-      case TAG_PRIMARY_IMMED1: {
-	  switch ((x & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) {
-	    case (_TAG_IMMED1_PID >> _TAG_PRIMARY_SIZE):	return PID_DEF;
-	    case (_TAG_IMMED1_PORT >> _TAG_PRIMARY_SIZE):	return PORT_DEF;
-	    case (_TAG_IMMED1_IMMED2 >> _TAG_PRIMARY_SIZE): {
-		switch ((x & _TAG_IMMED2_MASK) >> _TAG_IMMED1_SIZE) {
-		  case (_TAG_IMMED2_ATOM >> _TAG_IMMED1_SIZE):	return ATOM_DEF;
-		  case (_TAG_IMMED2_NIL >> _TAG_IMMED1_SIZE):	return NIL_DEF;
-		}
-		break;
-	    }
-	    case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE):	return SMALL_DEF;
-	  }
-	  break;
-      }
-    }
-    erts_snprintf(msg, sizeof(msg), "tag_val_def: %#lx", (unsigned long) x);
-    et_abort(msg, file, line);
-#undef file
-#undef line
-}
-
 /*
  * XXX: define NUMBER_CODE() here when new representation is used
  */
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 2b28762db5..0a71534790 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1129,11 +1129,11 @@ _ET_DECLARE_CHECKED(Uint,loader_y_reg_index,Uint)
 #define FIRST_VACANT_TAG_DEF    0x12
 
 #if ET_DEBUG
-extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
-#define tag_val_def(x)	tag_val_def_debug((x),__FILE__,__LINE__)
+ERTS_GLB_INLINE unsigned tag_val_def(Wterm, const char*, unsigned);
 #else
-extern unsigned tag_val_def(Wterm);
+ERTS_GLB_INLINE unsigned tag_val_def(Wterm);
 #endif
+
 #define not_eq_tags(X,Y)	(tag_val_def((X)) ^ tag_val_def((Y)))
 
 #define NUMBER_CODE(x,y)	((tag_val_def(x) << 5) | tag_val_def(y))
@@ -1152,5 +1152,80 @@ extern unsigned tag_val_def(Wterm);
 
 void erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz);
 
+#if ET_DEBUG
+#define ET_ASSERT(expr,file,line) \
+do { \
+    if (!(expr)) \
+	erl_assert_error("TYPE ASSERTION: " #expr, __FUNCTION__, file, line); \
+} while(0)
+#else
+#define ET_ASSERT(expr,file,line)	do { } while(0)
+#endif
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#if ET_DEBUG
+ERTS_GLB_INLINE unsigned tag_val_def(Wterm x, const char *file, unsigned line)
+#else
+ERTS_GLB_INLINE unsigned tag_val_def(Wterm x)
+#define file __FILE__
+#define line __LINE__
+#endif
+{
+    static char *msg = "tag_val_def error";
+
+    switch (x & _TAG_PRIMARY_MASK) {
+    case TAG_PRIMARY_LIST:
+	ET_ASSERT(_list_precond(x),file,line);
+	return LIST_DEF;
+      case TAG_PRIMARY_BOXED: {
+	  Eterm hdr = *boxed_val(x);
+	  ET_ASSERT(is_header(hdr),file,line);
+	  switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
+	    case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):	return TUPLE_DEF;
+	    case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):	return BIG_DEF;
+	    case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):	return BIG_DEF;
+	    case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):	return REF_DEF;
+	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):	return FLOAT_DEF;
+	    case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE):     return EXPORT_DEF;
+	    case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):	return FUN_DEF;
+	    case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PID_DEF;
+	    case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):	return EXTERNAL_PORT_DEF;
+	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):	return EXTERNAL_REF_DEF;
+	    case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):	return MAP_DEF;
+	    case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
+	    case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
+	    case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE):	return BINARY_DEF;
+	    case (_TAG_HEADER_BIN_MATCHSTATE >> _TAG_PRIMARY_SIZE): return MATCHSTATE_DEF;
+	  }
+
+	  break;
+      }
+      case TAG_PRIMARY_IMMED1: {
+	  switch ((x & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) {
+	    case (_TAG_IMMED1_PID >> _TAG_PRIMARY_SIZE):	return PID_DEF;
+	    case (_TAG_IMMED1_PORT >> _TAG_PRIMARY_SIZE):	return PORT_DEF;
+	    case (_TAG_IMMED1_IMMED2 >> _TAG_PRIMARY_SIZE): {
+		switch ((x & _TAG_IMMED2_MASK) >> _TAG_IMMED1_SIZE) {
+		  case (_TAG_IMMED2_ATOM >> _TAG_IMMED1_SIZE):	return ATOM_DEF;
+		  case (_TAG_IMMED2_NIL >> _TAG_IMMED1_SIZE):	return NIL_DEF;
+		}
+		break;
+	    }
+	    case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE):	return SMALL_DEF;
+	  }
+	  break;
+      }
+    }
+    erl_assert_error(msg, __FUNCTION__, file, line);
+#undef file
+#undef line
+}
+#endif
+
+#if ET_DEBUG
+#define tag_val_def(X) tag_val_def(X, __FILE__, __LINE__)
+#endif
+
 #endif	/* __ERL_TERM_H */
 
-- 
cgit v1.2.3


From 4a736966eba5398a22697db6ca67e1e3dd3cd0f2 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 10 Dec 2015 10:58:33 +0100
Subject: erts: Add enif_cpu/now_time and enif_make_unique_integer

---
 erts/emulator/beam/erl_bif_unique.c    | 20 ++++++++-------
 erts/emulator/beam/erl_bif_unique.h    | 12 ++++++---
 erts/emulator/beam/erl_nif.c           | 45 ++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_nif.h           |  5 ++++
 erts/emulator/beam/erl_nif_api_funcs.h |  6 +++++
 erts/emulator/beam/erl_trace.c         |  5 ++--
 6 files changed, 77 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index c4a39b8897..c9c8561f64 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -266,17 +266,19 @@ static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive)
 }
 
 Uint
-erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES])
+erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES],
+                                  int positive)
 {
     Uint sz;
-    bld_unique_integer_term(NULL, &sz, val[0], val[1], 0);
+    bld_unique_integer_term(NULL, &sz, val[0], val[1], positive);
     return sz;
 }
 
 Eterm
-erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES])
+erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES],
+    int positive)
 {
-    return bld_unique_integer_term(hpp, NULL, val[0], val[1], 0);
+    return bld_unique_integer_term(hpp, NULL, val[0], val[1], positive);
 }
 
 void
@@ -426,16 +428,16 @@ erts_raw_get_unique_monotonic_integer(void)
 }
 
 Uint
-erts_raw_unique_monotonic_integer_heap_size(Sint64 raw)
+erts_raw_unique_monotonic_integer_heap_size(Sint64 raw, int positive)
 {
-    return get_unique_monotonic_integer_heap_size(raw, 0);
+    return get_unique_monotonic_integer_heap_size(raw, positive);
 }
 
 Eterm
-erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw)
+erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw, int positive)
 {
-    Uint hsz = get_unique_monotonic_integer_heap_size(raw, 0);
-    Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, 0);
+    Uint hsz = get_unique_monotonic_integer_heap_size(raw, positive);
+    Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, positive);
     *hpp += hsz;
     return res;
 }
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 37d5d91c39..94fd6163f2 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -41,8 +41,9 @@ void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
  * not necessarily correspond to the end result.
  */
 Sint64 erts_raw_get_unique_monotonic_integer(void);
-Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw);
-Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw);
+Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw, int positive);
+Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw,
+                                                   int positive);
 
 Sint64 erts_get_min_unique_monotonic_integer(void);
 
@@ -53,8 +54,11 @@ Eterm erts_debug_get_unique_monotonic_integer_state(Process *c_p);
 #define ERTS_UNIQUE_INT_RAW_VALUES 2
 #define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2)
 
-Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]);
-Eterm erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]);
+Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES],
+                                       int positive);
+Eterm erts_raw_make_unique_integer(Eterm **hpp,
+                                   Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES],
+                                   int postive);
 void erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]);
 Sint64 erts_get_min_unique_integer(void);
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8580bc2689..703c96777a 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1158,6 +1158,51 @@ int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list
     return 1;
 }
 
+ERL_NIF_TERM
+enif_now_time(ErlNifEnv *env)
+{
+    Uint mega, sec, micro;
+    Eterm *hp;
+    get_now(&mega, &sec, &micro);
+    hp = alloc_heap(env, 4);
+    return TUPLE3(hp, make_small(mega), make_small(sec), make_small(micro));
+}
+
+ERL_NIF_TERM
+enif_cpu_time(ErlNifEnv *env)
+{
+#ifdef HAVE_ERTS_NOW_CPU
+    Uint mega, sec, micro;
+    Eterm *hp;
+    erts_get_now_cpu(&mega, &sec, &micro);
+    hp = alloc_heap(env, 4);
+    return TUPLE3(hp, make_small(mega), make_small(sec), make_small(micro));
+#else
+    return enif_make_badarg(env);
+#endif
+}
+
+ERL_NIF_TERM
+enif_make_unique_integer(ErlNifEnv *env, ErlNifUniqueInteger properties)
+{
+    int monotonic = properties & ERL_NIF_UNIQUE_MONOTONIC;
+    int positive = properties & ERL_NIF_UNIQUE_POSITIVE;
+    Eterm *hp;
+    Uint hsz;
+
+    if (monotonic) {
+        Sint64 raw_unique = erts_raw_get_unique_monotonic_integer();
+        hsz = erts_raw_unique_monotonic_integer_heap_size(raw_unique, positive);
+        hp = alloc_heap(env, hsz);
+        return erts_raw_make_unique_monotonic_integer_value(&hp, raw_unique, positive);
+    } else {
+        Uint64 raw_unique[ERTS_UNIQUE_INT_RAW_VALUES];
+        erts_raw_get_unique_integer(raw_unique);
+        hsz = erts_raw_unique_integer_heap_size(raw_unique, positive);
+        hp = alloc_heap(env, hsz);
+        return erts_raw_make_unique_integer(&hp, raw_unique, positive);
+    }
+}
 
 ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); }
 void enif_mutex_destroy(ErlNifMutex *mtx) {  erl_drv_mutex_destroy(mtx); }
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index c9c8b1d0af..959fcb5412 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -197,6 +197,11 @@ typedef enum {
     ERL_NIF_MAP_ITERATOR_TAIL = ERL_NIF_MAP_ITERATOR_LAST
 } ErlNifMapIteratorEntry;
 
+typedef enum {
+    ERL_NIF_UNIQUE_POSITIVE = (1 << 0),
+    ERL_NIF_UNIQUE_MONOTONIC = (1 << 1)
+} ErlNifUniqueInteger;
+
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
 typedef struct {
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 1448a508a2..3fe60e3df8 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -163,6 +163,9 @@ ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* val
 ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_monotonic_time, (ErlNifTimeUnit));
 ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_time_offset, (ErlNifTimeUnit));
 ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTimeUnit, ErlNifTimeUnit));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_now_time, (ErlNifEnv *env));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_cpu_time, (ErlNifEnv *env));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_unique_integer, (ErlNifEnv *env, ErlNifUniqueInteger properties));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -318,6 +321,9 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_monotonic_time ERL_NIF_API_FUNC_MACRO(enif_monotonic_time)
 #  define enif_time_offset ERL_NIF_API_FUNC_MACRO(enif_time_offset)
 #  define enif_convert_time_unit ERL_NIF_API_FUNC_MACRO(enif_convert_time_unit)
+#  define enif_now_time ERL_NIF_API_FUNC_MACRO(enif_now_time)
+#  define enif_cpu_time ERL_NIF_API_FUNC_MACRO(enif_cpu_time)
+#  define enif_make_unique_integer ERL_NIF_API_FUNC_MACRO(enif_make_unique_integer)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 41876b8c62..9033c1b75c 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -176,7 +176,7 @@ take_timestamp(ErtsTraceTimeStamp *tsp, int ts_type)
 	    hsz += 3; /* 2-tuple */
 	    raw_unique = erts_raw_get_unique_monotonic_integer();
 	    tsp->u.monotonic.raw_unique = raw_unique;
-	    hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique);
+	    hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique, 0);
 	}
 	return hsz;
     }
@@ -216,8 +216,7 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp)
 	    return emtime;
 
 	raw = tsp->u.monotonic.raw_unique;
-	unique = erts_raw_make_unique_monotonic_integer_value(hpp,
-							      raw);
+	unique = erts_raw_make_unique_monotonic_integer_value(hpp, raw, 0);
 	res = TUPLE2(*hpp, emtime, unique);
 	*hpp += 3;
 	return res;
-- 
cgit v1.2.3


From 348f3f2ee2d2707e30658c3600e05309ad0e72bf Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 10 Dec 2015 10:59:34 +0100
Subject: erts: Add enif_is_process/port_alive

---
 erts/emulator/beam/erl_nif.c           | 70 +++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_nif.h           |  7 +++-
 erts/emulator/beam/erl_nif_api_funcs.h |  6 +++
 3 files changed, 81 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 703c96777a..8440fd26b5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -404,12 +404,28 @@ static int is_offheap(const ErlOffHeap* oh)
 
 ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)
 {
+    if (caller_env->proc->common.id == ERTS_INVALID_PID)
+        return NULL;
     pid->pid = caller_env->proc->common.id;
     return pid;
 }
+
 int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid)
 {
-    return is_internal_pid(term) ? (pid->pid=term, 1) : 0;
+    if (is_internal_pid(term)) {
+        pid->pid=term;
+        return 1;
+    }
+    return 0;
+}
+
+int enif_get_local_port(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPort* port)
+{
+    if (is_internal_port(term)) {
+        port->port_id=term;
+        return 1;
+    }
+    return 0;
 }
 
 int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term)
@@ -1158,6 +1174,58 @@ int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list
     return 1;
 }
 
+int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc)
+{
+    ErtsProcLocks rp_locks = 0; /* We don't need any locks,
+                                   just to check if it is alive */
+    Eterm target = proc->pid;
+    Process* rp;
+    Process* c_p;
+    int scheduler = erts_get_scheduler_id() != 0;
+
+    if (env != NULL) {
+	c_p = env->proc;
+	if (target == c_p->common.id) {
+            /* We are alive! */
+	    return 1;
+	}
+    }
+    else {
+#ifdef ERTS_SMP
+	c_p = NULL;
+#else
+	erts_exit(ERTS_ABORT_EXIT,"enif_is_process_alive: "
+                  "env==NULL on non-SMP VM");
+#endif
+    }
+
+    rp = (scheduler
+	  ? erts_proc_lookup(target)
+	  : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+			      target, rp_locks, ERTS_P2P_FLG_INC_REFC));
+    if (rp == NULL) {
+        ASSERT(env == NULL || target != c_p->common.id);
+	return 0;
+    } else {
+        if (!scheduler)
+            erts_proc_dec_refc(rp);
+        return 1;
+    }
+}
+
+int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port)
+{
+    /* only allowed if called from scheduler */
+    if (erts_get_scheduler_id() == 0)
+        erts_exit(ERTS_ABORT_EXIT,"enif_is_port_alive: called from non-scheduler");
+
+    return erts_port_lookup(
+        port->port_id,
+        (erts_port_synchronous_ops
+         ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+         : ERTS_PORT_SFLGS_INVALID_LOOKUP)) != NULL;
+}
+
 ERL_NIF_TERM
 enif_now_time(ErlNifEnv *env)
 {
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 959fcb5412..6ee22a693c 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -150,7 +150,12 @@ typedef enum
 typedef struct
 {
     ERL_NIF_TERM pid;  /* internal, may change */
-}ErlNifPid;
+} ErlNifPid;
+
+typedef struct
+{
+    ERL_NIF_TERM port_id;  /* internal, may change */
+}ErlNifPort;
 
 typedef ErlDrvSysInfo ErlNifSysInfo;
 
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 3fe60e3df8..21864c8b35 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -166,6 +166,9 @@ ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTim
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_now_time, (ErlNifEnv *env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_cpu_time, (ErlNifEnv *env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_unique_integer, (ErlNifEnv *env, ErlNifUniqueInteger properties));
+ERL_NIF_API_FUNC_DECL(int, enif_is_process_alive, (ErlNifEnv *env, ErlNifPid *pid));
+ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port_id));
+ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -324,6 +327,9 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_now_time ERL_NIF_API_FUNC_MACRO(enif_now_time)
 #  define enif_cpu_time ERL_NIF_API_FUNC_MACRO(enif_cpu_time)
 #  define enif_make_unique_integer ERL_NIF_API_FUNC_MACRO(enif_make_unique_integer)
+#  define enif_is_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_process_alive)
+#  define enif_is_port_alive ERL_NIF_API_FUNC_MACRO(enif_is_port_alive)
+#  define enif_get_local_port ERL_NIF_API_FUNC_MACRO(enif_get_local_port)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From 1bd56e2b5141a3afdca4e854e9b667807bf4e2f3 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 17 Dec 2015 14:14:54 +0100
Subject: erts: Add enif_term_to_binary and enif_binary_to_term

---
 erts/emulator/beam/erl_nif.c           | 70 ++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_nif_api_funcs.h |  4 ++
 2 files changed, 74 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8440fd26b5..21bf752bb6 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -638,6 +638,76 @@ unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size,
     return binary_bytes(*termp);
 }
 
+int enif_term_to_binary(ErlNifEnv *dst_env, ERL_NIF_TERM term,
+                        ErlNifBinary *bin)
+{
+    Sint size;
+    byte *bp;
+    Binary* refbin;
+
+    size = erts_encode_ext_size(term);
+    if (!enif_alloc_binary(size, bin))
+        return 0;
+
+    refbin = bin->ref_bin;
+
+    bp = bin->data;
+
+    erts_encode_ext(term, &bp);
+
+    bin->size = bp - bin->data;
+    refbin->orig_size = bin->size;
+
+    ASSERT(bin->data + bin->size == bp);
+
+    return 1;
+}
+
+int enif_binary_to_term(ErlNifEnv *dst_env, ErlNifBinary *bin,
+                        ERL_NIF_TERM *term)
+{
+    Sint size;
+    ErtsHeapFactory factory;
+
+    if ((size = erts_decode_ext_size(bin->data, bin->size)) < 0)
+        return 0;
+
+    if (size > 0) {
+        byte *bp;
+
+        if (is_internal_pid(dst_env->proc->common.id)) {
+            flush_env(dst_env);
+            erts_factory_proc_prealloc_init(&factory, dst_env->proc, size);
+            cache_env(dst_env);
+        } else {
+
+            /* this is guaranteed to create a heap fragment */
+            if (!alloc_heap(dst_env, size))
+                return 0;
+
+            erts_factory_heap_frag_init(&factory, dst_env->heap_frag);
+        }
+
+        bp = bin->data;
+        *term = erts_decode_ext(&factory, &bp);
+
+        if (is_non_value(*term)) {
+            return 0;
+        }
+
+        erts_factory_close(&factory);
+    }
+    else {
+        erts_factory_dummy_init(&factory);
+        *term = erts_decode_ext(&factory, &bin->data);
+        if (is_non_value(*term)) {
+            return 0;
+        }
+        ASSERT(is_immed(*term));
+    }
+    return 1;
+}
+
 int enif_is_identical(Eterm lhs, Eterm rhs)
 {
     return EQ(lhs,rhs);
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 21864c8b35..0333e95331 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -169,6 +169,8 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_unique_integer, (ErlNifEnv *env, E
 ERL_NIF_API_FUNC_DECL(int, enif_is_process_alive, (ErlNifEnv *env, ErlNifPid *pid));
 ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port_id));
 ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id));
+ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin));
+ERL_NIF_API_FUNC_DECL(int, enif_binary_to_term, (ErlNifEnv *env, ErlNifBinary *bin, ERL_NIF_TERM *term));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -330,6 +332,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_is_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_process_alive)
 #  define enif_is_port_alive ERL_NIF_API_FUNC_MACRO(enif_is_port_alive)
 #  define enif_get_local_port ERL_NIF_API_FUNC_MACRO(enif_get_local_port)
+#  define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary)
+#  define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From 209c5cf22b5cdc70eb48e6afdcddfa7132471aab Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Mon, 1 Feb 2016 11:01:40 +0100
Subject: erts: Add enif_port_command

---
 erts/emulator/beam/erl_nif.c           |  23 ++++
 erts/emulator/beam/erl_nif_api_funcs.h |   2 +
 erts/emulator/beam/erl_port.h          |   2 +
 erts/emulator/beam/erl_port_task.c     |  30 ++++-
 erts/emulator/beam/erl_port_task.h     |   2 +
 erts/emulator/beam/io.c                | 223 +++++++++++++++++++++++++++++++--
 6 files changed, 266 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 21bf752bb6..6ed89780b4 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -375,6 +375,29 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     return 1;
 }
 
+int
+enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port,
+                  ErlNifEnv *msg_env, ERL_NIF_TERM msg)
+{
+
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int scheduler = esdp ? esdp->no : 0;
+    Port *prt;
+
+    if (scheduler == 0 || !env)
+        return 0;
+
+    prt = erts_port_lookup(to_port->port_id,
+                           (erts_port_synchronous_ops
+                            ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+                            : ERTS_PORT_SFLGS_INVALID_LOOKUP));
+
+    if (!prt)
+        return 0;
+
+    return erts_port_output_async(prt, env->proc->common.id, msg);
+}
+
 ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
 {
     Uint sz;
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 0333e95331..32afb53bfc 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -171,6 +171,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port
 ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id));
 ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin));
 ERL_NIF_API_FUNC_DECL(int, enif_binary_to_term, (ErlNifEnv *env, ErlNifBinary *bin, ERL_NIF_TERM *term));
+ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -334,6 +335,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_get_local_port ERL_NIF_API_FUNC_MACRO(enif_get_local_port)
 #  define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary)
 #  define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term)
+#  define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index fa97707a87..dbc6ead216 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -949,6 +949,8 @@ ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm
 ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
 ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);
 
+int erts_port_output_async(Port *, Eterm, Eterm);
+
 /*
  * Signals from ports to ports. Used by sys drivers.
  */
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index e85395e062..b200344af5 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -180,10 +180,9 @@ p2p_sig_data_to_task(ErtsProc2PortSigData *sigdp)
     return ptp;
 }
 
-ErtsProc2PortSigData *
-erts_port_task_alloc_p2p_sig_data(void)
+static ERTS_INLINE ErtsProc2PortSigData *
+p2p_sig_data_init(ErtsPortTask *ptp)
 {
-    ErtsPortTask *ptp = port_task_alloc();
 
     ptp->type = ERTS_PORT_TASK_PROC_SIG;
     ptp->u.alive.flags = ERTS_PT_FLG_SIG_DEP;
@@ -194,6 +193,31 @@ erts_port_task_alloc_p2p_sig_data(void)
     return &ptp->u.alive.td.psig.data;
 }
 
+ErtsProc2PortSigData *
+erts_port_task_alloc_p2p_sig_data(void)
+{
+    ErtsPortTask *ptp = port_task_alloc();
+
+    return p2p_sig_data_init(ptp);
+}
+
+ErtsProc2PortSigData *
+erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr)
+{
+    ErtsPortTask *ptp = erts_alloc(ERTS_ALC_T_PORT_TASK,
+                                   sizeof(ErtsPortTask) + extra);
+
+    *extra_ptr = ptp+1;
+
+    return p2p_sig_data_init(ptp);
+}
+
+void
+erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp)
+{
+    schedule_port_task_free(p2p_sig_data_to_task(sigdp));
+}
+
 static ERTS_INLINE Eterm
 task_caller(ErtsPortTask *ptp)
 {
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 335f7a77d5..56cef4e352 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -269,6 +269,8 @@ int erts_port_task_schedule(Eterm,
 void erts_port_task_free_port(Port *);
 int erts_port_is_scheduled(Port *);
 ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void);
+ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr);
+void erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp);
 
 #ifdef ERTS_SMP
 void erts_enqueue_port(ErtsRunQueue *rq, Port *pp);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 797e4cdadc..093334aab7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1749,7 +1749,6 @@ cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp)
 	    driver_free_binary(ev->binv[i]);
     if (cbinp)
 	driver_free_binary(cbinp);
-    erts_free(ERTS_ALC_T_DRV_CMD_DATA, ev);
 }
 
 static int
@@ -1887,6 +1886,188 @@ port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si
     return ERTS_PORT_REDS_CMD_OUTPUT;
 }
 
+
+/*
+ * This erts_port_output will always create a port task.
+ * The call is treated as a port_command call, i.e. no
+ * badsig i generated if the input in invalid. However
+ * an error_logger message is generated.
+ */
+int
+erts_port_output_async(Port *prt, Eterm from, Eterm list)
+{
+
+    ErtsPortOpResult res;
+    ErtsProc2PortSigData *sigdp;
+    erts_driver_t *drv = prt->drv_ptr;
+    size_t size;
+    int task_flags;
+    ErtsProc2PortSigCallback port_sig_callback;
+    ErlDrvBinary *cbin = NULL;
+    ErlIOVec *evp = NULL;
+    char *buf = NULL;
+    ErtsPortTaskHandle *ns_pthp;
+
+    if (drv->outputv) {
+        ErlIOVec ev;
+	SysIOVec* ivp;
+	ErlDrvBinary**  bvp;
+        int vsize;
+	Uint csize;
+	Uint pvsize;
+	Uint pcsize;
+        size_t iov_offset, binv_offset, alloc_size;
+        Uint blimit = 0;
+        char *ptr;
+        int i;
+
+        Eterm* bptr = NULL;
+        Uint offset;
+
+        if (is_binary(list)) {
+            /* We optimize for when we get a procbin without offset */
+            Eterm real_bin;
+            int bitoffs;
+            int bitsize;
+            ERTS_GET_REAL_BIN(list, real_bin, offset, bitoffs, bitsize);
+            bptr = binary_val(real_bin);
+            if (*bptr == HEADER_PROC_BIN && bitoffs == 0) {
+                size = binary_size(list);
+                vsize = 1;
+            } else
+                bptr = NULL;
+        }
+
+        if (!bptr) {
+            if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size))
+                goto bad_value;
+
+            /* To pack or not to pack (small binaries) ...? */
+            if (vsize >= SMALL_WRITE_VEC) {
+                /* Do pack */
+                vsize = pvsize + 1;
+                csize = pcsize;
+                blimit = ERL_SMALL_IO_BIN_LIMIT;
+            }
+            cbin = driver_alloc_binary(csize);
+            if (!cbin)
+                erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize));
+        }
+
+
+        iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec));
+	binv_offset = iov_offset;
+        binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec));
+        alloc_size = binv_offset;
+	alloc_size += (vsize+1)*sizeof(ErlDrvBinary *);
+
+        sigdp = erts_port_task_alloc_p2p_sig_data_extra(alloc_size, (void**)&ptr);
+
+        evp = (ErlIOVec *) ptr;
+        ivp = evp->iov = (SysIOVec *) (ptr + iov_offset);
+        bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset);
+
+        ivp[0].iov_base = NULL;
+	ivp[0].iov_len = 0;
+	bvp[0] = NULL;
+
+        if (bptr) {
+            ProcBin* pb = (ProcBin *) bptr;
+
+            ivp[1].iov_base = pb->bytes+offset;
+            ivp[1].iov_len = size;
+            bvp[1] = Binary2ErlDrvBinary(pb->val);
+
+            evp->vsize = 1;
+        } else {
+
+            evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit);
+            if (evp->vsize < 0) {
+                if (evp != &ev)
+                    erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp);
+                driver_free_binary(cbin);
+                goto bad_value;
+            }
+        }
+#if 0
+	/* This assertion may say something useful, but it can
+        be falsified during the emulator test suites. */
+	ASSERT(evp->vsize == vsize);
+#endif
+	evp->vsize++;
+	evp->size = size;  /* total size */
+
+        /* Need to increase refc on all binaries */
+        for (i = 1; i < evp->vsize; i++)
+            if (bvp[i])
+                driver_binary_inc_refc(bvp[i]);
+
+	sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV;
+	sigdp->u.outputv.from = from;
+	sigdp->u.outputv.evp = evp;
+	sigdp->u.outputv.cbinp = cbin;
+	port_sig_callback = port_sig_outputv;
+    } else {
+        ErlDrvSizeT ERTS_DECLARE_DUMMY(r);
+
+	/*
+	 * Apperently there exist code that write 1 byte to
+	 * much in buffer. Where it resides I don't know, but
+	 * we can live with one byte extra allocated...
+	 */
+
+        if (erts_iolist_size(list, &size))
+            goto bad_value;
+
+        buf = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, size + 1);
+
+        r = erts_iolist_to_buf(list, buf, size);
+        ASSERT(ERTS_IOLIST_TO_BUF_SUCCEEDED(r));
+
+	sigdp = erts_port_task_alloc_p2p_sig_data();
+	sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUT;
+	sigdp->u.output.from = from;
+	sigdp->u.output.bufp = buf;
+	sigdp->u.output.size = size;
+	port_sig_callback = port_sig_output;
+    }
+    sigdp->flags = 0;
+    ns_pthp = NULL;
+    task_flags = 0;
+
+    res = erts_schedule_proc2port_signal(NULL,
+					 prt,
+					 ERTS_INVALID_PID,
+					 NULL,
+					 sigdp,
+					 task_flags,
+					 ns_pthp,
+					 port_sig_callback);
+
+    if (res != ERTS_PORT_OP_SCHEDULED) {
+	if (drv->outputv)
+	    cleanup_scheduled_outputv(evp, cbin);
+	else
+	    cleanup_scheduled_output(buf);
+	return 1;
+    }
+    return 1;
+
+bad_value:
+
+    /*
+     * We call badsig directly here as this function is called with
+     * the main lock of the calling process still held.
+     * At the moment this operation is always not a bang_op, so
+     * only an error_logger message should be generated, no badsig.
+    */
+
+    badsig_received(0, prt, erts_atomic32_read_nob(&prt->state), 1);
+
+    return 0;
+
+}
+
 ErtsPortOpResult
 erts_port_output(Process *c_p,
 		 int flags,
@@ -1896,7 +2077,7 @@ erts_port_output(Process *c_p,
 		 Eterm *refp)
 {
     ErtsPortOpResult res;
-    ErtsProc2PortSigData *sigdp;
+    ErtsProc2PortSigData *sigdp = NULL;
     erts_driver_t *drv = prt->drv_ptr;
     size_t size;
     int try_call;
@@ -1978,10 +2159,13 @@ erts_port_output(Process *c_p,
 	    evp = &ev;
 	}
 	else {
-	    char *ptr = erts_alloc((try_call
-				    ? ERTS_ALC_T_TMP
-				    : ERTS_ALC_T_DRV_CMD_DATA), alloc_size);
-
+            char *ptr;
+            if (try_call) {
+                ptr = erts_alloc(ERTS_ALC_T_TMP, alloc_size);
+            } else {
+                sigdp = erts_port_task_alloc_p2p_sig_data_extra(
+                    alloc_size, (void**)&ptr);
+            }
 	    evp = (ErlIOVec *) ptr;
 	    ivp = evp->iov = (SysIOVec *) (ptr + iov_offset);
 	    bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset);
@@ -2010,9 +2194,12 @@ erts_port_output(Process *c_p,
 	bvp[0] = NULL;
 	evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit);
 	if (evp->vsize < 0) {
-	    if (evp != &ev)
-		erts_free(try_call ? ERTS_ALC_T_TMP : ERTS_ALC_T_DRV_CMD_DATA,
-			  evp);
+            if (evp != &ev) {
+                if (try_call)
+                    erts_free(ERTS_ALC_T_TMP, evp);
+                else
+                    erts_port_task_free_p2p_sig_data(sigdp);
+            }
 	    driver_free_binary(cbin);
 	    goto bad_value;
 	}
@@ -2064,8 +2251,10 @@ erts_port_output(Process *c_p,
 		/* Fall through... */
 	    case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
 		driver_free_binary(cbin);
-		if (evp != &ev)
+		if (evp != &ev) {
+                    ASSERT(!sigdp);
 		    erts_free(ERTS_ALC_T_TMP, evp);
+                }
 		if (try_call_res != ERTS_TRY_IMM_DRV_CALL_OK)
 		    return ERTS_PORT_OP_DROPPED;
 		if (c_p)
@@ -2076,8 +2265,10 @@ erts_port_output(Process *c_p,
 		if (async_nosuspend
 		    && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) {
 		    driver_free_binary(cbin);
-		    if (evp != &ev)
+                    if (evp != &ev) {
+                        ASSERT(!sigdp);
 			erts_free(ERTS_ALC_T_TMP, evp);
+                    }
 		    return ((sched_flags & ERTS_PTS_FLG_EXIT)
 			    ? ERTS_PORT_OP_DROPPED
 			    : ERTS_PORT_OP_BUSY);
@@ -2092,9 +2283,16 @@ erts_port_output(Process *c_p,
 		if (bvp[i])
 		    driver_binary_inc_refc(bvp[i]);
 
-	    new_evp = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, alloc_size);
+            /* The port task and iovec is allocated in the
+               same structure as an optimization. This
+               is especially important in erts_port_output_async
+               of when !try_call */
+            ASSERT(sigdp == NULL);
+            sigdp = erts_port_task_alloc_p2p_sig_data_extra(
+                alloc_size, (void**)&new_evp);
 
 	    if (evp != &ev) {
+                /* Copy from TMP alloc to port task */
 		sys_memcpy((void *) new_evp, (void *) evp, alloc_size);
 		new_evp->iov = (SysIOVec *) (((char *) new_evp)
 					     + iov_offset);
@@ -2142,7 +2340,6 @@ erts_port_output(Process *c_p,
 	    evp = new_evp;
 	}
 
-	sigdp = erts_port_task_alloc_p2p_sig_data();
 	sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV;
 	sigdp->u.outputv.from = from;
 	sigdp->u.outputv.evp = evp;
-- 
cgit v1.2.3


From a2a86dadc648dda68b5221a7c1d83b9238be1e25 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 12 Feb 2016 18:52:20 +0100
Subject: erts: Improve enif_binary_to_term

* Accept a raw data buffer instead of ErlNifBinary
* Accept option ERL_NIF_BIN2TERM_SAFE
* Return number of read bytes
---
 erts/emulator/beam/beam_load.c         |  8 +++----
 erts/emulator/beam/erl_nif.c           | 41 +++++++++++++++++-----------------
 erts/emulator/beam/erl_nif.h           |  4 ++++
 erts/emulator/beam/erl_nif_api_funcs.h |  2 +-
 erts/emulator/beam/external.c          | 12 ++++++++--
 erts/emulator/beam/external.h          |  2 +-
 erts/emulator/beam/io.c                |  4 ++--
 7 files changed, 43 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index f115df935f..d3d278fb81 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1548,7 +1548,7 @@ read_literal_table(LoaderState* stp)
             erts_factory_heap_frag_init(&factory,
 					new_literal_fragment(heap_size));
 	    factory.alloc_type = ERTS_ALC_T_PREPARED_CODE;
-            val = erts_decode_ext(&factory, &p);
+            val = erts_decode_ext(&factory, &p, 0);
 
             if (is_non_value(val)) {
                 LoadError1(stp, "literal %d: bad external format", i);
@@ -1559,7 +1559,7 @@ read_literal_table(LoaderState* stp)
         }
         else {
             erts_factory_dummy_init(&factory);
-            val = erts_decode_ext(&factory, &p);
+            val = erts_decode_ext(&factory, &p, 0);
             if (is_non_value(val)) {
                 LoadError1(stp, "literal %d: bad external format", i);
             }
@@ -5719,7 +5719,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
     if (ext != NULL) {
 	ErtsHeapFactory factory;
 	erts_factory_proc_prealloc_init(&factory, p, code_hdr->attr_size_on_heap);
-	result = erts_decode_ext(&factory, &ext);
+	result = erts_decode_ext(&factory, &ext, 0);
 	if (is_value(result)) {
 	    erts_factory_close(&factory);
 	}
@@ -5742,7 +5742,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
     if (ext != NULL) {
 	ErtsHeapFactory factory;
 	erts_factory_proc_prealloc_init(&factory, p, code_hdr->compile_size_on_heap);
-	result = erts_decode_ext(&factory, &ext);
+	result = erts_decode_ext(&factory, &ext, 0);
 	if (is_value(result)) {
 	    erts_factory_close(&factory);
 	}
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 6ed89780b4..d8f2272c6d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -686,17 +686,25 @@ int enif_term_to_binary(ErlNifEnv *dst_env, ERL_NIF_TERM term,
     return 1;
 }
 
-int enif_binary_to_term(ErlNifEnv *dst_env, ErlNifBinary *bin,
-                        ERL_NIF_TERM *term)
+size_t enif_binary_to_term(ErlNifEnv *dst_env,
+                           const unsigned char* data,
+                           size_t data_sz,
+                           ERL_NIF_TERM *term,
+                           ErlNifBinaryToTerm opts)
 {
     Sint size;
     ErtsHeapFactory factory;
+    byte *bp = (byte*) data;
 
-    if ((size = erts_decode_ext_size(bin->data, bin->size)) < 0)
+    ERTS_CT_ASSERT(ERL_NIF_BIN2TERM_SAFE == ERTS_DIST_EXT_BTT_SAFE);
+
+    if (opts & ~ERL_NIF_BIN2TERM_SAFE) {
+        return 0;
+    }
+    if ((size = erts_decode_ext_size(bp, data_sz)) < 0)
         return 0;
 
     if (size > 0) {
-        byte *bp;
 
         if (is_internal_pid(dst_env->proc->common.id)) {
             flush_env(dst_env);
@@ -710,25 +718,18 @@ int enif_binary_to_term(ErlNifEnv *dst_env, ErlNifBinary *bin,
 
             erts_factory_heap_frag_init(&factory, dst_env->heap_frag);
         }
+    } else {
+        erts_factory_dummy_init(&factory);
+    }
 
-        bp = bin->data;
-        *term = erts_decode_ext(&factory, &bp);
-
-        if (is_non_value(*term)) {
-            return 0;
-        }
+    *term = erts_decode_ext(&factory, &bp, (Uint32)opts);
 
-        erts_factory_close(&factory);
-    }
-    else {
-        erts_factory_dummy_init(&factory);
-        *term = erts_decode_ext(&factory, &bin->data);
-        if (is_non_value(*term)) {
-            return 0;
-        }
-        ASSERT(is_immed(*term));
+    if (is_non_value(*term)) {
+        return 0;
     }
-    return 1;
+    erts_factory_close(&factory);
+    ASSERT(bp > data);
+    return bp - data;
 }
 
 int enif_is_identical(Eterm lhs, Eterm rhs)
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 6ee22a693c..bc1f59bc90 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -207,6 +207,10 @@ typedef enum {
     ERL_NIF_UNIQUE_MONOTONIC = (1 << 1)
 } ErlNifUniqueInteger;
 
+typedef enum {
+    ERL_NIF_BIN2TERM_SAFE = 0x20000000
+} ErlNifBinaryToTerm;
+
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
 typedef struct {
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 32afb53bfc..35058afe7c 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -170,7 +170,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_is_process_alive, (ErlNifEnv *env, ErlNifPid *pi
 ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port_id));
 ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id));
 ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin));
-ERL_NIF_API_FUNC_DECL(int, enif_binary_to_term, (ErlNifEnv *env, ErlNifBinary *bin, ERL_NIF_TERM *term));
+ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsigned char* data, size_t sz, ERL_NIF_TERM *term, unsigned int opts));
 ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg));
 
 /*
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index aac1490f0c..f5eb13421d 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -967,15 +967,23 @@ erts_decode_dist_ext(ErtsHeapFactory* factory,
     return THE_NON_VALUE;
 }
 
-Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext)
+Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags)
 {
+    ErtsDistExternal ede, *edep;
     Eterm obj;
     byte *ep = *ext;
     if (*ep++ != VERSION_MAGIC) {
         erts_factory_undo(factory);
 	return THE_NON_VALUE;
     }
-    ep = dec_term(NULL, factory, ep, &obj, NULL);
+    if (flags) {
+        ASSERT(flags == ERTS_DIST_EXT_BTT_SAFE);
+        ede.flags = flags; /* a dummy struct just for the flags */
+        edep = &ede;
+    } else {
+        edep = NULL;
+    }
+    ep = dec_term(edep, factory, ep, &obj, NULL);
     if (!ep) {
 #ifdef DEBUG
 	bin_write(ERTS_PRINT_STDERR,NULL,*ext,500);
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index d12051c6b4..87eff2fe9f 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -191,7 +191,7 @@ Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *);
 
 Sint erts_decode_ext_size(byte*, Uint);
 Sint erts_decode_ext_size_ets(byte*, Uint);
-Eterm erts_decode_ext(ErtsHeapFactory*, byte**);
+Eterm erts_decode_ext(ErtsHeapFactory*, byte**, Uint32 flags);
 Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*);
 
 Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 093334aab7..c7b21c0386 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4570,7 +4570,7 @@ port_sig_call(Port *prt,
 
                 (void) erts_factory_message_create(&factory, rp, &rp_locks, hsz);
 		endp = (byte *) resp_bufp;
-		msg = erts_decode_ext(&factory, &endp);
+		msg = erts_decode_ext(&factory, &endp, 0);
 		if (is_value(msg)) {
                     hp = erts_produce_heap(&factory,
                                            3,
@@ -4689,7 +4689,7 @@ erts_port_call(Process* c_p,
 	    hsz += 3;
             erts_factory_proc_prealloc_init(&factory, c_p, hsz);
 	    endp = (byte *) resp_bufp;
-	    term = erts_decode_ext(&factory, &endp);
+	    term = erts_decode_ext(&factory, &endp, 0);
 	    if (term == THE_NON_VALUE)
 		return ERTS_PORT_OP_BADARG;
             hp = erts_produce_heap(&factory,3,0);
-- 
cgit v1.2.3


From 043ab9055917a4f6d89f1fa2788079e0618928c4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 15 Feb 2016 19:23:12 +0100
Subject: erts: Remove printout when dec_term fails in DEBUG

---
 erts/emulator/beam/external.c | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index f5eb13421d..10f03636ec 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -985,9 +985,6 @@ Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags)
     }
     ep = dec_term(edep, factory, ep, &obj, NULL);
     if (!ep) {
-#ifdef DEBUG
-	bin_write(ERTS_PRINT_STDERR,NULL,*ext,500);
-#endif
 	return THE_NON_VALUE;
     }
     *ext = ep;
-- 
cgit v1.2.3


From 1c630ab8793425aed1e487d25c71b8bbd9325e8b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 15 Feb 2016 19:28:52 +0100
Subject: erts: Fix bug in enif_term_to_binary

Wait until after dec_term and factory_close to do cache_env(),
otherwise we will cache the wrong state.
---
 erts/emulator/beam/erl_nif.c | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d8f2272c6d..c6ece8c1c7 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -705,19 +705,8 @@ size_t enif_binary_to_term(ErlNifEnv *dst_env,
         return 0;
 
     if (size > 0) {
-
-        if (is_internal_pid(dst_env->proc->common.id)) {
-            flush_env(dst_env);
-            erts_factory_proc_prealloc_init(&factory, dst_env->proc, size);
-            cache_env(dst_env);
-        } else {
-
-            /* this is guaranteed to create a heap fragment */
-            if (!alloc_heap(dst_env, size))
-                return 0;
-
-            erts_factory_heap_frag_init(&factory, dst_env->heap_frag);
-        }
+        flush_env(dst_env);
+        erts_factory_proc_prealloc_init(&factory, dst_env->proc, size);
     } else {
         erts_factory_dummy_init(&factory);
     }
@@ -728,6 +717,8 @@ size_t enif_binary_to_term(ErlNifEnv *dst_env,
         return 0;
     }
     erts_factory_close(&factory);
+    cache_env(dst_env);
+
     ASSERT(bp > data);
     return bp - data;
 }
-- 
cgit v1.2.3


From ed81bb9ed8114d2059783e2c2fdae526d3a36e1e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 15 Feb 2016 19:26:54 +0100
Subject: erts: Fix bug in enif_send

Let cache_env() set env->heap_frag to same as MBUF(p)
as it is in any other case.
---
 erts/emulator/beam/erl_nif.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c6ece8c1c7..beef6983cd 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -209,6 +209,7 @@ static void flush_env(ErlNifEnv* env)
 */
 static void cache_env(ErlNifEnv* env)
 {
+    env->heap_frag = MBUF(env->proc);
     if (env->heap_frag == NULL) {
 	ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
 	ASSERT(env->hp <= HEAP_TOP(env->proc));
@@ -216,10 +217,6 @@ static void cache_env(ErlNifEnv* env)
 	env->hp = HEAP_TOP(env->proc);
     }
     else {
-	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);       
-	env->heap_frag = MBUF(env->proc);
-	ASSERT(env->heap_frag != NULL);
 	env->hp = env->heap_frag->mem + env->heap_frag->used_size;
 	env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
     }
-- 
cgit v1.2.3


From f0a1185b95bc8d5c20954cbf5f70767f1f354cc8 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 30 Mar 2016 18:25:20 +0200
Subject: Fix scheduling of system tasks

---
 erts/emulator/beam/erl_process.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1fb8502ebe..4530317710 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6684,10 +6684,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st)
 	erts_aint32_t e;
 	n = e = a;
 
-	if (a & ERTS_PSFLG_FREE) {
-	    res = 0;
+	if (a & ERTS_PSFLG_FREE)
 	    goto cleanup; /* We don't want to schedule free processes... */
-	}
 
 	enqueue = ERTS_ENQUEUE_NOT;
 	n |= ERTS_PSFLG_ACTIVE_SYS;
-- 
cgit v1.2.3


From 7d06e5ecd47ef071eb8c4e58ead70e79fa4b02b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 31 Mar 2016 06:55:31 +0200
Subject: Fix unsafe transformation of apply/3 with fixed arguments

62473daf introduced an unsafe optimization in the loader.
See the comments in the test case for an explanation of
the problem.
---
 erts/emulator/beam/beam_load.c | 7 +++++++
 erts/emulator/beam/ops.tab     | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d3d278fb81..16cbdbffea 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2734,6 +2734,13 @@ same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label)
 	Target.val == Label.val;
 }
 
+static int
+is_killed_apply(LoaderState* stp, GenOpArg Reg, GenOpArg Live)
+{
+    return Reg.type == TAG_x && Live.type == TAG_u &&
+	Live.val+2 <= Reg.val;
+}
+
 static int
 is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live)
 {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 9e53b4bfcc..772460c177 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -303,7 +303,7 @@ move_window5 x x x x x y
 # Swap registers.
 move R1=x Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp
 
-swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed(Tmp, Live) => \
+swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed_apply(Tmp, Live) => \
   swap R1 R2 | line Loc | apply Live
 
 swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \
-- 
cgit v1.2.3


From d97dc2c64be5530b22c8e4f9e11d246285ea8a17 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 31 Mar 2016 16:06:10 +0200
Subject: Fix bad refc management of process struct

---
 erts/emulator/beam/erl_process.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b4b97d7df1..2b468a9ad9 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9293,17 +9293,18 @@ Process *schedule(Process *p, int calls)
 				      | ERTS_PSFLG_PENDING_EXIT
 				      | ERTS_PSFLG_ACTIVE_SYS))
 			    == ERTS_PSFLG_SUSPENDED)) {
-			if (state & ERTS_PSFLG_FREE) {
+			if (proxy_p) {
+			    free_proxy_proc(proxy_p);
+			    proxy_p = NULL;
+			}
+			else if (state & ERTS_PSFLG_FREE) {
+			    /* free and not queued by proxy */
 #ifdef ERTS_SMP
 			    erts_smp_proc_dec_refc(p);
 #else
 			    erts_free_proc(p);
 #endif
 			}
-			if (proxy_p) {
-			    free_proxy_proc(proxy_p);
-			    proxy_p = NULL;
-			}
 			goto pick_next_process;
 		    }
 		    state = new;
-- 
cgit v1.2.3


From f86eabba45b351890ea8a12bd976c02b799c0e93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 1 Apr 2016 16:16:46 +0200
Subject: erts: Don't search for non-existing Map keys twice

* For maps:get/2,3 and maps:find/2, searching for an immediate key, e.g. an atom,
  the search was performed twice if the key did not exist in the map.
---
 erts/emulator/beam/erl_map.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4b6931f848..6b747256e1 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -196,13 +196,13 @@ erts_maps_get(Eterm key, Eterm map)
 		    return &vs[i];
 		}
 	    }
-	}
-
-	for (i = 0; i < n; i++) {
-	    if (EQ(ks[i], key)) {
-		return &vs[i];
-	    }
-	}
+	} else {
+            for (i = 0; i < n; i++) {
+                if (EQ(ks[i], key)) {
+                    return &vs[i];
+                }
+            }
+        }
 	return NULL;
     }
     ASSERT(is_hashmap(map));
-- 
cgit v1.2.3


From c048886458ce68280ba32647a93a04902c929988 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 1 Apr 2016 20:11:34 +0200
Subject: erts: Fix race for process_flag(trap_exit,true)

and a concurrent exit signal.

We now actually guarantee that the process will not die
from exit signal *after* the call to process_flag(trap_exit,true)
has returned.

The race is narrow and probably quite hard to observe even if you
manage to provoke it. Has only been confirmed with the help of
return trace and a sleep in send_exit_signal().

Solution:
Seize status lock to prevent send_exit_signal() from reading
an old status (without TRAP_EXIT) and then writing PENDING_EXIT
after TRAP_EXIT has been set by process_flag_2().
---
 erts/emulator/beam/bif.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index b43137597e..11c694233f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1566,14 +1566,17 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 	*       true. For more info, see implementation of
 	*       erts_send_exit_signal().
 	*/
+       erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
        if (trap_exit)
 	   state = erts_smp_atomic32_read_bor_mb(&BIF_P->state,
 						 ERTS_PSFLG_TRAP_EXIT);
        else
 	   state = erts_smp_atomic32_read_band_mb(&BIF_P->state,
 						  ~ERTS_PSFLG_TRAP_EXIT);
+       erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
+
 #ifdef ERTS_SMP
-       if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
+       if (state & ERTS_PSFLG_PENDING_EXIT) {
 	   erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
 	   ERTS_BIF_EXITED(BIF_P);
        }
-- 
cgit v1.2.3


From d86548e01746aa14074b4df4c157f6d06f3be697 Mon Sep 17 00:00:00 2001
From: John Eckersberg <jeckersb@redhat.com>
Date: Wed, 16 Dec 2015 11:03:42 -0500
Subject: Add patch to crash dump on large distribution

https://bugzilla.redhat.com/show_bug.cgi?id=1291822
https://bugzilla.redhat.com/show_bug.cgi?id=1291855
https://bugzilla.redhat.com/show_bug.cgi?id=1291856
https://bugzilla.redhat.com/show_bug.cgi?id=1291857
---
 erts/emulator/beam/dist.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index fa385f105d..3cfd164224 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1963,7 +1963,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (size > (Uint) INT_MAX)
-	erts_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_DUMP_EXIT,
 		 "Absurdly large distribution output data buffer "
 		 "(%beu bytes) passed.\n",
 		 size);
@@ -2003,7 +2003,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (size > (Uint) INT_MAX)
-	erts_exit(ERTS_ABORT_EXIT,
+	erts_exit(ERTS_DUMP_EXIT,
 		 "Absurdly large distribution output data buffer "
 		 "(%beu bytes) passed.\n",
 		 size);
-- 
cgit v1.2.3


From 4b507534f27c343fe2b53f07bbe52e94c81e381f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 7 Dec 2015 17:10:37 +0100
Subject: ops.tab: Remove useless transformation

The transformation on the following line will do the job.
---
 erts/emulator/beam/ops.tab | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 772460c177..78000160e3 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1539,7 +1539,6 @@ gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \
 # GCing arithmetic instructions.
 #
 
-gen_plus Fail Live Y=y X=x Dst => i_plus Fail Live X Y Dst
 gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst
 
 gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst
-- 
cgit v1.2.3


From ec26a0c56cb6e28cc5a35ef72116275e5eeef823 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 1 Apr 2016 14:03:51 +0200
Subject: Refactor calls to transform_engine()

We used to set last_op_next and last_op to NULL just in case.
Setting last_op_next to causes a rescan of the instructions
to find the last instruction in the chain, so we would want
to avoid that unless really necessary.
---
 erts/emulator/beam/beam_load.c | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 16cbdbffea..2f2b433999 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2028,11 +2028,7 @@ load_code(LoaderState* stp)
 	ASSERT(arity == last_op->arity);
 
     do_transform:
-	if (stp->genop == NULL) {
-	    last_op_next = NULL;
-	    goto get_next_instr;
-	}
-
+	ASSERT(stp->genop != NULL);
 	if (gen_opc[stp->genop->op].transform != -1) {
 	    int need;
 	    tmp_op = stp->genop;
@@ -2045,25 +2041,34 @@ load_code(LoaderState* stp)
 	    }
 	    switch (transform_engine(stp)) {
 	    case TE_FAIL:
-		last_op_next = NULL;
-		last_op = NULL;
+		/*
+		 * No transformation found. stp->genop != NULL and
+		 * last_op_next is still valid. Go ahead and load
+		 * the instruction.
+		 */
 		break;
 	    case TE_OK:
+		/*
+		 * Some transformation was applied. last_op_next is
+		 * no longer valid and stp->genop may be NULL.
+		 * Try to transform again.
+		 */
+		if (stp->genop == NULL) {
+		    last_op_next = &stp->genop;
+		    goto get_next_instr;
+		}
 		last_op_next = NULL;
-		last_op = NULL;
 		goto do_transform;
 	    case TE_SHORT_WINDOW:
-		last_op_next = NULL;
-		last_op = NULL;
+		/*
+		 * No transformation applied. stp->genop != NULL and
+		 * last_op_next is still valid. Fetch a new instruction
+		 * before trying the transformation again.
+		 */
 		goto get_next_instr;
 	    }
 	}
 
-	if (stp->genop == NULL) {
-	    last_op_next = NULL;
-	    goto get_next_instr;
-	}
-
 	/*
 	 * From the collected generic instruction, find the specific
 	 * instruction.
@@ -2584,7 +2589,10 @@ load_code(LoaderState* stp)
 	{
 	    GenOp* next = stp->genop->next;
 	    FREE_GENOP(stp, stp->genop);
-	    stp->genop = next;
+	    if ((stp->genop = next) == NULL) {
+		last_op_next = &stp->genop;
+		goto get_next_instr;
+	    }
 	    goto do_transform;
 	}
     }
-- 
cgit v1.2.3


From 336489820c3d3ea89813d9cfd1e0cb667d3a7707 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 11 Aug 2015 18:00:21 +0200
Subject: erts: Teach lttng to configure and build system

Introduce a wrapper API for lttng.
---
 erts/emulator/beam/erl_bif_info.c  |  3 ++
 erts/emulator/beam/erlang_lttng.c  | 32 +++++++++++++++
 erts/emulator/beam/erlang_lttng.h  | 36 +++++++++++++++++
 erts/emulator/beam/lttng-wrapper.h | 83 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 154 insertions(+)
 create mode 100644 erts/emulator/beam/erlang_lttng.c
 create mode 100644 erts/emulator/beam/erlang_lttng.h
 create mode 100644 erts/emulator/beam/lttng-wrapper.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 8c748c9bf7..5c03e33b74 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -126,6 +126,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #ifdef ERTS_FRMPTR
 				     " [frame-pointer]"
 #endif
+#ifdef USE_LTTNG
+				     " [lttng]"
+#endif
 #ifdef USE_DTRACE
 				     " [dtrace]"
 #endif
diff --git a/erts/emulator/beam/erlang_lttng.c b/erts/emulator/beam/erlang_lttng.c
new file mode 100644
index 0000000000..fce40eedc1
--- /dev/null
+++ b/erts/emulator/beam/erlang_lttng.c
@@ -0,0 +1,32 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#ifdef USE_LTTNG
+#define TRACEPOINT_CREATE_PROBES
+/*
+ * The header containing our TRACEPOINT_EVENTs.
+ */
+#define TRACEPOINT_DEFINE
+#include "erlang_lttng.h"
+#endif /* USE_LTTNG */
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
new file mode 100644
index 0000000000..48956cf242
--- /dev/null
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -0,0 +1,36 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef USE_LTTNG
+#undef TRACEPOINT_PROVIDER
+#define TRACEPOINT_PROVIDER com_ericsson_otp
+
+#undef TRACEPOINT_INCLUDE
+#define TRACEPOINT_INCLUDE "erlang_lttng.h"
+
+#if !defined(__ERLANG_LTTNG_H__) || defined(TRACEPOINT_HEADER_MULTI_READ)
+#define __ERLANG_LTTNG_H__
+
+#include <lttng/tracepoint.h>
+
+
+#endif /* __ERLANG_LTTNG_H__ */
+#include <lttng/tracepoint-event.h>
+#endif /* USE_LTTNG */
diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h
new file mode 100644
index 0000000000..8362dc555e
--- /dev/null
+++ b/erts/emulator/beam/lttng-wrapper.h
@@ -0,0 +1,83 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef __LTTNG_WRAPPER_H__
+#define __LTTNG_WRAPPER_H__
+
+#ifdef USE_LTTNG
+
+#include "erlang_lttng.h"
+#define USE_LTTNG_VM_TRACEPOINTS
+
+#define LTTNG_BUFFER_SZ      (256)
+#define LTTNG_PROC_BUFFER_SZ (16)
+#define LTTNG_PORT_BUFFER_SZ (20)
+
+#define lttng_decl_procbuf(Name) \
+    char Name[LTTNG_PROC_BUFFER_SZ]
+
+#define lttng_decl_portbuf(Name) \
+    char Name[LTTNG_PORT_BUFFER_SZ]
+
+#define lttng_pid_to_str(pid, name) \
+    erts_snprintf(name, LTTNG_PROC_BUFFER_SZ, "%T", (pid))
+
+#define lttng_portid_to_str(pid, name) \
+    erts_snprintf(name, LTTNG_PORT_BUFFER_SZ, "%T", (pid))
+
+#define lttng_proc_to_str(p, name) \
+    lttng_pid_to_str(((p) ? (p)->common.id : ERTS_INVALID_PID), name)
+
+#define lttng_port_to_str(p, name) \
+    lttng_portid_to_str(((p) ? (p)->common.id : ERTS_INVALID_PORT), name)
+
+/* ErtsRunQueue->ErtsSchedulerData->Uint */
+#define lttng_rq_to_id(RQ) \
+    (RQ)->scheduler->no
+
+#define LTTNG_ENABLED(Name) \
+    tracepoint_enabled(com_ericsson_otp, Name)
+
+/* include a special LTTNG_DO for do_tracepoint ? */
+#define LTTNG1(Name, Arg1) \
+    tracepoint(com_ericsson_otp, Name, (Arg1))
+
+#define LTTNG2(Name, Arg1, Arg2) \
+    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2))
+
+#define LTTNG3(Name, Arg1, Arg2, Arg3) \
+    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3))
+
+#define LTTNG4(Name, Arg1, Arg2, Arg3, Arg4) \
+    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4))
+
+#define LTTNG5(Name, Arg1, Arg2, Arg3, Arg4, Arg5) \
+    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4), (Arg5))
+
+#else /* USE_LTTNG */
+
+#define LTTNG1(Name, Arg1) do {} while(0)
+#define LTTNG2(Name, Arg1, Arg2) do {} while(0)
+#define LTTNG3(Name, Arg1, Arg2, Arg3) do {} while(0)
+#define LTTNG4(Name, Arg1, Arg2, Arg3, Arg4) do {} while(0)
+#define LTTNG5(Name, Arg1, Arg2, Arg3, Arg4, Arg5) do {} while(0)
+
+#endif /* USE_LTTNG */
+#endif /* __LTTNG_WRAPPER_H__ */
-- 
cgit v1.2.3


From 955cd62970391b40123f2b9dee3bb3380233c17e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 28 Jan 2016 11:45:34 +0100
Subject: erts: Update lttng-wrapper with mfa conversion

---
 erts/emulator/beam/lttng-wrapper.h | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h
index 8362dc555e..294872c365 100644
--- a/erts/emulator/beam/lttng-wrapper.h
+++ b/erts/emulator/beam/lttng-wrapper.h
@@ -29,6 +29,7 @@
 #define LTTNG_BUFFER_SZ      (256)
 #define LTTNG_PROC_BUFFER_SZ (16)
 #define LTTNG_PORT_BUFFER_SZ (20)
+#define LTTNG_MFA_BUFFER_SZ  (256)
 
 #define lttng_decl_procbuf(Name) \
     char Name[LTTNG_PROC_BUFFER_SZ]
@@ -36,6 +37,12 @@
 #define lttng_decl_portbuf(Name) \
     char Name[LTTNG_PORT_BUFFER_SZ]
 
+#define lttng_decl_mfabuf(Name) \
+    char Name[LTTNG_MFA_BUFFER_SZ]
+
+#define lttng_decl_carrier_stats(Name) \
+    lttng_carrier_stats_t Name##_STATSTRUCT, *Name = &Name##_STATSTRUCT
+
 #define lttng_pid_to_str(pid, name) \
     erts_snprintf(name, LTTNG_PROC_BUFFER_SZ, "%T", (pid))
 
@@ -48,6 +55,23 @@
 #define lttng_port_to_str(p, name) \
     lttng_portid_to_str(((p) ? (p)->common.id : ERTS_INVALID_PORT), name)
 
+#define lttng_mfa_to_str(m,f,a, Name) \
+    erts_snprintf(Name, LTTNG_MFA_BUFFER_SZ, "%T:%T/%lu", (Eterm)(m), (Eterm)(f), (Uint)(a))
+
+#define lttng_proc_to_mfa_str(p, Name)                              \
+    do {                                                            \
+        if (ERTS_PROC_IS_EXITING((p))) {                            \
+            strcpy(Name, "<exiting>");                              \
+        } else {                                                    \
+            BeamInstr *_fptr = find_function_from_pc((p)->i);       \
+            if (_fptr) {                                            \
+                lttng_mfa_to_str(_fptr[0],_fptr[1],_fptr[2], Name); \
+            } else {                                                \
+                strcpy(Name, "<unknown>");                          \
+            }                                                       \
+        }                                                           \
+    } while(0)
+
 /* ErtsRunQueue->ErtsSchedulerData->Uint */
 #define lttng_rq_to_id(RQ) \
     (RQ)->scheduler->no
-- 
cgit v1.2.3


From ce5d152cd863b68dcc2b5c7a566d1e5e1a5c5dab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 14 Aug 2015 18:20:51 +0200
Subject: erts: Add lttng tracepoints for memory carriers

* carrier_create
* carrier_destroy
* carrier_pool_put
* carrier_pool_get
---
 erts/emulator/beam/erl_alloc.h      |  8 ++-
 erts/emulator/beam/erl_alloc_util.c | 33 +++++++++++++
 erts/emulator/beam/erl_alloc_util.h | 13 +++++
 erts/emulator/beam/erlang_lttng.h   | 99 +++++++++++++++++++++++++++++++++++++
 4 files changed, 148 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 71e4713624..ee2013bd93 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -235,9 +235,9 @@ void *erts_alloc(ErtsAlcType_t type, Uint size)
     void *res;
     ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
     res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
-	ERTS_ALC_T2N(type),
-	erts_allctrs[ERTS_ALC_T2A(type)].extra,
-	size);
+            ERTS_ALC_T2N(type),
+            erts_allctrs[ERTS_ALC_T2A(type)].extra,
+            size);
     if (!res)
 	erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
     ERTS_MSACC_POP_STATE_X();
@@ -564,5 +564,3 @@ NAME##_free(TYPE *p)							\
 #undef ERTS_ALC_ATTRIBUTES
 
 #endif /* #ifndef ERL_ALLOC_H__ */
-
-
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 5e7dd7cce8..6e682019ba 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -52,6 +52,7 @@
 #ifdef ERTS_ENABLE_LOCK_COUNT
 #include "erl_lock_count.h"
 #endif
+#include "lttng-wrapper.h"
 
 #if defined(ERTS_ALLOC_UTIL_HARD_DEBUG) && defined(__GNUC__)
 #warning "* * * * * * * * * *"
@@ -3125,6 +3126,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
 
     erts_smp_atomic_set_wb(&crr->allctr,
 			   ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
+    LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr));
 }
 
 static void
@@ -3240,6 +3242,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
     first_old_traitor = allctr->cpool.traitor_list.next;
     cpool_entrance = NULL;
 
+    LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size);
     /*
      * Search my own pooled_list,
      * i.e my abandoned carriers that were in the pool last time I checked.
@@ -3925,6 +3928,21 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 
     }
 
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+    if (LTTNG_ENABLED(carrier_create)) {
+        lttng_decl_carrier_stats(mbc_stats);
+        lttng_decl_carrier_stats(sbc_stats);
+        LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats);
+        LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats);
+        LTTNG5(carrier_create,
+                ERTS_ALC_A2AD(allctr->alloc_no),
+                allctr->ix,
+                crr_sz,
+                mbc_stats,
+                sbc_stats);
+    }
+#endif
+
     DEBUG_SAVE_ALIGNMENT(crr);
     return blk;
 }
@@ -4148,6 +4166,21 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
                 allctr->remove_mbc(allctr, crr);
 	}
 
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+        if (LTTNG_ENABLED(carrier_destroy)) {
+            lttng_decl_carrier_stats(mbc_stats);
+            lttng_decl_carrier_stats(sbc_stats);
+            LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats);
+            LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats);
+            LTTNG5(carrier_destroy,
+                ERTS_ALC_A2AD(allctr->alloc_no),
+                allctr->ix,
+                crr_sz,
+                mbc_stats,
+                sbc_stats);
+        }
+#endif
+
 #ifdef ERTS_SMP
 	schedule_dealloc_carrier(allctr, crr);
 #else
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index b7d717ed23..afdff1a71e 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -30,6 +30,7 @@
 #endif
 
 #include "erl_mseg.h"
+#include "lttng-wrapper.h"
 
 #define ERTS_AU_PREF_ALLOC_BITS 11
 #define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS)
@@ -417,6 +418,18 @@ typedef struct {
     } blocks;
 } CarriersStats_t;
 
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+#define LTTNG_CARRIER_STATS_TO_LTTNG_STATS(CSP, LSP)            \
+    do {                                                        \
+        (LSP)->carriers.size = (CSP)->curr.norm.mseg.size       \
+                             + (CSP)->curr.norm.sys_alloc.size; \
+        (LSP)->carriers.no   = (CSP)->curr.norm.mseg.no         \
+                             + (CSP)->curr.norm.sys_alloc.no;   \
+        (LSP)->blocks.size   = (CSP)->blocks.curr.size;         \
+        (LSP)->blocks.no     = (CSP)->blocks.curr.no;           \
+    } while (0)
+#endif
+
 #ifdef ERTS_SMP
 
 typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index 48956cf242..2fab6fb698 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -31,6 +31,105 @@
 #include <lttng/tracepoint.h>
 
 
+#ifndef LTTNG_CARRIER_STATS
+#define LTTNG_CARRIER_STATS
+typedef struct {
+    unsigned long no;
+    unsigned long size;
+} lttng_stat_values_t;
+
+typedef struct {
+    lttng_stat_values_t carriers;
+    lttng_stat_values_t blocks;
+} lttng_carrier_stats_t;
+#endif
+
+
+/* Memory Allocator */
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    carrier_create,
+    TP_ARGS(
+        const char*, type,
+        int, instance,
+        unsigned long, size,
+        lttng_carrier_stats_t *, mbcs,
+        lttng_carrier_stats_t *, sbcs
+    ),
+    TP_FIELDS(
+        ctf_string(type, type)
+        ctf_integer(int, instance, instance)
+        ctf_integer(unsigned long, size, size)
+        ctf_float(float, mbc_ratio, (float)mbcs->blocks.size/(float)mbcs->carriers.size)
+        ctf_integer(unsigned long, mbc_carriers, mbcs->carriers.no)
+        ctf_integer(unsigned long, mbc_carriers_size, mbcs->carriers.size)
+        ctf_integer(unsigned long, mbc_blocks, mbcs->blocks.no)
+        ctf_integer(unsigned long, mbc_blocks_size, mbcs->blocks.size)
+        ctf_integer(unsigned long, sbc_carriers, sbcs->carriers.no)
+        ctf_integer(unsigned long, sbc_carriers_size, sbcs->carriers.size)
+        ctf_integer(unsigned long, sbc_blocks, sbcs->blocks.no)
+        ctf_integer(unsigned long, sbc_blocks_size, sbcs->blocks.size)
+    )
+)
+
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    carrier_destroy,
+    TP_ARGS(
+        const char*, type,
+        int, instance,
+        unsigned long, size,
+        lttng_carrier_stats_t *, mbcs,
+        lttng_carrier_stats_t *, sbcs
+    ),
+    TP_FIELDS(
+        ctf_string(type, type)
+        ctf_integer(int, instance, instance)
+        ctf_integer(unsigned long, size, size)
+        ctf_float(float, mbc_ratio, (float)mbcs->blocks.size/(float)mbcs->carriers.size)
+        ctf_integer(unsigned long, mbc_carriers, mbcs->carriers.no)
+        ctf_integer(unsigned long, mbc_carriers_size, mbcs->carriers.size)
+        ctf_integer(unsigned long, mbc_blocks, mbcs->blocks.no)
+        ctf_integer(unsigned long, mbc_blocks_size, mbcs->blocks.size)
+        ctf_integer(unsigned long, sbc_carriers, sbcs->carriers.no)
+        ctf_integer(unsigned long, sbc_carriers_size, sbcs->carriers.size)
+        ctf_integer(unsigned long, sbc_blocks, sbcs->blocks.no)
+        ctf_integer(unsigned long, sbc_blocks_size, sbcs->blocks.size)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    carrier_pool_put,
+    TP_ARGS(
+        const char*, name,
+        int, instance,
+        unsigned long, size
+    ),
+    TP_FIELDS(
+        ctf_string(type, name)
+        ctf_integer(int, instance, instance)
+        ctf_integer(unsigned long, size, size)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    carrier_pool_get,
+    TP_ARGS(
+        const char*, name,
+        int, instance,
+        unsigned long, size
+    ),
+    TP_FIELDS(
+        ctf_string(type, name)
+        ctf_integer(int, instance, instance)
+        ctf_integer(unsigned long, size, size)
+    )
+)
+
 #endif /* __ERLANG_LTTNG_H__ */
 #include <lttng/tracepoint-event.h>
 #endif /* USE_LTTNG */
-- 
cgit v1.2.3


From 73344dc2a5451c6f2e4b1ea4f69de17aa358e88f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 17 Aug 2015 15:25:26 +0200
Subject: erts: Add lttng tracepoints for scheduler events

* scheduler_poll
---
 erts/emulator/beam/erl_process.c  |  8 +++++++-
 erts/emulator/beam/erlang_lttng.h | 14 ++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a706ebf595..46b45b1d3e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -43,6 +43,7 @@
 #include "erl_thr_queue.h"
 #include "erl_async.h"
 #include "dtrace-wrapper.h"
+#include "lttng-wrapper.h"
 #include "erl_ptab.h"
 #include "erl_bif_unique.h"
 #define ERTS_WANT_TIMER_WHEEL_API
@@ -3238,6 +3239,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
 
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
+            LTTNG2(scheduler_poll, esdp->no, 1);
 	    erl_sys_schedule(1); /* Might give us something to do */
 
 	    ERTS_MSACC_POP_STATE_M();
@@ -3361,6 +3363,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	ASSERT(!erts_port_task_have_outstanding_io_tasks());
 
 	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+        LTTNG2(scheduler_poll, esdp->no, 0);
 
 	erl_sys_schedule(0);
 
@@ -9581,7 +9584,10 @@ Process *schedule(Process *p, int calls)
 	    erts_sys_schedule_interrupt(0);
 #endif
 	    erts_smp_runq_unlock(rq);
-	    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+
+            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+            LTTNG2(scheduler_poll, esdp->no, 1);
+
 	    erl_sys_schedule(1);
 	    ERTS_MSACC_POP_STATE_M();
 
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index 2fab6fb698..5b5dc5548e 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -30,6 +30,20 @@
 
 #include <lttng/tracepoint.h>
 
+/* Schedulers */
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    scheduler_poll,
+    TP_ARGS(
+        int, id,
+        int, runnable
+    ),
+    TP_FIELDS(
+        ctf_integer(int, scheduler, id)
+        ctf_integer(int, runnable, runnable)
+    )
+)
 
 #ifndef LTTNG_CARRIER_STATS
 #define LTTNG_CARRIER_STATS
-- 
cgit v1.2.3


From d31e4e33ac73c0c3239a3bf8a2919165bbed9e88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 22 Jan 2016 11:52:13 +0100
Subject: erts: Add lttng tracepoints for drivers

* driver_event
* driver_flush
* driver_finish
* driver_init
* driver_output
* driver_outputv
* driver_process_exit
* driver_ready_async
* driver_ready_input
* driver_ready_output
* driver_start
* driver_stop
* driver_stop_select
* driver_timeout
---
 erts/emulator/beam/erl_bif_ddll.c  |   2 +
 erts/emulator/beam/erl_port_task.c |  17 +++
 erts/emulator/beam/erlang_lttng.h  | 249 +++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/io.c            |  92 +++++++++++++-
 4 files changed, 359 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index f006856b6a..b526eda41d 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -48,6 +48,7 @@
 #include "erl_version.h"
 #include "erl_bif_unique.h"
 #include "dtrace-wrapper.h"
+#include "lttng-wrapper.h"
 
 #ifdef ERTS_SMP
 #define DDLL_SMP 1
@@ -1619,6 +1620,7 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name)
 	    if (q->finish) {
 		int fpe_was_unmasked = erts_block_fpe();
 		DTRACE1(driver_finish, q->name);
+                LTTNG1(driver_finish, q->name);
 		(*(q->finish))();
 		erts_unblock_fpe(fpe_was_unmasked);
 	    }
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index e85395e062..377e47adde 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -35,6 +35,7 @@
 #include "dist.h"
 #include "erl_check_io.h"
 #include "dtrace-wrapper.h"
+#include "lttng-wrapper.h"
 #include <stdarg.h>
 
 /*
@@ -69,6 +70,18 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q
 #else
 #define  DTRACE_DRIVER(PROBE_NAME, PP) do {} while(0)
 #endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+#define LTTNG_DRIVER(TRACEPOINT, PP)                              \
+    if (LTTNG_ENABLED(TRACEPOINT)) {                              \
+        lttng_decl_portbuf(port_str);                             \
+        lttng_decl_procbuf(proc_str);                             \
+        lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(PP), proc_str);  \
+        lttng_port_to_str((PP), port_str);                        \
+        LTTNG3(TRACEPOINT, proc_str, port_str, (PP)->name);       \
+    }
+#else
+#define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0)
+#endif
 
 #define ERTS_SMP_LC_VERIFY_RQ(RQ, PP)					\
     do {								\
@@ -1728,6 +1741,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 		reds = ERTS_PORT_REDS_TIMEOUT;
 		if (!(state & ERTS_PORT_SFLGS_DEAD)) {
 		    DTRACE_DRIVER(driver_timeout, pp);
+		    LTTNG_DRIVER(driver_timeout, pp);
 		    (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data);
 		}
 	    }
@@ -1736,6 +1750,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    reds = ERTS_PORT_REDS_INPUT;
 	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
             DTRACE_DRIVER(driver_ready_input, pp);
+            LTTNG_DRIVER(driver_ready_input, pp);
 	    /* NOTE some windows drivers use ->ready_input
 	       for input and output */
 	    (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data,
@@ -1747,6 +1762,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    reds = ERTS_PORT_REDS_OUTPUT;
 	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
             DTRACE_DRIVER(driver_ready_output, pp);
+            LTTNG_DRIVER(driver_ready_output, pp);
 	    (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data,
 					 ptp->u.alive.td.io.event);
 	    reset_executed_io_task_handle(ptp);
@@ -1756,6 +1772,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    reds = ERTS_PORT_REDS_EVENT;
 	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
             DTRACE_DRIVER(driver_event, pp);
+            LTTNG_DRIVER(driver_event, pp);
 	    (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data,
 				  ptp->u.alive.td.io.event,
 				  ptp->u.alive.td.io.event_data);
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index 5b5dc5548e..b5cd90d93e 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -59,6 +59,255 @@ typedef struct {
 #endif
 
 
+/* Port and Driver Scheduling */
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_start,
+    TP_ARGS(
+        char*, pid,
+        char*, driver,
+        char*, port
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(driver, driver)
+        ctf_string(port, port)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_init,
+    TP_ARGS(
+        char*, driver,
+        int, major,
+        int, minor,
+        int, flags
+    ),
+    TP_FIELDS(
+        ctf_string(driver, driver)
+        ctf_integer(int, major, major)
+        ctf_integer(int, minor, minor)
+        ctf_integer(int, flags, flags)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_outputv,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver,
+        size_t, bytes
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+        ctf_integer(size_t, bytes, bytes)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_output,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver,
+        size_t, bytes
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+        ctf_integer(size_t, bytes, bytes)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_ready_input,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_ready_output,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_event,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_timeout,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_stop_select,
+    TP_ARGS(
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_flush,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_stop,
+    TP_ARGS(
+        char*, pid,
+        char*, driver,
+        char*, port
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(driver, driver)
+        ctf_string(port, port)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_process_exit,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_ready_async,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_finish,
+    TP_ARGS(
+        char*, driver
+    ),
+    TP_FIELDS(
+        ctf_string(driver, driver)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_call,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver,
+        unsigned int, command,
+        size_t, bytes
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+        ctf_integer(unsigned int, command, command)
+        ctf_integer(size_t, bytes, bytes)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    driver_control,
+    TP_ARGS(
+        char*, pid,
+        char*, port,
+        char*, driver,
+        unsigned int, command,
+        size_t, bytes
+    ),
+    TP_FIELDS(
+        ctf_string(pid, pid)
+        ctf_string(port, port)
+        ctf_string(driver, driver)
+        ctf_integer(unsigned int, command, command)
+        ctf_integer(size_t, bytes, bytes)
+    )
+)
+
+
 /* Memory Allocator */
 
 TRACEPOINT_EVENT(
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 797e4cdadc..b213be23db 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -47,6 +47,7 @@
 #define ERTS_WANT_EXTERNAL_TAGS
 #include "external.h"
 #include "dtrace-wrapper.h"
+#include "lttng-wrapper.h"
 #include "erl_map.h"
 #include "erl_bif_unique.h"
 #include "erl_hl_timer.h"
@@ -717,7 +718,19 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
             DTRACE3(driver_start, process_str, driver->name, port_str);
         }
 #endif
+
 	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
+
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+        if (LTTNG_ENABLED(driver_start)) {
+            lttng_decl_portbuf(port_str);
+            lttng_decl_procbuf(proc_str);
+            lttng_pid_to_str(pid, proc_str);
+            lttng_port_to_str(port, port_str);
+            LTTNG3(driver_start, proc_str, driver->name, port_str);
+        }
+#endif
+
 	fpe_was_unmasked = erts_block_fpe();
 	drv_data = (*driver->start)(ERTS_Port2ErlDrvPort(port), name, opts);
 	if (((SWord) drv_data) == -1)
@@ -1724,6 +1737,15 @@ call_driver_outputv(int bang_op,
 	    DTRACE4(driver_outputv, process_str, port_str, prt->name, size);
 	}
 #endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+        if (LTTNG_ENABLED(driver_outputv)) {
+            lttng_decl_portbuf(port_str);
+            lttng_decl_procbuf(proc_str);
+            lttng_pid_to_str(caller, proc_str);
+            lttng_port_to_str(prt, port_str);
+            LTTNG4(driver_outputv, proc_str, port_str, prt->name, size);
+        }
+#endif
 
 	prt->caller = caller;
 	(*drv->outputv)((ErlDrvData) prt->drv_data, evp);
@@ -1826,6 +1848,15 @@ call_driver_output(int bang_op,
 	    DTRACE4(driver_output, process_str, port_str, prt->name, size);
 	}
 #endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+        if (LTTNG_ENABLED(driver_output)) {
+            lttng_decl_portbuf(port_str);
+            lttng_decl_procbuf(proc_str);
+            lttng_pid_to_str(caller, proc_str);
+            lttng_port_to_str(prt, port_str);
+            LTTNG4(driver_output, proc_str, port_str, prt->name, size);
+        }
+#endif
 
 	prt->caller = caller;
 	(*drv->output)((ErlDrvData) prt->drv_data, bufp, size);
@@ -1949,7 +1980,6 @@ erts_port_output(Process *c_p,
 	DTRACE4(port_command, process_str, port_str, prt->name, "command");
     }
 #endif
-
     if (drv->outputv) {
 	ErlIOVec ev;
 	SysIOVec iv[SMALL_WRITE_VEC];
@@ -3489,6 +3519,17 @@ static void flush_port(Port *p)
             DTRACE3(driver_flush, process_str, port_str, p->name);
         }
 #endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+        if (LTTNG_ENABLED(driver_flush)) {
+            lttng_decl_portbuf(port_str);
+            lttng_decl_procbuf(proc_str);
+            lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(p), proc_str);
+            lttng_port_to_str(p, port_str);
+            LTTNG3(driver_flush, proc_str, port_str, p->name);
+        }
+#endif
+
+
         if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(p, am_in, am_flush);
 	}
@@ -3551,6 +3592,16 @@ terminate_port(Port *prt)
             DTRACE3(driver_stop, process_str, drv->name, port_str);
         }
 #endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+        if (LTTNG_ENABLED(driver_stop)) {
+            lttng_decl_portbuf(port_str);
+            lttng_decl_procbuf(proc_str);
+            lttng_pid_to_str(connected_id, proc_str);
+            lttng_port_to_str(prt, port_str);
+            LTTNG3(driver_stop, proc_str, drv->name, port_str);
+        }
+#endif
+
 	(*drv->stop)((ErlDrvData)prt->drv_data);
 	erts_unblock_fpe(fpe_was_unmasked);
 	ERTS_MSACC_POP_STATE_M();
@@ -3881,6 +3932,16 @@ call_driver_control(Eterm caller,
     
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
 
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+    if (LTTNG_ENABLED(driver_control)) {
+        lttng_decl_procbuf(proc_str);
+        lttng_decl_portbuf(port_str);
+        lttng_pid_to_str(caller, proc_str);
+        lttng_port_to_str(prt, port_str);
+        LTTNG5(driver_control, proc_str, port_str, prt->name, command, size);
+    }
+#endif
+
     prt->caller = caller;
     cres = prt->drv_ptr->control((ErlDrvData) prt->drv_data,
 				 command,
@@ -4294,6 +4355,15 @@ call_driver_call(Eterm caller,
         DTRACE5(driver_call, process_str, port_str, prt->name, command, size);
     }
 #endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+    if (LTTNG_ENABLED(driver_call)) {
+        lttng_decl_procbuf(proc_str);
+        lttng_decl_portbuf(port_str);
+        lttng_pid_to_str(caller,proc_str);
+        lttng_port_to_str(prt, port_str);
+        LTTNG5(driver_call, proc_str, port_str, prt->name, command, size);
+    }
+#endif
 
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
 
@@ -5055,6 +5125,15 @@ int async_ready(Port *p, void* data)
                 DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p)
                 DTRACE3(driver_ready_async, process_str, port_str, p->name);
             }
+#endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS 
+            if (LTTNG_ENABLED(driver_ready_async)) {
+                lttng_decl_portbuf(port_str);
+                lttng_decl_procbuf(proc_str);
+                lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(p), proc_str);
+                lttng_port_to_str(p, port_str);
+                LTTNG3(driver_ready_async, proc_str, port_str, p->name);
+            }
 #endif
 	    (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data);
 	    need_free = 0;
@@ -7101,6 +7180,15 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
         DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(prt), prt)
         DTRACE3(driver_process_exit, process_str, port_str, prt->name);
     }
+#endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+    if (LTTNG_ENABLED(driver_process_exit)) {
+        lttng_decl_portbuf(port_str);
+        lttng_decl_procbuf(proc_str);
+        lttng_pid_to_str(ERTS_PORT_GET_CONNECTED(prt), proc_str);
+        lttng_port_to_str(prt, port_str);
+        LTTNG3(driver_process_exit, proc_str, port_str, prt->name);
+    }
 #endif
     fpe_was_unmasked = erts_block_fpe();
     (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor);
@@ -7579,6 +7667,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 	int fpe_was_unmasked = erts_block_fpe();
         DTRACE4(driver_init, drv->name, drv->version.major, drv->version.minor,
                 drv->flags);
+        LTTNG4(driver_init, drv->name, drv->version.major, drv->version.minor,
+                drv->flags);
 	res = (*de->init)();
 	erts_unblock_fpe(fpe_was_unmasked);
 	return res;
-- 
cgit v1.2.3


From 8735e04e031284bd73d0cf5b9fddebf624623c02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 19 Aug 2015 17:43:33 +0200
Subject: erts: Add lttng tracepoints for async pool queue

* aio_pool_get
* aio_pool_add
---
 erts/emulator/beam/erl_async.c     | 16 ++++++++++++++++
 erts/emulator/beam/erl_thr_queue.c | 32 ++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_thr_queue.h |  4 ++++
 erts/emulator/beam/erlang_lttng.h  | 28 ++++++++++++++++++++++++++++
 4 files changed, 80 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index cdeeb5281b..69240f7886 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -28,6 +28,7 @@
 #include "erl_thr_queue.h"
 #include "erl_async.h"
 #include "dtrace-wrapper.h"
+#include "lttng-wrapper.h"
 
 #define ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ 20
 
@@ -281,6 +282,13 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
 #endif
 
     erts_thr_q_enqueue(&q->thr_q, a);
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+    if (LTTNG_ENABLED(aio_pool_add)) {
+        lttng_decl_portbuf(port_str);
+        lttng_portid_to_str(a->port, port_str);
+        LTTNG2(aio_pool_add, port_str, -1);
+    }
+#endif
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(aio_pool_add)) {
         DTRACE_CHARBUF(port_str, 16);
@@ -317,6 +325,14 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 	    if (saved_fin_deq)
 		erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq);
 #endif
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+            if (LTTNG_ENABLED(aio_pool_get)) {
+                lttng_decl_portbuf(port_str);
+                int length = erts_thr_q_length_dirty(q);
+                lttng_portid_to_str(a->port, port_str);
+                LTTNG2(aio_pool_get, port_str, length);
+            }
+#endif
 #ifdef USE_VM_PROBES
             if (DTRACE_ENABLED(aio_pool_get)) {
                 DTRACE_CHARBUF(port_str, 16);
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index 7ff456b915..30c9d70c59 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -780,3 +780,35 @@ erts_thr_q_dequeue(ErtsThrQ_t *q)
     return res;
 #endif
 }
+
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+int
+erts_thr_q_length_dirty(ErtsThrQ_t *q)
+{
+    int n = 0;
+#ifndef USE_THREADS
+    void *res;
+    ErtsThrQElement_t *tmp;
+
+    for (tmp = q->first; tmp != NULL; tmp = tmp->next) {
+        n++;
+    }
+#else
+    ErtsThrQElement_t *e;
+    erts_aint_t inext;
+
+    e = ErtsThrQDirtyReadEl(&q->head.head);
+    inext = erts_atomic_read_acqb(&e->next);
+
+    while (inext != ERTS_AINT_NULL) {
+        e = (ErtsThrQElement_t *) inext;
+        if (e != &q->tail.data.marker) {
+            /* don't count marker */
+            n++;
+        }
+        inext = erts_atomic_read_acqb(&e->next);
+    }
+#endif
+    return n;
+}
+#endif
diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h
index 27a6d03224..f5e5522948 100644
--- a/erts/emulator/beam/erl_thr_queue.h
+++ b/erts/emulator/beam/erl_thr_queue.h
@@ -190,6 +190,10 @@ void erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *,
 int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *);
 void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *);
 
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+int erts_thr_q_length_dirty(ErtsThrQ_t *);
+#endif
+
 #ifdef ERTS_SMP
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q);
 #endif
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index b5cd90d93e..a202ce68c9 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -307,6 +307,34 @@ TRACEPOINT_EVENT(
     )
 )
 
+/* Async pool */
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    aio_pool_get,
+    TP_ARGS(
+        char*, port,
+        int, length
+    ),
+    TP_FIELDS(
+        ctf_string(port, port)
+        ctf_integer(int, length, length)
+    )
+)
+
+TRACEPOINT_EVENT(
+    com_ericsson_otp,
+    aio_pool_add,
+    TP_ARGS(
+        char*, port,
+        int, length
+    ),
+    TP_FIELDS(
+        ctf_string(port, port)
+        ctf_integer(int, length, length)
+    )
+)
+
 
 /* Memory Allocator */
 
-- 
cgit v1.2.3


From 977d50fcbc119a2d6d3bf2512605e62a3f22741b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 21 Jan 2016 10:33:40 +0100
Subject: Refactor and fix dtrace define in erl_message

---
 erts/emulator/beam/erl_message.c | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 88efb2c59f..bc0a55068b 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -766,17 +766,7 @@ erts_send_message(Process* sender,
 		utag = DT_UTAG(sender);
 	    else
 		utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp);
-#ifdef DTRACE_TAG_HARDDEBUG
-	    erts_fprintf(stderr,
-			 "Dtrace -> (%T) Spreading tag (%T) with "
-			 "message %T!\r\n",sender->common.id, utag, message);
-#endif
 	}
-#endif
-        BM_MESSAGE_COPIED(msize);
-        BM_SWAP_TIMER(copy,send);
-
-#ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(message_send)) {
             if (have_seqtrace(stoken)) {
 		tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
@@ -787,6 +777,9 @@ erts_send_message(Process* sender,
 		    msize, tok_label, tok_lastcnt, tok_serial);
         }
 #endif
+        BM_MESSAGE_COPIED(msize);
+        BM_SWAP_TIMER(copy,send);
+
     } else {
         Eterm *hp;
 
@@ -822,8 +815,10 @@ erts_send_message(Process* sender,
 	    BM_MESSAGE_COPIED(msz);
 	    BM_SWAP_TIMER(copy,send);
 	}
+#ifdef USE_VM_PROBES
         DTRACE6(message_send, sender_name, receiver_name,
                 msize, tok_label, tok_lastcnt, tok_serial);
+#endif
     }
 
     res = queue_message(sender,
-- 
cgit v1.2.3


From 84f2e9b3b1bb3990a7bea7b9d45768ee1a820804 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 2 Mar 2016 15:06:06 +0100
Subject: erts: Extend erlang:system_info/1 with lttng

Let erlang:system_info(dynamic_trace) be able to return 'lttng' if enabled.
---
 erts/emulator/beam/erl_bif_info.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5c03e33b74..dd796199bb 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2751,6 +2751,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 #elif defined(USE_SYSTEMTAP)
 	DECL_AM(systemtap);
 	BIF_RET(AM_systemtap);
+#elif defined(USE_LTTNG)
+	DECL_AM(lttng);
+	BIF_RET(AM_lttng);
 #else
 	BIF_RET(am_none);
 #endif
-- 
cgit v1.2.3


From 58762e934f603b3fe036aa3b9d8a5930b77ae5f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 5 Apr 2016 18:56:16 +0200
Subject: erts: Don't use ratio in carrier lttng tracepoints

---
 erts/emulator/beam/erlang_lttng.h | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index a202ce68c9..43ceeda671 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -352,7 +352,6 @@ TRACEPOINT_EVENT(
         ctf_string(type, type)
         ctf_integer(int, instance, instance)
         ctf_integer(unsigned long, size, size)
-        ctf_float(float, mbc_ratio, (float)mbcs->blocks.size/(float)mbcs->carriers.size)
         ctf_integer(unsigned long, mbc_carriers, mbcs->carriers.no)
         ctf_integer(unsigned long, mbc_carriers_size, mbcs->carriers.size)
         ctf_integer(unsigned long, mbc_blocks, mbcs->blocks.no)
@@ -379,7 +378,6 @@ TRACEPOINT_EVENT(
         ctf_string(type, type)
         ctf_integer(int, instance, instance)
         ctf_integer(unsigned long, size, size)
-        ctf_float(float, mbc_ratio, (float)mbcs->blocks.size/(float)mbcs->carriers.size)
         ctf_integer(unsigned long, mbc_carriers, mbcs->carriers.no)
         ctf_integer(unsigned long, mbc_carriers_size, mbcs->carriers.size)
         ctf_integer(unsigned long, mbc_blocks, mbcs->blocks.no)
-- 
cgit v1.2.3


From 57551877d85ad7659201235e27498be42809fefb Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 18 Feb 2016 13:52:54 +0100
Subject: erts: Add enif_send with NULL as msg env

This is an optimization for reducing the number of heap fragments
allocated when sending a message where the majority of the
message payload is on the sending process' heap.
---
 erts/emulator/beam/erl_nif.c | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d3030070b7..1012ced44b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -312,7 +312,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     Process* rp;
     Process* c_p;
     ErtsMessage *mp;
-    ErlHeapFragment* frags;
     Eterm receiver = to_pid->pid;
     int flush_me = 0;
     int scheduler = erts_get_scheduler_id() != 0;
@@ -340,25 +339,32 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 	ASSERT(env == NULL || receiver != c_p->common.id);
 	return 0;
     }
-    flush_env(msg_env);
-    frags = menv->env.heap_frag; 
-    ASSERT(frags == MBUF(&menv->phony_proc));
-    if (frags != NULL) {
-	/* Move all offheap's from phony proc to the first fragment.
-	   Quick and dirty... */
-	ASSERT(!is_offheap(&frags->off_heap));
-	frags->off_heap = MSO(&menv->phony_proc);
-	clear_offheap(&MSO(&menv->phony_proc));
-	menv->env.heap_frag = NULL; 
-	MBUF(&menv->phony_proc) = NULL;
+    if (menv) {
+        flush_env(msg_env);
+        mp = erts_alloc_message(0, NULL);
+        mp->data.heap_frag = menv->env.heap_frag;
+        ASSERT(mp->data.heap_frag == MBUF(&menv->phony_proc));
+        if (mp->data.heap_frag != NULL) {
+            /* Move all offheap's from phony proc to the first fragment.
+               Quick and dirty... */
+            ASSERT(!is_offheap(&mp->data.heap_frag->off_heap));
+            mp->data.heap_frag->off_heap = MSO(&menv->phony_proc);
+            clear_offheap(&MSO(&menv->phony_proc));
+            menv->env.heap_frag = NULL;
+            MBUF(&menv->phony_proc) = NULL;
+        }
+        ASSERT(!is_offheap(&MSO(&menv->phony_proc)));
+    } else {
+        Uint sz = size_object(msg);
+        Eterm *hp;
+        mp = erts_alloc_message(sz, &hp);
+        msg = copy_struct(msg, sz, &hp, &mp->hfrag.off_heap);
+        ASSERT(hp == mp->hfrag.mem+mp->hfrag.used_size);
     }
-    ASSERT(!is_offheap(&MSO(&menv->phony_proc)));
 
     if (flush_me) {	
 	flush_env(env); /* Needed for ERTS_HOLE_CHECK */ 
     }
-    mp = erts_alloc_message(0, NULL);
-    mp->data.heap_frag = frags;
     erts_queue_message(rp, &rp_locks, mp, msg, am_undefined);
     if (c_p == rp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-- 
cgit v1.2.3


From 6d51b25958393d95dee32baafd708aa3909ddb5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 1 Apr 2016 16:07:37 +0200
Subject: Eliminate allocation of variables in transform_engine()

When an instruction with a variable number operands (such as
select_val) is seen of the left side of a transformation, the
'next_arg' instruction will allocate a buffer to fit all variables and
all operands will be copied into the buffer. Very often, the 'commit'
instruction will never be reached because of a test or predicate
failing or because of a short window; in that case, the variable
buffer will be deallocated.

Note that originally there were only few instructions with a variable
number of operands, but now common operations such as tuple building
also have a variable number of operands.

To avoid those frequent allocations and deallocations, modify the
'next_arg' instruction to only save a pointer to the first of the
"rest" arguments. Also move the deallocation of the instructions
on the left side from the 'commit' instruction to the 'end'
instruction to ensure that 'store_rest_args' will still work.
---
 erts/emulator/beam/beam_load.c | 94 ++++++++++++------------------------------
 1 file changed, 27 insertions(+), 67 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2f2b433999..c6c35e74c9 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4813,31 +4813,25 @@ transform_engine(LoaderState* st)
     Uint op;
     int ap;			/* Current argument. */
     Uint* restart;		/* Where to restart if current match fails. */
-    GenOpArg def_vars[TE_MAX_VARS]; /* Default buffer for variables. */
-    GenOpArg* var = def_vars;
-    int num_vars = 0;
+    GenOpArg var[TE_MAX_VARS];	/* Buffer for variables. */
+    GenOpArg* rest_args = NULL;
+    int num_rest_args = 0;
     int i;			/* General index. */
     Uint mask;
     GenOp* instr;
+    GenOp* first = st->genop;
+    GenOp* keep = NULL;
     Uint* pc;
-    int rval;
     static Uint restart_fail[1] = {TOP_fail};
 
-    ASSERT(gen_opc[st->genop->op].transform != -1);
-    pc = op_transform + gen_opc[st->genop->op].transform;
-    restart = pc;
+    ASSERT(gen_opc[first->op].transform != -1);
+    restart = op_transform + gen_opc[first->op].transform;
 
  restart:
-    if (var != def_vars) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) var);
-	var = def_vars;
-    }
     ASSERT(restart != NULL);
     pc = restart;
     ASSERT(*pc < NUM_TOPS);	/* Valid instruction? */
-    instr = st->genop;
-
-#define RETURN(r) rval = (r); goto do_return;
+    instr = first;
 
 #ifdef DEBUG
     restart = NULL;
@@ -4855,7 +4849,7 @@ transform_engine(LoaderState* st)
 		 * We'll need at least one more instruction to decide whether
 		 * this combination matches or not.
 		 */
-		RETURN(TE_SHORT_WINDOW);
+		return TE_SHORT_WINDOW;
 	    }
 	    if (*pc++ != instr->op)
 		goto restart;
@@ -5017,19 +5011,9 @@ transform_engine(LoaderState* st)
 #if defined(TOP_rest_args)
 	case TOP_rest_args:
 	    {
-		int n = *pc++;
 		int formal_arity = gen_opc[instr->op].arity;
-		int j = formal_arity;
-
-		num_vars = n + (instr->arity - formal_arity);
-		var = erts_alloc(ERTS_ALC_T_LOADER_TMP,
-				 num_vars * sizeof(GenOpArg));
-		for (i = 0; i < n; i++) {
-		    var[i] = def_vars[i];
-		}
-		while (i < num_vars) {
-		    var[i++] = instr->a[j++];
-		}
+		num_rest_args = instr->arity - formal_arity;
+		rest_args = instr->a + formal_arity;
 	    }
 	    break;
 #endif
@@ -5038,16 +5022,8 @@ transform_engine(LoaderState* st)
 	    break;
 	case TOP_commit:
 	    instr = instr->next; /* The next_instr was optimized away. */
-
-	    /*
-	     * The left-hand side of this transformation matched.
-	     * Delete all matched instructions.
-	     */
-	    while (st->genop != instr) {
-		GenOp* next = st->genop->next;
-		FREE_GENOP(st, st->genop);
-		st->genop = next;
-	    }
+	    keep = instr;
+	    st->genop = instr;
 #ifdef DEBUG
 	    instr = 0;
 #endif
@@ -5077,22 +5053,19 @@ transform_engine(LoaderState* st)
 		    lastp = &((*lastp)->next);
 		}
 		 
-		instr = instr->next; /* The next_instr was optimized away. */
-
-		/*
-		 * The left-hand side of this transformation matched.
-		 * Delete all matched instructions.
-		 */
-		while (st->genop != instr) {
-		    GenOp* next = st->genop->next;
-		    FREE_GENOP(st, st->genop);
-		    st->genop = next;
-		}
-		*lastp = st->genop;
+		keep = instr->next; /* The next_instr was optimized away. */
+		*lastp = keep;
 		st->genop = new_instr;
 	    }
-	    RETURN(TE_OK);
+	    /* FALLTHROUGH */
 #endif
+	case TOP_end:
+	    while (first != keep) {
+		GenOp* next = first->next;
+		FREE_GENOP(st, first);
+		first = next;
+	    }
+	    return TE_OK;
 	case TOP_new_instr:
 	    /*
 	     * Note that the instructions are generated in reverse order.
@@ -5123,14 +5096,10 @@ transform_engine(LoaderState* st)
 #if defined(TOP_store_rest_args)
 	case TOP_store_rest_args:
 	    {
-		int n = *pc++;
-		int num_extra = num_vars - n;
-
-		ASSERT(n <= num_vars);
-		GENOP_ARITY(instr, instr->arity+num_extra);
+		GENOP_ARITY(instr, instr->arity+num_rest_args);
 		memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg));
-		memcpy(instr->a+ap, var+n, num_extra*sizeof(GenOpArg));
-		ap += num_extra;
+		memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg));
+		ap += num_rest_args;
 	    }
 	    break;
 #endif
@@ -5142,21 +5111,12 @@ transform_engine(LoaderState* st)
 	case TOP_try_me_else_fail:
 	    restart = restart_fail;
 	    break;
-	case TOP_end:
-	    RETURN(TE_OK);
 	case TOP_fail:
-	    RETURN(TE_FAIL);
+	    return TE_FAIL;
 	default:
 	    ASSERT(0);
 	}
     }
-#undef RETURN
-
- do_return:
-    if (var != def_vars) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) var);
-    }
-    return rval;
 }
 
 static void
-- 
cgit v1.2.3


From c6cabe0b76dda183d209498e1e4e13e3407dcf9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 7 Dec 2015 14:51:44 +0100
Subject: Simplify window management for the transformation engine

Generic instructions have a min_window field. Its purpose is to
avoid calling transform_engine() when there are too few instructions
in the current "transformation window" for a transformation to
succeed.

Currently it does not do much good since the window size will be
decremented by one before being used. The reason for the subtraction
is probably that in some circumstances in the past, the loader could
read past the end of the BEAM module while attempting to fetch
instructions to increase the window size. Therefore, it would not
be safe to just remove the subtraction by one.

The simplest and safest solution seems to always ensure that there
are always at least TWO instructions when calling transform_engine().
That will be safe, as long as a BEAM module is always finished with
an int_code_end/0 that is not involved in any transformation.
---
 erts/emulator/beam/beam_load.c | 16 ++++++++--------
 erts/emulator/beam/beam_load.h |  1 -
 2 files changed, 8 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index c6c35e74c9..5d03c98657 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2030,14 +2030,14 @@ load_code(LoaderState* stp)
     do_transform:
 	ASSERT(stp->genop != NULL);
 	if (gen_opc[stp->genop->op].transform != -1) {
-	    int need;
-	    tmp_op = stp->genop;
-
-	    for (need = gen_opc[stp->genop->op].min_window-1; need > 0; need--) {
-		if (tmp_op == NULL) {
-		    goto get_next_instr;
-		}
-		tmp_op = tmp_op->next;
+	    if (stp->genop->next == NULL) {
+		/*
+		 * Simple heuristic: Most transformations requires
+		 * at least two instructions, so make sure that
+		 * there are. That will reduce the number of
+		 * TE_SHORT_WINDOWs.
+		 */
+		goto get_next_instr;
 	    }
 	    switch (transform_engine(stp)) {
 	    case TE_FAIL:
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 22ab71c868..68f4b96893 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -33,7 +33,6 @@ typedef struct gen_op_entry {
    int specific;
    int num_specific;
    int transform;
-   int min_window;
 } GenOpEntry;
 
 extern GenOpEntry gen_opc[];
-- 
cgit v1.2.3


From e93a66110aa27a5b8228fb46a3459a6de0e626d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 7 Dec 2015 16:12:21 +0100
Subject: Introduce a 'rename' instruction

Introduce a 'rename' instruction that can be used to optimize
simple renaming with unchanged operands such as:

  get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst

By allowing it to lower the arity of instruction, transformations
such as the following can be handled:

  trim N Remaining => i_trim N

All in all, currently 67 transformations can be optimized in this
way, including some commonly used ones.
---
 erts/emulator/beam/beam_load.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 5d03c98657..ad174664ae 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5077,6 +5077,12 @@ transform_engine(LoaderState* st)
 	    instr->arity = gen_opc[op].arity;
 	    ap = 0;
 	    break;
+#ifdef TOP_rename
+	case TOP_rename:
+	    instr->op = op = *pc++;
+	    instr->arity = gen_opc[op].arity;
+	    return TE_OK;
+#endif
 	case TOP_store_type:
 	    i = *pc++;
 	    instr->a[ap].type = i;
-- 
cgit v1.2.3


From 937f527054f13dd524588c064cd5d76e3cfd23eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 5 Apr 2016 12:56:57 +0200
Subject: Avoid rebuilding unchanged instructions

In transformations such as:

  move S X0=x==0 | line Loc  | call_ext Ar Func => \
  line Loc       | move S X0 | call_ext Ar Func

we can avoid rebuilding the last instruction in the sequence
by introducing a 'keep' instruction.

Currently, there are only 13 transformations that are hit by
this optimization, but most of them are frequently used.
---
 erts/emulator/beam/beam_load.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index ad174664ae..bdb451a6fe 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5028,7 +5028,16 @@ transform_engine(LoaderState* st)
 	    instr = 0;
 #endif
 	    break;
-
+#if defined(TOP_keep)
+	case TOP_keep:
+	    /* Keep the current instruction unchanged. */
+	    keep = instr;
+	    st->genop = instr;
+#ifdef DEBUG
+	    instr = 0;
+#endif
+	    break;
+#endif
 #if defined(TOP_call_end)
 	case TOP_call_end:
 	    {
-- 
cgit v1.2.3


From 4f33597d52a0cef2e47b07578bc8a35a17c2f969 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 6 Apr 2016 07:02:36 +0200
Subject: Don't let the loader do the compiler's job

Optimizations that are possible to do by the compiler should be
done by the compiler and not by the loader.

If the compiler has done its job correctly, attempting to do the two
transformations only wastes time.
---
 erts/emulator/beam/beam_load.c | 7 -------
 erts/emulator/beam/ops.tab     | 5 -----
 2 files changed, 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index bdb451a6fe..a98900460e 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2735,13 +2735,6 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
     return 0;
 }
 
-static int
-same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label)
-{
-    return Target.type = TAG_f && Label.type == TAG_u &&
-	Target.val == Label.val;
-}
-
 static int
 is_killed_apply(LoaderState* stp, GenOpArg Reg, GenOpArg Live)
 {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 78000160e3..485c072540 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -181,11 +181,6 @@ i_jump_on_val_zero y f I
 i_jump_on_val x f I I
 i_jump_on_val y f I I
 
-jump Target | label Lbl | same_label(Target, Lbl) => label Lbl
-
-is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \
-  is_eq_exact Fail S1 S2 | label L2
-
 %macro: get_list GetList -pack
 get_list x x x
 get_list x x y
-- 
cgit v1.2.3


From 921c838b8142d3c8d1739c6b30c6d88e39e5f147 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 6 Apr 2016 07:13:26 +0200
Subject: Eliminate unnecessary renaming of bs_put_utf16/3

There is no reason to rename bs_put_utf16/3.

(We rename instructions if we'll need to change the operands or
if we will need to avoid an endless transformation loop. Neither
of these reasons apply to bs_put_utf16/3.)
---
 erts/emulator/beam/beam_emu.c | 2 +-
 erts/emulator/beam/ops.tab    | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a390422040..09a41f2b56 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4102,7 +4102,7 @@ do {						\
      StoreBifResult(1, result);
  }
 
- OpCase(i_bs_put_utf16_jIs): {
+ OpCase(bs_put_utf16_jIs): {
      Eterm arg;
 
      GetArg1(2, arg);
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 485c072540..15f27835a8 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1350,9 +1350,7 @@ bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
 
 i_bs_put_utf8 j s
 
-bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src
-
-i_bs_put_utf16 j I s
+bs_put_utf16 j I s
 
 bs_put_utf32 Fail=j Flags=u Src=s => \
    i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
-- 
cgit v1.2.3


From f821130175400d72dbf29bf82f81f7495fb71272 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Nov 2015 15:21:55 +0100
Subject: erts: Rename atom '' from am_Cookie to am_Empty

---
 erts/emulator/beam/atom.names | 2 +-
 erts/emulator/beam/dist.c     | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 169b071cd7..2eb63febe7 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -43,7 +43,7 @@ atom false true
 atom Underscore='_'
 atom Noname='nonode@nohost'
 atom EOT='$end_of_table'
-atom Cookie=''
+atom Empty=''
 
 #
 # Used in the Beam emulator loop. (Smaller literals usually means tighter code.)
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index fa385f105d..3a6da373c3 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -906,9 +906,9 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
 
     if (token != NIL)
 	ctl = TUPLE4(&ctx->ctl_heap[0],
-		     make_small(DOP_SEND_TT), am_Cookie, remote, token);
+		     make_small(DOP_SEND_TT), am_Empty, remote, token);
     else
-	ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote);
+	ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Empty, remote);
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
@@ -963,10 +963,10 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message,
 
     if (token != NIL)
 	ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_REG_SEND_TT),
-		     sender->common.id, am_Cookie, remote_name, token);
+		     sender->common.id, am_Empty, remote_name, token);
     else
 	ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_REG_SEND),
-		     sender->common.id, am_Cookie, remote_name);
+		     sender->common.id, am_Empty, remote_name);
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
-- 
cgit v1.2.3


From 995f81ed452f95c065bd94d62cb69dbf5289be76 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Nov 2015 16:14:33 +0100
Subject: erts: Refactor ETS compressed encoding of local node

Instead of INTERNAL_CREATION (255), use empty atom for node name
to mean the local node (regardless of node name or creation).

The purpose is to get rid of special value 255, for future expansion
of creation to 32-bit.
---
 erts/emulator/beam/erl_vm.h   |  1 -
 erts/emulator/beam/external.c | 43 +++++++++++++++++++++----------------------
 2 files changed, 21 insertions(+), 23 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 98f27a1725..357094633e 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -165,7 +165,6 @@ extern int erts_atom_table_size;/* Atom table size */
 extern int erts_pd_initial_size;/* Initial Process dictionary table size */
 
 #define ORIG_CREATION 0
-#define INTERNAL_CREATION 255
 
 /* macros for extracting bytes from uint16's */
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 10f03636ec..de0c155fc0 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -51,7 +51,7 @@
 
 #define MAX_STRING_LEN 0xffff
 
-#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION)
+#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION)
 
 #undef ERTS_DEBUG_USE_DIST_SEP
 #ifdef DEBUG
@@ -2152,12 +2152,12 @@ static byte*
 enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
 {
     Uint on, os;
+    Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS))
+		      ? am_Empty : pid_node_name(pid));
 
     *ep++ = PID_EXT;
     /* insert  atom here containing host and sysname  */
-    ep = enc_atom(acmp, pid_node_name(pid), ep, dflags);
-
-    /* two bytes for each number and serial */
+    ep = enc_atom(acmp, sysname, ep, dflags);
 
     on = pid_number(pid);
     os = pid_serial(pid);
@@ -2166,8 +2166,7 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
     ep += 4;
     put_int32(os, ep);
     ep += 4;
-    *ep++ = (is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) ?
-	INTERNAL_CREATION : pid_creation(pid);
+    *ep++ = pid_creation(pid);
     return ep;
 }
 
@@ -2249,14 +2248,13 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp)
 
 static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
 {
-    switch (creation) {
-    case INTERNAL_CREATION:
+    if (sysname == am_Empty)  /* && DFLAG_INTERNAL_TAGS */
 	return erts_this_node;
-    case ORIG_CREATION:
-	if (sysname == erts_this_node->sysname) {
-	    creation = erts_this_node->creation;
-	}
-    }
+
+    if (sysname == erts_this_node->sysname
+	&& (creation == erts_this_node->creation || creation == ORIG_CREATION))
+	return erts_this_node;
+
     return erts_find_or_insert_node(sysname,creation);
 }
 
@@ -2528,6 +2526,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	case REF_DEF:
 	case EXTERNAL_REF_DEF: {
 	    Uint32 *ref_num;
+	    Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj))
+			     ? am_Empty : ref_node_name(obj));
 
 	    ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
 
@@ -2535,9 +2535,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    i = ref_no_of_numbers(obj);
 	    put_int16(i, ep);
 	    ep += 2;
-	    ep = enc_atom(acmp,ref_node_name(obj),ep,dflags);
-	    *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) ?
-		INTERNAL_CREATION : ref_creation(obj);
+	    ep = enc_atom(acmp, sysname, ep, dflags);
+	    *ep++ = ref_creation(obj);
 	    ref_num = ref_numbers(obj);
 	    for (j = 0; j < i; j++) {
 		put_int32(ref_num[j], ep);
@@ -2546,17 +2545,17 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    break;
 	}
 	case PORT_DEF:
-	case EXTERNAL_PORT_DEF:
-
+	case EXTERNAL_PORT_DEF: {
+	    Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj))
+			     ? am_Empty : port_node_name(obj));
 	    *ep++ = PORT_EXT;
-	    ep = enc_atom(acmp,port_node_name(obj),ep,dflags);
+	    ep = enc_atom(acmp, sysname, ep, dflags);
 	    j = port_number(obj);
 	    put_int32(j, ep);
 	    ep += 4;
-	    *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) ?
-		INTERNAL_CREATION : port_creation(obj);
+	    *ep++ = port_creation(obj);
 	    break;
-
+	}
 	case LIST_DEF:
 	    {
 		int is_str;
-- 
cgit v1.2.3


From bf0bf9cf009fa8ccc7fc364fdbbdeb6f491efe43 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Nov 2015 16:47:40 +0100
Subject: erts: Support 32-bit creation for external pid,port,refs

from future nodes.
---
 erts/emulator/beam/erl_node_tables.c |  28 +------
 erts/emulator/beam/erl_node_tables.h |   2 +-
 erts/emulator/beam/erl_term.h        |  10 +--
 erts/emulator/beam/external.c        | 157 ++++++++++++++++++++++++++---------
 erts/emulator/beam/external.h        |   5 +-
 5 files changed, 125 insertions(+), 77 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 8617f42d7b..79da705e0f 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -497,31 +497,7 @@ node_table_hash(void *venp)
     Uint32 cre = ((ErlNode *) venp)->creation;
     HashValue h = atom_tab(atom_val(((ErlNode *) venp)->sysname))->slot.bucket.hvalue;
 
-    h *= PRIME0;
-    h += cre & 0xff;
-
-#if MAX_CREATION >= (1 << 8)
-    h *= PRIME1;
-    h += (cre >> 8) & 0xff;
-#endif
-
-#if MAX_CREATION >= (1 << 16)
-    h *= PRIME2;
-    h += (cre >> 16) & 0xff;
-#endif
-
-#if MAX_CREATION >= (1 << 24)
-    h *= PRIME3;
-    h += (cre >> 24) & 0xff;
-#endif
-
-#if 0
-/* XXX Problems in older versions of GCC */
- #if MAX_CREATION >= (1UL << 32)
- #error "MAX_CREATION larger than size of expected creation storage (Uint32)"
- #endif
-#endif
-    return h;
+    return (h + cre) * PRIME0;
 }
 
 static int
@@ -599,7 +575,7 @@ erts_node_table_info(int to, void *to_arg)
 }
 
 
-ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation)
+ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation)
 {
     ErlNode *res;
     ErlNode ne;
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index fb2f2a5407..2b93f1f08a 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -182,7 +182,7 @@ Uint erts_dist_table_size(void);
 void erts_dist_table_info(int, void *);
 void erts_set_dist_entry_not_connected(DistEntry *);
 void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint);
-ErlNode *erts_find_or_insert_node(Eterm, Uint);
+ErlNode *erts_find_or_insert_node(Eterm, Uint32);
 void erts_schedule_delete_node(ErlNode *);
 void erts_set_this_node(Eterm, Uint);
 Uint erts_node_table_size(void);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 0a71534790..fc58853b5e 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -558,14 +558,6 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm)
 
 #define _GETBITS(X,Pos,Size) (((X) >> (Pos)) & ~(~((Uint) 0) << (Size)))
 
-/*
- * Creation in node specific data (pids, ports, refs)
- */
-
-#define _CRE_SIZE  		2
-
-/* MAX value for the creation field in pid, port and reference */
-#define MAX_CREATION	(1 << _CRE_SIZE)
 
 /*
  *  PID layout (internal pids):
@@ -579,7 +571,7 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm)
  *
  *  n : number
  *
- *  Old pid layout:
+ *  Very old pid layout:
  *
  *   |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
  *   |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index de0c155fc0..5ea155f83f 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -51,7 +51,18 @@
 
 #define MAX_STRING_LEN 0xffff
 
-#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION)
+/* MAX value for the creation field in pid, port and reference
+   for the local node and for the current external format.
+
+   Larger creation values than this are allowed in external pid, port and refs
+   encoded with NEW_PID_EXT, NEW_PORT_EXT and NEWER_REFERENCE_EXT.
+   The point here is to prepare for future upgrade to 32-bit creation.
+   OTP-19 (erts-8.0) can handle big creation values from other (newer) nodes,
+   but do not use big creation values for the local node yet,
+   as we still may have to communicate with older nodes.
+*/
+#define ERTS_MAX_LOCAL_CREATION (3)
+#define is_valid_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_LOCAL_CREATION)
 
 #undef ERTS_DEBUG_USE_DIST_SEP
 #ifdef DEBUG
@@ -97,7 +108,7 @@ static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 struct B2TContext_t;
 static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
-static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*);
+static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*, byte tag);
 static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*);
 static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1);
 
@@ -2154,8 +2165,9 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
     Uint on, os;
     Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS))
 		      ? am_Empty : pid_node_name(pid));
+    Uint32 creation = pid_creation(pid);
+    byte* tagp = ep++;
 
-    *ep++ = PID_EXT;
     /* insert  atom here containing host and sysname  */
     ep = enc_atom(acmp, sysname, ep, dflags);
 
@@ -2166,7 +2178,15 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
     ep += 4;
     put_int32(os, ep);
     ep += 4;
-    *ep++ = pid_creation(pid);
+    if (creation <= ERTS_MAX_LOCAL_CREATION) {
+        *tagp = PID_EXT;
+        *ep++ = creation;
+    } else {
+        ASSERT(is_external_pid(pid));
+        *tagp = NEW_PID_EXT;
+        put_int32(creation, ep);
+        ep += 4;
+    }
     return ep;
 }
 
@@ -2246,7 +2266,7 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp)
     return ep;
 }
 
-static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
+static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation)
 {
     if (sysname == am_Empty)  /* && DFLAG_INTERNAL_TAGS */
 	return erts_this_node;
@@ -2259,13 +2279,14 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
 }
 
 static byte*
-dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp)
+dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep,
+        Eterm* objp, byte tag)
 {
     Eterm sysname;
     Uint data;
     Uint num;
     Uint ser;
-    Uint cre;
+    Uint32 cre;
     ErlNode *node;
 
     *objp = NIL;		/* In case we fail, don't leave a hole in the heap */
@@ -2281,12 +2302,19 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp)
     ep += 4;
     if (ser > ERTS_MAX_PID_SERIAL)
 	return NULL;
-    cre = get_int8(ep);
-    ep += 1;
 
-    if (!is_valid_creation(cre)) {
-	return NULL;
+    if (tag == PID_EXT) {
+        cre = get_int8(ep);
+        ep += 1;
+        if (!is_valid_creation(cre)) {
+            return NULL;
+        }
+    } else {
+        ASSERT(tag == NEW_PID_EXT);
+        cre = get_int32(ep);
+        ep += 4;
     }
+
     data = make_pid_data(ser, num);
 
     /*
@@ -2528,15 +2556,24 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    Uint32 *ref_num;
 	    Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj))
 			     ? am_Empty : ref_node_name(obj));
+            Uint32 creation = ref_creation(obj);
+            byte* tagp = ep++;
 
 	    ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
 
-	    *ep++ = NEW_REFERENCE_EXT;
 	    i = ref_no_of_numbers(obj);
 	    put_int16(i, ep);
 	    ep += 2;
 	    ep = enc_atom(acmp, sysname, ep, dflags);
-	    *ep++ = ref_creation(obj);
+            if (creation <= ERTS_MAX_LOCAL_CREATION) {
+                *tagp = NEW_REFERENCE_EXT;
+                *ep++ = creation;
+            } else {
+                ASSERT(is_external_ref(obj));
+                *tagp = NEWER_REFERENCE_EXT;
+                put_int32(creation, ep);
+                ep += 4;
+            }
 	    ref_num = ref_numbers(obj);
 	    for (j = 0; j < i; j++) {
 		put_int32(ref_num[j], ep);
@@ -2548,12 +2585,22 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	case EXTERNAL_PORT_DEF: {
 	    Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj))
 			     ? am_Empty : port_node_name(obj));
-	    *ep++ = PORT_EXT;
+            Uint32 creation = port_creation(obj);
+            byte* tagp = ep++;
+
 	    ep = enc_atom(acmp, sysname, ep, dflags);
 	    j = port_number(obj);
 	    put_int32(j, ep);
 	    ep += 4;
-	    *ep++ = port_creation(obj);
+            if (creation <= ERTS_MAX_LOCAL_CREATION) {
+                *tagp = PORT_EXT;
+                *ep++ = creation;
+            } else {
+                ASSERT(is_external_port(obj));
+                *tagp = NEW_PORT_EXT;
+                put_int32(creation, ep);
+                ep += 4;
+            }
 	    break;
 	}
 	case LIST_DEF:
@@ -3259,20 +3306,23 @@ dec_term_atom_common:
 		hp += FLOAT_SIZE_OBJECT;
 		break;
 	    }
-	case PID_EXT:
+        case PID_EXT:
+        case NEW_PID_EXT:
 	    factory->hp = hp;
-	    ep = dec_pid(edep, factory, ep, objp);
+	    ep = dec_pid(edep, factory, ep, objp, ep[-1]);
 	    hp = factory->hp;
 	    if (ep == NULL) {
 		goto error;
 	    }
 	    break;
-	case PORT_EXT:
+        case PORT_EXT:
+        case NEW_PORT_EXT:
 	    {
 		Eterm sysname;
 		ErlNode *node;
 		Uint num;
-		Uint cre;
+		Uint32 cre;
+                byte tag = ep[-1];
 
 		if ((ep = dec_atom(edep, ep, &sysname)) == NULL) {
 		    goto error;
@@ -3281,12 +3331,17 @@ dec_term_atom_common:
 		    goto error;
 		}
 		ep += 4;
-		cre = get_int8(ep);
-		ep++;
-		if (!is_valid_creation(cre)) {
-		    goto error;
-		}
-
+                if (tag == PORT_EXT) {
+                    cre = get_int8(ep);
+                    ep++;
+                    if (!is_valid_creation(cre)) {
+                        goto error;
+                    }
+                }
+                else {
+                    cre = get_int32(ep);
+                    ep += 4;
+                }
 		node = dec_get_node(sysname, cre);
 		if(node == erts_this_node) {
 		    *objp = make_internal_port(num);
@@ -3311,7 +3366,7 @@ dec_term_atom_common:
 		Eterm sysname;
 		ErlNode *node;
 		int i;
-		Uint cre;
+		Uint32 cre;
 		Uint32 *ref_num;
 		Uint32 r0;
 		Uint ref_words;
@@ -3335,9 +3390,6 @@ dec_term_atom_common:
 		ref_words = get_int16(ep);
 		ep += 2;
 
-		if (ref_words > ERTS_MAX_REF_NUMBERS)
-		    goto error;
-
 		if ((ep = dec_atom(edep, ep, &sysname)) == NULL)
 		    goto error;
 
@@ -3350,8 +3402,23 @@ dec_term_atom_common:
 		ep += 4;
 		if (r0 >= MAX_REFERENCE)
 		    goto error;
+		goto ref_ext_common;
+
+            case NEWER_REFERENCE_EXT:
+		ref_words = get_int16(ep);
+		ep += 2;
+
+		if ((ep = dec_atom(edep, ep, &sysname)) == NULL)
+		    goto error;
+
+		cre = get_int32(ep);
+		ep += 4;
+		r0 = get_int32(ep); /* allow full word */
+		ep += 4;
 
 	    ref_ext_common:
+		if (ref_words > ERTS_MAX_REF_NUMBERS)
+		    goto error;
 
 		node = dec_get_node(sysname, cre);
 		if(node == erts_this_node) {
@@ -3705,9 +3772,9 @@ dec_term_atom_common:
 		*objp = make_fun(funp);
 
 		/* Creator pid */
-		if (*ep != PID_EXT 
-		    || (ep = dec_pid(edep, factory, ++ep,
-				     &funp->creator))==NULL) { 
+		if ((*ep != PID_EXT && *ep != NEW_PID_EXT)
+		    || (ep = dec_pid(edep, factory, ep+1,
+				     &funp->creator, *ep))==NULL) {
 		    goto error;
 		}
 
@@ -4009,20 +4076,29 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    else
 		result += 1 + 4 + 1 + i;  /* tag,size,sign,digits */
 	    break;
+        case EXTERNAL_PID_DEF:
+            if (external_pid_creation(obj) > ERTS_MAX_LOCAL_CREATION)
+                result += 3;
+            /*fall through*/
 	case PID_DEF:
-	case EXTERNAL_PID_DEF:
 	    result += (1 + encode_size_struct2(acmp, pid_node_name(obj), dflags) +
 		       4 + 4 + 1);
 	    break;
+        case EXTERNAL_REF_DEF:
+            if (external_ref_creation(obj) > ERTS_MAX_LOCAL_CREATION)
+                result += 3;
+            /*fall through*/
 	case REF_DEF:
-	case EXTERNAL_REF_DEF:
 	    ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
 	    i = ref_no_of_numbers(obj);
 	    result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) +
 		       1 + 4*i);
 	    break;
-	case PORT_DEF:
-	case EXTERNAL_PORT_DEF:
+        case EXTERNAL_PORT_DEF:
+            if (external_port_creation(obj) > ERTS_MAX_LOCAL_CREATION)
+                result += 3;
+            /*fall through*/
+        case PORT_DEF:
 	    result += (1 + encode_size_struct2(acmp, port_node_name(obj), dflags) +
 		      4 + 1);
 	    break;
@@ -4349,19 +4425,22 @@ init_done:
 	    SKIP(1+atom_extra_skip);
 	    atom_extra_skip = 0;
 	    break;
-	case PID_EXT:
+        case PID_EXT:
+        case NEW_PID_EXT:
 	    atom_extra_skip = 9;
 	    /* In case it is an external pid */
 	    heap_size += EXTERNAL_THING_HEAD_SIZE + 1;
 	    terms++;
 	    break;
-	case PORT_EXT:
+        case PORT_EXT:
+        case NEW_PORT_EXT:
 	    atom_extra_skip = 5;
 	    /* In case it is an external port */
 	    heap_size += EXTERNAL_THING_HEAD_SIZE + 1;
 	    terms++;
 	    break;
-	case NEW_REFERENCE_EXT:
+        case NEW_REFERENCE_EXT:
+	case NEWER_REFERENCE_EXT:
 	    {
 		int id_words;
 
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 87eff2fe9f..49198fb47f 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -18,8 +18,6 @@
  * %CopyrightEnd%
  */
 
-/* Same order as the ordering of terms in erlang */
-
 /* Since there are 255 different External tag values to choose from
    There is no reason to not be extravagant.
    Hence, the different tags for large/small tuple e.t.c
@@ -37,9 +35,12 @@
 #define SMALL_ATOM_EXT    's'
 #define REFERENCE_EXT     'e'
 #define NEW_REFERENCE_EXT 'r'
+#define NEWER_REFERENCE_EXT 'Z'
 #define PORT_EXT          'f'
+#define NEW_PORT_EXT      'Y'
 #define NEW_FLOAT_EXT     'F'
 #define PID_EXT           'g'
+#define NEW_PID_EXT       'X'
 #define SMALL_TUPLE_EXT   'h'
 #define LARGE_TUPLE_EXT   'i'
 #define NIL_EXT           'j'
-- 
cgit v1.2.3


From e4b8b9be39601b1c79dfd5d8e807ae4c404b6935 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 7 Apr 2016 20:38:55 +0200
Subject: erts: Add DFLAG_BIG_CREATION

to let future nodes know that we can handle
NEW_PID_EXT, NEW_PORT_EXT and NEWER_REFERENCE_EXT.
---
 erts/emulator/beam/dist.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index fb777d9ac1..e3ff6ebad1 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -43,6 +43,7 @@
 #define DFLAG_INTERNAL_TAGS       0x8000
 #define DFLAG_UTF8_ATOMS          0x10000
 #define DFLAG_MAP_TAG             0x20000
+#define DFLAG_BIG_CREATION        0x40000
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
@@ -51,7 +52,8 @@
 			       | DFLAG_EXTENDED_PIDS_PORTS	\
 			       | DFLAG_EXPORT_PTR_TAG		\
 			       | DFLAG_BIT_BINARIES             \
-			       | DFLAG_MAP_TAG)
+			       | DFLAG_MAP_TAG                  \
+                               | DFLAG_BIG_CREATION)
 
 /* opcodes used in distribution messages */
 #define DOP_LINK		1
-- 
cgit v1.2.3


From 3c0f98c1b0e74eb58a4e4e82a3ec89dc7ecdb579 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 7 Apr 2016 10:06:44 +0200
Subject: Simplify the raise instruction to reduce code size

The raise/2 instruction is almost always used like this:

  raise x(2) x(1)

Therefore, we can translate it to an internal i_raise/0
instruction that uses x(2) x(1) as its implicit operands.

We will also remove the backward compatibility with R10-0.  It is
unlikely that anyone still is using BEAM files compiled with the R10-0
compiler, especially since most of those modules cannot be loaded. The
loader will refuse to load any module that uses the old non-GCIng
arithmetic instructions or the non-GCing versions of length/1 or
size/1.

Doing these changes will reduce both the size of the loaded BEAM
code and size of the code in process_main().
---
 erts/emulator/beam/beam_emu.c | 54 ++++++++++---------------------------------
 erts/emulator/beam/ops.tab    |  9 +++++++-
 2 files changed, 20 insertions(+), 43 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 09a41f2b56..d648a2f23c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3360,48 +3360,18 @@ do {						\
      goto do_schedule;
  }
 
- OpCase(raise_ss): {
-     /* This was not done very well in R10-0; then, we passed the tag in
-	the first argument and hoped that the existing c_p->ftrace was
-	still correct. But the ftrace-object already includes the tag
-	(or rather, the freason). Now, we pass the original ftrace in
-	the first argument. We also handle atom tags in the first
-	argument for backwards compatibility.
-     */
-     Eterm raise_val1;
-     Eterm raise_val2;
-     GetArg2(0, raise_val1, raise_val2);
-     c_p->fvalue = raise_val2;
-     if (c_p->freason == EXC_NULL) {
-       /* a safety check for the R10-0 case; should not happen */
-       c_p->ftrace = NIL;
-       c_p->freason = EXC_ERROR;
-     }
-     /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */
-     switch (raise_val1) {
-     case am_throw:
-       c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
-       break;
-     case am_error:
-       c_p->freason = EXC_ERROR & ~EXF_SAVETRACE;
-       break;
-     case am_exit:
-       c_p->freason = EXC_EXIT & ~EXF_SAVETRACE;
-       break;
-     default:
-       {/* R10-1 and later
-	   XXX note: should do sanity check on given trace if it can be
-	   passed from a user! Currently only expecting generated calls.
-	*/
-	 struct StackTrace *s;
-	 c_p->ftrace = raise_val1;
-	 s = get_trace_from_exc(raise_val1);
-	 if (s == NULL) {
-	   c_p->freason = EXC_ERROR;
-	 } else {
-	   c_p->freason = PRIMARY_EXCEPTION(s->freason);
-	 }
-       }
+ OpCase(i_raise): {
+     Eterm raise_trace = x(2);
+     Eterm raise_value = x(1);
+     struct StackTrace *s;
+
+     c_p->fvalue = raise_value;
+     c_p->ftrace = raise_trace;
+     s = get_trace_from_exc(raise_trace);
+     if (s == NULL) {
+	 c_p->freason = EXC_ERROR;
+     } else {
+	 c_p->freason = PRIMARY_EXCEPTION(s->freason);
      }
      goto find_func_info;
  }
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 15f27835a8..96a3a72bb5 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -251,7 +251,14 @@ case_end x
 badmatch x
 
 if_end
-raise s s
+
+# Operands for raise/2 are almost always in x(2) and x(1).
+# Optimize for that case.
+raise x==2 x==1 => i_raise
+raise Trace=y Value=y => move Trace x=2 | move Value x=1 | i_raise
+raise Trace Value => move Trace x=3 | move Value x=1 | move x=3 x=2 | i_raise
+
+i_raise
 
 # Internal now, but could be useful to make known to the compiler.
 badarg j
-- 
cgit v1.2.3


From c56c6d36c3060e62925f036ad020da1753e48829 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Sat, 9 Apr 2016 00:54:40 +0200
Subject: erts: Optimize '++' operator

This also optimizes the BIF lists:append/2

Use one pass to check for properness and copying LHS list.
If LHS turns out not being a proper list, bail and reset htop.
If we run out of heap, allocate a heap-fragment and calculate
the remaining length as normal, thus checking for properness,
and then continue copying.

Measurements shows this being ~50% faster.
---
 erts/emulator/beam/erl_bif_lists.c | 93 +++++++++++++++++++++++++++++++-------
 1 file changed, 77 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index fe64e76575..647a218851 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -40,32 +40,93 @@ static BIF_RETTYPE append(Process* p, Eterm A, Eterm B)
     Eterm list;
     Eterm copy;
     Eterm last;
-    size_t need;
-    Eterm* hp;
+    Eterm* hp = NULL;
     Sint i;
 
-    if ((i = erts_list_length(A)) < 0) {
-	BIF_ERROR(p, BADARG);
+    list = A;
+
+    if (is_nil(list)) {
+        BIF_RET(B);
     }
-    if (i == 0) {
-	BIF_RET(B);
-    } else if (is_nil(B)) {
-	BIF_RET(A);
+
+    if (is_not_list(list)) {
+        BIF_ERROR(p, BADARG);
     }
 
-    need = 2*i;
-    hp = HAlloc(p, need);
-    list = A;
+    /* optimistic append on heap first */
+
+    if ((i = HeapWordsLeft(p) / 2) < 4) {
+        goto list_tail;
+    }
+
+    hp   = HEAP_TOP(p);
     copy = last = CONS(hp, CAR(list_val(list)), make_list(hp+2));
     list = CDR(list_val(list));
-    hp += 2;
+    hp  += 2;
+    i   -= 2; /* don't use the last 2 words (extra i--;) */
+
+    while(i-- && is_list(list)) {
+        Eterm* listp = list_val(list);
+        last = CONS(hp, CAR(listp), make_list(hp+2));
+        list = CDR(listp);
+        hp += 2;
+    }
+
+    /* A is proper and B is NIL return A as-is, don't update HTOP */
+
+    if (is_nil(list) && is_nil(B)) {
+        BIF_RET(A);
+    }
+
+    if (is_nil(list)) {
+        HEAP_TOP(p) = hp;
+        CDR(list_val(last)) = B;
+        BIF_RET(copy);
+    }
+
+list_tail:
+
+    if ((i = erts_list_length(list)) < 0) {
+        BIF_ERROR(p, BADARG);
+    }
+
+    /* remaining list was proper and B is NIL */
+    if (is_nil(B)) {
+        BIF_RET(A);
+    }
+
+    if (hp) {
+        /* Note: fall through case, already written
+         * on the heap.
+         * The last 2 words of the heap is not written yet
+         */
+        Eterm *hp_save = hp;
+        ASSERT(i != 0);
+        HEAP_TOP(p) = hp + 2;
+        if (i == 1) {
+            hp[0] = CAR(list_val(list));
+            hp[1] = B;
+            BIF_RET(copy);
+        }
+        hp   = HAlloc(p, 2*(i - 1));
+        last = CONS(hp_save, CAR(list_val(list)), make_list(hp));
+    } else {
+        hp   = HAlloc(p, 2*i);
+        copy = last = CONS(hp, CAR(list_val(list)), make_list(hp+2));
+        hp  += 2;
+    }
+
+    list = CDR(list_val(list));
     i--;
+
+    ASSERT(i > -1);
     while(i--) {
-	Eterm* listp = list_val(list);
-	last = CONS(hp, CAR(listp), make_list(hp+2));
-	list = CDR(listp);
-	hp += 2;
+        Eterm* listp = list_val(list);
+        last = CONS(hp, CAR(listp), make_list(hp+2));
+        list = CDR(listp);
+        hp  += 2;
     }
+
     CDR(list_val(last)) = B;
     BIF_RET(copy);
 }
-- 
cgit v1.2.3


From 97cdb6bd87f38b06ee477dbb17d4d2f7854b21e1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 13 Apr 2016 15:18:13 +0200
Subject: erts: Fix bug in open_port with {args,ImproperList}

---
 erts/emulator/beam/erl_bif_port.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 27c24197ea..e4d4559f48 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -953,6 +953,8 @@ static char **convert_args(Eterm l)
     }
 
     n = erts_list_length(l);
+    if (n < 0)
+        return NULL;
     /* We require at least one element in argv[0] + NULL at end */
     pp = erts_alloc(ERTS_ALC_T_TMP, (n + 2) * sizeof(char **));
     pp[i++] = erts_default_arg0;
-- 
cgit v1.2.3


From 350c2beb2ed583aa6b0345e8118615bd3c5ccc9d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 13 Apr 2016 20:02:21 +0200
Subject: erts: Fix decoding of pids, ports and refs with big creations

Must skip 3 extra bytes after node name atom.
---
 erts/emulator/beam/external.c | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 5ea155f83f..243ba8d9f3 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -4425,22 +4425,32 @@ init_done:
 	    SKIP(1+atom_extra_skip);
 	    atom_extra_skip = 0;
 	    break;
-        case PID_EXT:
         case NEW_PID_EXT:
+	    atom_extra_skip = 12;
+	    goto case_PID;
+        case PID_EXT:
 	    atom_extra_skip = 9;
+	case_PID:
 	    /* In case it is an external pid */
 	    heap_size += EXTERNAL_THING_HEAD_SIZE + 1;
 	    terms++;
 	    break;
-        case PORT_EXT:
         case NEW_PORT_EXT:
+	    atom_extra_skip = 8;
+	    goto case_PORT;
+        case PORT_EXT:
 	    atom_extra_skip = 5;
+	case_PORT:
 	    /* In case it is an external port */
 	    heap_size += EXTERNAL_THING_HEAD_SIZE + 1;
 	    terms++;
 	    break;
-        case NEW_REFERENCE_EXT:
 	case NEWER_REFERENCE_EXT:
+	    atom_extra_skip = 4;
+	    goto case_NEW_REFERENCE;
+        case NEW_REFERENCE_EXT:
+	    atom_extra_skip = 1;
+	case_NEW_REFERENCE:
 	    {
 		int id_words;
 
@@ -4451,7 +4461,7 @@ init_done:
 		    goto error;
 
 		ep += 2;
-		atom_extra_skip = 1 + 4*id_words;
+		atom_extra_skip += 4*id_words;
 		/* In case it is an external ref */
 #if defined(ARCH_64)
 		heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1;
-- 
cgit v1.2.3


From 21041ee50217d90dc21dd484b9dd26c469e8cf87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 13 Apr 2016 15:07:52 +0200
Subject: beam_debug: Correct masking when unpacking packed operands

---
 erts/emulator/beam/beam_debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index e37bd4d78c..7fddf15dab 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -431,7 +431,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 		packed >>= 10;
 		break;
 	    case '0':		/* Tight shift */
-		*ap++ = packed & (BEAM_TIGHT_MASK / sizeof(Eterm));
+		*ap++ = packed & BEAM_TIGHT_MASK;
 		packed >>= BEAM_TIGHT_SHIFT;
 		break;
 	    case '6':		/* Shift 16 steps */
-- 
cgit v1.2.3


From 9b2ee6cfdc09090df81be40ca1d7358c3d273743 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 9 Apr 2016 09:51:10 +0200
Subject: Optimize get_tuple_element instructions that target Y registers

Several improvements in the compiler (e.g. c288ab87fd6) has
lead to an Y register being the target for get_tuple_element
instructions. Therefore, introduce i_get_tuple_element2y
that combines two consecutive get_tuple_element instructions
that target Y registers.
---
 erts/emulator/beam/beam_emu.c | 11 +++++++++++
 erts/emulator/beam/ops.tab    |  6 ++++++
 2 files changed, 17 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index d648a2f23c..ce92f60fd2 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -702,6 +702,17 @@ void** beam_ops;
     dst[1] = E2;					\
   } while (0)
 
+#define GetTupleElement2Y(Src, Element, D1, D2)		\
+  do {							\
+    Eterm* src;						\
+    Eterm E1, E2;					\
+    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
+    E1 = src[0];					\
+    E2 = src[1];					\
+    D1 = E1;						\
+    D2 = E2;						\
+  } while (0)
+
 #define GetTupleElement3(Src, Element, Dest)		\
   do {							\
     Eterm* src;						\
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 96a3a72bb5..aa720c5b0c 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -229,6 +229,9 @@ i_get_tuple_element y P y
 %macro: i_get_tuple_element2 GetTupleElement2 -pack
 i_get_tuple_element2 x P x
 
+%macro: i_get_tuple_element2y GetTupleElement2Y -pack
+i_get_tuple_element2y x P y y
+
 %macro: i_get_tuple_element3 GetTupleElement3 -pack
 i_get_tuple_element3 x P x
 
@@ -649,6 +652,9 @@ get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \
 get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \
    succ(P1, P2) | succ(D1, D2) => i_get_tuple_element2 Reg P1 D1
 
+get_tuple_element Reg=x P1 D1=y | get_tuple_element Reg=x P2 D2=y | \
+   succ(P1, P2) => i_get_tuple_element2y Reg P1 D1 D2
+
 get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst
 
 is_integer Fail=f i =>
-- 
cgit v1.2.3


From c74f998b9ad46785aaa2557a4033c056574dc937 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 12 Apr 2016 06:08:07 +0200
Subject: v3_kernel: Construct literal lists properly

Use cerl:make_list/1 instead of a home-made make_list/1 to ensure that
literal lists are constructed as literals. In a future release, we
would like to forbid in the loader construction of literal lists using
instructions like:

   put_list {atom,a} [] Dst

The proper way is:

   move {literal,[a]} {x,0}

Also update the comment about "put_list Const [] Dst" in ops.tab.
---
 erts/emulator/beam/ops.tab | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 96a3a72bb5..83b0299ddd 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -529,9 +529,9 @@ i_put_tuple x I
 i_put_tuple y I
 
 #
-# The instruction "put_list Const [] Dst" will not be generated by
-# the current BEAM compiler. But until R15A, play it safe by handling
-# that instruction with the following transformation.
+# The instruction "put_list Const [] Dst" were generated in rare
+# circumstances up to and including OTP 18. Starting with OTP 19,
+# AFAIK, it should never be generated.
 #
 put_list Const=c n Dst => move Const x | put_list x n Dst
 
-- 
cgit v1.2.3


From 3e6f813222057941515df9cdb1f5d89ac3dd9cf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 14 Apr 2016 15:35:57 +0200
Subject: erts: Don't use function location when process is terminating

---
 erts/emulator/beam/erl_trace.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index fb3f0d4e62..1654ea58d9 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3020,6 +3020,7 @@ profile_runnable_proc(Process *p, Eterm status){
     Eterm *hp, msg;
     Eterm where = am_undefined;
     ErlHeapFragment *bp = NULL;
+    int use_current = 1;
 
 #ifndef ERTS_SMP
 #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
@@ -3032,12 +3033,19 @@ profile_runnable_proc(Process *p, Eterm status){
     Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1;
 #endif
 	
-    if (!p->current) {
-        p->current = find_function_from_pc(p->i);
+    if (ERTS_PROC_IS_EXITING(p)) {
+        use_current = 0;
+        /* could probably set 'where' to 'exiting' here,
+         * though it's not documented as such */
+    } else {
+        if (!p->current) {
+            p->current = find_function_from_pc(p->i);
+        }
+        use_current = p->current != NULL;
     }
 
 #ifdef ERTS_SMP
-    if (!p->current) {
+    if (!use_current) {
 	hsz -= 4;
     }
 
@@ -3045,7 +3053,7 @@ profile_runnable_proc(Process *p, Eterm status){
     hp = bp->mem;
 #endif
 
-    if (p->current) {
+    if (use_current) {
 	where = TUPLE3(hp, p->current[0], p->current[1], make_small(p->current[2])); hp += 4;
     } else {
 	where = make_small(0);
-- 
cgit v1.2.3


From fa2f841dc17da2704c2307b6f04aaee85de2366d Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 10 Dec 2015 11:27:50 +0100
Subject: erts: Calculate flatmap value offset

---
 erts/emulator/beam/erl_map.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index a40070d00d..7af9100906 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -66,7 +66,7 @@ typedef struct flatmap_s {
 
 
 /* erl_term.h stuff */
-#define flatmap_get_values(x)        (((Eterm *)(x)) + 3)
+#define flatmap_get_values(x)        (((Eterm *)(x)) + sizeof(flatmap_t)/sizeof(Eterm))
 #define flatmap_get_keys(x)          (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1)
 #define flatmap_get_size(x)          (((flatmap_t*)(x))->size)
 
-- 
cgit v1.2.3


From 13bd4ca2493d8a76f9d835c27163b56ba86c2aef Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 10 Dec 2015 11:28:56 +0100
Subject: erts: Add erts_nif_get_funcs and erts_nif_call_function

These are convinience functions for calling nifs from erts
---
 erts/emulator/beam/erl_nif.c | 45 ++++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_nif.h |  2 +-
 2 files changed, 46 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1f5793f534..61dba4ef82 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2960,6 +2960,51 @@ void erl_nif_init()
     resource_type_list.name = THE_NON_VALUE;
 }
 
+int erts_nif_get_funcs(struct erl_module_nif* mod,
+                       ErlNifFunc **funcs)
+{
+    *funcs = mod->entry->funcs;
+    return mod->entry->num_of_funcs;
+}
+
+Eterm erts_nif_call_function(Process *p, struct erl_module_nif* mod,
+                             ErlNifFunc *fun, int argc, Eterm *argv)
+{
+    Eterm nif_result;
+#ifdef DEBUG
+    /* Verify that function is part of this module */
+    int i;
+    for (i = 0; i < mod->entry->num_of_funcs; i++)
+        if (fun == mod->entry->funcs+i)
+            break;
+    ASSERT(i < mod->entry->num_of_funcs);
+    if (p)
+        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN);
+#endif
+    if (p) {
+        struct enif_environment_t env;
+        ASSERT(is_internal_pid(p->common.id));
+        erts_pre_nif(&env, p, mod);
+        env.may_delay_enif_send = 1;
+        nif_result = (*fun->fptr)(&env, argc, argv);
+        if (env.exception_thrown)
+            nif_result = THE_NON_VALUE;
+        erts_post_nif(&env);
+    } else {
+        /* Nif call was done without a process context,
+           so we create a phony one. */
+        struct enif_msg_environment_t msg_env;
+        setup_nif_env(&msg_env, mod);
+        msg_env.env.may_delay_enif_send = 1;
+        nif_result = (*fun->fptr)(&msg_env.env, argc, argv);
+        if (msg_env.env.exception_thrown)
+            nif_result = THE_NON_VALUE;
+        enif_clear_env(&msg_env.env);
+    }
+
+    return nif_result;
+}
+
 #ifdef USE_VM_PROBES
 void dtrace_nifenv_str(ErlNifEnv *env, char *process_buf)
 {
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 44a2581436..33785ecaca 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -96,7 +96,7 @@ typedef enum {
 struct enif_environment_t;
 typedef struct enif_environment_t ErlNifEnv;
 
-typedef struct
+typedef struct enif_func_t
 {
     const char* name;
     unsigned arity;
-- 
cgit v1.2.3


From 37092dab15448ef6a078800e3ff0cc41880ea765 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 10 Dec 2015 11:10:46 +0100
Subject: erts: Implement tracer modules

Add the possibility to use modules as trace data receivers. The functions
in the module have to be nifs as otherwise complex trace probes will be
very hard to handle (complex means trace probes for ports for example).

This commit changes the way that the ptab->tracer field works from always
being an immediate, to now be NIL if no tracer is present or else be
the tuple {TracerModule, TracerState} where TracerModule is an atom that
is later used to lookup the appropriate tracer callbacks to call and
TracerState is just passed to the tracer callback. The default process and
port tracers have been rewritten to use the new API.

This commit also changes the order which trace messages are delivered to the
potential tracer process. Any enif_send done in a tracer module may be delayed
indefinitely because of lock order issues. If a message is delayed any other
trace message send from that process is also delayed so that order is preserved
for each traced entity. This means that for some trace events (i.e. send/receive)
the events may come in an unintuitive order (receive before send) to the
trace receiver. Timestamps are taken when the trace message is generated so
trace messages from differented processes may arrive with the timestamp
out of order.

Both the erlang:trace and seq_trace:set_system_tracer accept the new tracer
module tracers and also the backwards compatible arguments.

OTP-10267
---
 erts/emulator/beam/atom.names         |    7 +-
 erts/emulator/beam/beam_bif_load.c    |    7 +-
 erts/emulator/beam/beam_bp.c          |  169 ++-
 erts/emulator/beam/beam_bp.h          |   16 +-
 erts/emulator/beam/beam_emu.c         |   10 +-
 erts/emulator/beam/bif.c              |   66 +-
 erts/emulator/beam/bif.tab            |    5 +-
 erts/emulator/beam/code_ix.c          |    1 +
 erts/emulator/beam/dist.c             |   16 +-
 erts/emulator/beam/erl_alloc.c        |    2 +-
 erts/emulator/beam/erl_alloc.types    |    3 +
 erts/emulator/beam/erl_bif_ddll.c     |    2 +-
 erts/emulator/beam/erl_bif_info.c     |    5 +-
 erts/emulator/beam/erl_bif_port.c     |    7 +-
 erts/emulator/beam/erl_bif_trace.c    |  543 +++----
 erts/emulator/beam/erl_db_util.c      |   74 +-
 erts/emulator/beam/erl_gc.c           |    6 +-
 erts/emulator/beam/erl_hl_timer.c     |    6 +-
 erts/emulator/beam/erl_lock_check.c   |    5 +-
 erts/emulator/beam/erl_message.c      |  160 ++-
 erts/emulator/beam/erl_message.h      |  110 +-
 erts/emulator/beam/erl_msacc.c        |    2 +-
 erts/emulator/beam/erl_nif.c          |  252 +++-
 erts/emulator/beam/erl_nif.h          |    8 +-
 erts/emulator/beam/erl_process.c      |  216 ++-
 erts/emulator/beam/erl_process.h      |    8 +-
 erts/emulator/beam/erl_process_lock.c |  191 ++-
 erts/emulator/beam/erl_process_lock.h |   32 +-
 erts/emulator/beam/erl_ptab.h         |    8 +-
 erts/emulator/beam/erl_time_sup.c     |    2 +-
 erts/emulator/beam/erl_trace.c        | 2540 +++++++++++----------------------
 erts/emulator/beam/erl_trace.h        |   78 +-
 erts/emulator/beam/erlang_dtrace.d    |   29 +
 erts/emulator/beam/global.h           |   26 +-
 erts/emulator/beam/io.c               |   34 +-
 erts/emulator/beam/register.c         |    8 +-
 erts/emulator/beam/sys.h              |    9 +
 erts/emulator/beam/utils.c            |    2 +-
 38 files changed, 2115 insertions(+), 2550 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index d466f00028..aeea6cc989 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -197,6 +197,7 @@ atom dirty_cpu_schedulers_online
 atom dirty_io
 atom disable_trace
 atom disabled
+atom discard
 atom display_items
 atom dist
 atom dist_cmd
@@ -231,6 +232,7 @@ atom exception_trace
 atom extended
 atom Eq='=:='
 atom Eqeq='=='
+atom erl_tracer
 atom erlang
 atom ERROR='ERROR'
 atom error_handler
@@ -353,6 +355,7 @@ atom match
 atom match_limit
 atom match_limit_recursion
 atom match_spec
+atom match_spec_result
 atom max
 atom maximum
 atom max_tables max_processes
@@ -569,6 +572,7 @@ atom static
 atom stderr_to_stdout
 atom stop
 atom stream
+atom strict_monotonic
 atom strict_monotonic_timestamp
 atom sunrm
 atom suspend
@@ -596,8 +600,9 @@ atom total_active_tasks
 atom total_heap_size
 atom total_run_queue_lengths
 atom tpkt
-atom trace trace_ts traced 
+atom trace trace_ts traced
 atom trace_control_word
+atom trace_status
 atom tracer
 atom trap_exit
 atom trim
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 320dee6668..b10250dc49 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -719,13 +719,13 @@ set_default_trace_pattern(Eterm module)
     Binary *match_spec;
     Binary *meta_match_spec;
     struct trace_pattern_flags trace_pattern_flags;
-    Eterm meta_tracer_pid;
+    ErtsTracer meta_tracer;
 
     erts_get_default_trace_pattern(&trace_pattern_is_on,
 				   &match_spec,
 				   &meta_match_spec,
 				   &trace_pattern_flags,
-				   &meta_tracer_pid);
+				   &meta_tracer);
     if (trace_pattern_is_on) {
 	Eterm mfa[1];
 	mfa[0] = module;
@@ -733,7 +733,7 @@ set_default_trace_pattern(Eterm module)
 				      match_spec,
 				      meta_match_spec,
 				      1, trace_pattern_flags,
-				      meta_tracer_pid, 1);
+				      meta_tracer, 1);
     }
 }
 
@@ -1264,6 +1264,7 @@ delete_code(Module* modp)
     modp->curr.code_length = 0;
     modp->curr.catches = BEAM_CATCHES_NIL;
     modp->curr.nif = NULL;
+
 }
 
 
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 0a13454951..2ee98ed7b5 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -92,15 +92,15 @@ get_mtime(Process *c_p)
 /*
 ** Helpers
 */
-static Eterm do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
-			   int local, Binary* ms, Eterm tracer_pid);
+static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
+                                int local, Binary* ms, ErtsTracer tracer);
 static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags,
-		      enum erts_break_op count_op, Eterm tracer_pid);
+		      enum erts_break_op count_op, ErtsTracer tracer);
 static void set_function_break(BeamInstr *pc,
 			       Binary *match_spec,
 			       Uint break_flags,
 			       enum erts_break_op count_op,
-			       Eterm tracer_pid);
+			       ErtsTracer tracer);
 
 static void clear_break(BpFunctions* f, Uint break_flags);
 static int clear_function_break(BeamInstr *pc, Uint break_flags);
@@ -108,7 +108,7 @@ static int clear_function_break(BeamInstr *pc, Uint break_flags);
 static BpDataTime* get_time_break(BeamInstr *pc);
 static GenericBpData* check_break(BeamInstr *pc, Uint break_flags);
 
-static void bp_meta_unref(BpMetaPid* bmp);
+static void bp_meta_unref(BpMetaTracer* bmt);
 static void bp_count_unref(BpCount* bcp);
 static void bp_time_unref(BpDataTime* bdt);
 static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local);
@@ -302,7 +302,7 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local)
 	MatchSetUnref(dst->local_ms);
     }
     if (flags & ERTS_BPF_META_TRACE) {
-	bp_meta_unref(dst->meta_pid);
+	bp_meta_unref(dst->meta_tracer);
 	MatchSetUnref(dst->meta_ms);
     }
     if (flags & ERTS_BPF_COUNT) {
@@ -343,8 +343,8 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local)
 	MatchSetRef(dst->local_ms);
     }
     if (flags & ERTS_BPF_META_TRACE) {
-	dst->meta_pid = src->meta_pid;
-	erts_refc_inc(&dst->meta_pid->refc, 1);
+	dst->meta_tracer = src->meta_tracer;
+	erts_refc_inc(&dst->meta_tracer->refc, 1);
 	dst->meta_ms = src->meta_ms;
 	MatchSetRef(dst->meta_ms);
     }
@@ -436,13 +436,13 @@ uninstall_breakpoint(BeamInstr* pc)
 void
 erts_set_trace_break(BpFunctions* f, Binary *match_spec)
 {
-    set_break(f, match_spec, ERTS_BPF_LOCAL_TRACE, 0, am_true);
+    set_break(f, match_spec, ERTS_BPF_LOCAL_TRACE, 0, erts_tracer_true);
 }
 
 void
-erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, Eterm tracer_pid)
+erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer)
 {
-    set_break(f, match_spec, ERTS_BPF_META_TRACE, 0, tracer_pid);
+    set_break(f, match_spec, ERTS_BPF_META_TRACE, 0, tracer);
 }
 
 void
@@ -450,13 +450,13 @@ erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local)
 {
     Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE;
 
-    set_function_break(pc, match_spec, flags, 0, NIL);
+    set_function_break(pc, match_spec, flags, 0, erts_tracer_nil);
 }
 
 void
-erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid)
+erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, ErtsTracer tracer)
 {
-    set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer_pid);
+    set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer);
 }
 
 void
@@ -464,7 +464,7 @@ erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op)
 {
     set_function_break(pc, NULL,
 		       ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE,
-		       count_op, NIL);
+		       count_op, erts_tracer_nil);
 }
 
 void
@@ -474,21 +474,21 @@ erts_clear_time_trace_bif(BeamInstr *pc) {
 
 void
 erts_set_debug_break(BpFunctions* f) {
-    set_break(f, NULL, ERTS_BPF_DEBUG, 0, NIL);
+    set_break(f, NULL, ERTS_BPF_DEBUG, 0, erts_tracer_nil);
 }
 
 void
 erts_set_count_break(BpFunctions* f, enum erts_break_op count_op)
 {
     set_break(f, 0, ERTS_BPF_COUNT|ERTS_BPF_COUNT_ACTIVE,
-	      count_op, NIL);
+	      count_op, erts_tracer_nil);
 }
 
 void
 erts_set_time_break(BpFunctions* f, enum erts_break_op count_op)
 {
     set_break(f, 0, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE,
-	      count_op, NIL);
+	      count_op, erts_tracer_nil);
 }
 
 void
@@ -625,19 +625,26 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 
     if (bp_flags & ERTS_BPF_LOCAL_TRACE) {
 	ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0);
-	(void) do_call_trace(c_p, I, reg, 1, bp->local_ms, am_true);
+	(void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true);
     } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) {
-	(void) do_call_trace(c_p, I, reg, 0, bp->local_ms, am_true);
+	(void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true);
     }
 
     if (bp_flags & ERTS_BPF_META_TRACE) {
-	Eterm old_pid;
-	Eterm new_pid;
-
-	old_pid = (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid);
-	new_pid = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_pid);
-	if (new_pid != old_pid) {
-	    erts_smp_atomic_set_nob(&bp->meta_pid->pid, new_pid);
+	ErtsTracer old_tracer, new_tracer;
+
+	old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
+
+	new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer);
+	if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) {
+            if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
+                    &bp->meta_tracer->tracer,
+                    (erts_aint_t)new_tracer,
+                    (erts_aint_t)old_tracer)) {
+                ERTS_TRACER_CLEAR(&old_tracer);
+            } else {
+                ERTS_TRACER_CLEAR(&new_tracer);
+            }
 	}
     }
 
@@ -645,7 +652,8 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 	erts_smp_atomic_inc_nob(&bp->count->acount);
     }
 
-    if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && erts_is_tracer_proc_valid(c_p)) {
+    if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE
+        && ERTS_TRACER_PROC_IS_ENABLED(c_p)) {
 	Eterm w;
 	erts_trace_time_call(c_p, I, bp->time);
 	w = (BeamInstr) *c_p->cp;
@@ -690,7 +698,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     Eterm (*func)(Process*, Eterm*, BeamInstr*);
     Export* ep = bif_export[bif_index];
     Uint32 flags = 0, flags_meta = 0;
-    Eterm meta_tracer_pid = NIL;
+    ErtsTracer meta_tracer = erts_tracer_nil;
     int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif
 					   * is actually in the
 					   * export entry */
@@ -718,23 +726,32 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	IS_TRACED_FL(p, F_TRACE_CALLS)) {
 	int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE);
 	flags = erts_call_trace(p, ep->code, bp->local_ms, args,
-				local, &ERTS_TRACER_PROC(p));
+				local, &ERTS_TRACER(p));
     }
     if (bp_flags & ERTS_BPF_META_TRACE) {
-	Eterm tpid1, tpid2;
+	ErtsTracer old_tracer;
 
-	tpid1 = tpid2 =
-	    (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid);
+        meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
+        old_tracer = meta_tracer;
 	flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args,
-				     0, &tpid2);
-	meta_tracer_pid = tpid2;
-	if (tpid1 != tpid2) {
-	    erts_smp_atomic_set_nob(&bp->meta_pid->pid, tpid2);
+				     0, &meta_tracer);
+
+	if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) {
+            ErtsTracer new_tracer = erts_tracer_nil;
+            erts_tracer_update(&new_tracer, meta_tracer);
+	    if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
+                    &bp->meta_tracer->tracer,
+                    (erts_aint_t)new_tracer,
+                    (erts_aint_t)old_tracer)) {
+                ERTS_TRACER_CLEAR(&old_tracer);
+            } else {
+                ERTS_TRACER_CLEAR(&new_tracer);
+            }
 	}
     }
     if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE &&
 	IS_TRACED_FL(p, F_TRACE_CALLS) &&
-	erts_is_tracer_proc_valid(p)) {
+	ERTS_TRACER_PROC_IS_ENABLED(p)) {
 	BeamInstr *pc = (BeamInstr *)ep->code+3;
 	erts_trace_time_call(p, pc, bp->time);
     }
@@ -778,8 +795,6 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	if (reason != TRAP) {
 	    Eterm class;
 	    Eterm value = p->fvalue;
-	    DeclareTmpHeapNoproc(nocatch,3);
-	    UseTmpHeapNoproc(3);
 	    /* Expand error value like in handle_error() */
 	    if (reason & EXF_ARGLIST) {
 		Eterm *tp;
@@ -788,7 +803,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 		value = tp[1];
 	    }
 	    if ((reason & EXF_THROWN) && (p->catches <= 0)) {
-		value = TUPLE2(nocatch, am_nocatch, value);
+                Eterm *hp = HAlloc(p, 3);
+		value = TUPLE2(hp, am_nocatch, value);
 		reason = EXC_ERROR;
 	    }
 	    /* Note: expand_error_value() could theoretically
@@ -801,11 +817,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 
 	    if (flags_meta & MATCH_SET_EXCEPTION_TRACE) {
 		erts_trace_exception(p, ep->code, class, value,
-				     &meta_tracer_pid);
+				     &meta_tracer);
 	    }
 	    if (flags & MATCH_SET_EXCEPTION_TRACE) {
 		erts_trace_exception(p, ep->code, class, value,
-				     &ERTS_TRACER_PROC(p));
+				     &ERTS_TRACER(p));
 	    }
 	    if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) {
 		/* can only happen if(local)*/
@@ -827,7 +843,6 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 		    }
 		}
 	    }
-	    UnUseTmpHeapNoproc(3);
 	    if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) {
 		erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 		ERTS_TRACE_FLAGS(p) |= F_EXCEPTION_TRACE;
@@ -836,11 +851,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	}
     } else {
 	if (flags_meta & MATCH_SET_RX_TRACE) {
-	    erts_trace_return(p, ep->code, result, &meta_tracer_pid);
+	    erts_trace_return(p, ep->code, result, &meta_tracer);
 	}
 	/* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */
 	if (flags & MATCH_SET_RX_TRACE) {
-	    erts_trace_return(p, ep->code, result, &ERTS_TRACER_PROC(p));
+	    erts_trace_return(p, ep->code, result, &ERTS_TRACER(p));
 	}
 	if (flags & MATCH_SET_RETURN_TO_TRACE) {
 	    /* can only happen if(local)*/
@@ -857,9 +872,9 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     return result;
 }
 
-static Eterm
+static ErtsTracer
 do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
-	      int local, Binary* ms, Eterm tracer_pid)
+	      int local, Binary* ms, ErtsTracer tracer)
 {
     Eterm* cpp;
     int return_to_trace = 0;
@@ -899,7 +914,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 	ASSERT(is_CP(*cpp));
     }
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-    flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer_pid);
+    flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     if (cpp) {
 	c_p->cp = cp_save;
@@ -910,7 +925,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 	need += 1;
     }
     if (flags & MATCH_SET_RX_TRACE) {
-	need += 3;
+	need += 3 + size_object(tracer);
     }
     if (need) {
 	ASSERT(c_p->htop <= E && E <= c_p->hend);
@@ -926,14 +941,15 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 	E[0] = make_cp(c_p->cp);
 	c_p->cp = beam_return_to_trace;
     }
-    if (flags & MATCH_SET_RX_TRACE) {
+    if (flags & MATCH_SET_RX_TRACE)
+    {
 	E -= 3;
+        c_p->stop = E;
 	ASSERT(c_p->htop <= E && E <= c_p->hend);
 	ASSERT(is_CP((Eterm) (UWord) (I - 3)));
-	ASSERT(am_true == tracer_pid ||
-	       is_internal_pid(tracer_pid) || is_internal_port(tracer_pid));
+	ASSERT(IS_TRACER_VALID(tracer));
 	E[2] = make_cp(c_p->cp);
-	E[1] = tracer_pid;
+        E[1] = copy_object(tracer, c_p);
 	E[0] = make_cp(I - 3); /* We ARE at the beginning of an
 				  instruction,
 				  the funcinfo is above i. */
@@ -942,9 +958,9 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	ERTS_TRACE_FLAGS(c_p) |= F_EXCEPTION_TRACE;
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
-    }
-    c_p->stop = E;
-    return tracer_pid;
+    } else
+        c_p->stop = E;
+    return tracer;
 }
 
 void
@@ -1098,9 +1114,9 @@ erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local)
     return 0;
 }
 
-int 
+int
 erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
-		     Eterm *tracer_pid_ret)
+		     ErtsTracer *tracer_ret)
 {
     GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE);
     
@@ -1108,9 +1124,8 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
 	if (match_spec_ret) {
 	    *match_spec_ret = bp->meta_ms;
 	}
-	if (tracer_pid_ret) {
-	    *tracer_pid_ret =
-		(Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid);
+	if (tracer_ret) {
+            *tracer_ret = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
 	}
 	return 1;
     }
@@ -1391,7 +1406,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 
 static void
 set_break(BpFunctions* f, Binary *match_spec, Uint break_flags,
-	  enum erts_break_op count_op, Eterm tracer_pid)
+	  enum erts_break_op count_op, ErtsTracer tracer)
 {
     Uint i;
     Uint n;
@@ -1400,13 +1415,13 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags,
     for (i = 0; i < n; i++) {
 	BeamInstr* pc = f->matching[i].pc;
 	set_function_break(pc, match_spec, break_flags,
-			   count_op, tracer_pid);
+			   count_op, tracer);
     }
 }
 
 static void
 set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
-		   enum erts_break_op count_op, Eterm tracer_pid)
+		   enum erts_break_op count_op, ErtsTracer tracer)
 {
     GenericBp* g;
     GenericBpData* bp;
@@ -1439,7 +1454,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	MatchSetUnref(bp->local_ms);
     } else if (common & ERTS_BPF_META_TRACE) {
 	MatchSetUnref(bp->meta_ms);
-	bp_meta_unref(bp->meta_pid);
+	bp_meta_unref(bp->meta_tracer);
     } else if (common & ERTS_BPF_COUNT) {
 	if (count_op == erts_break_stop) {
 	    bp->flags &= ~ERTS_BPF_COUNT_ACTIVE;
@@ -1474,13 +1489,15 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	MatchSetRef(match_spec);
 	bp->local_ms = match_spec;
     } else if (break_flags & ERTS_BPF_META_TRACE) {
-	BpMetaPid* bmp;
+	BpMetaTracer* bmt;
+        ErtsTracer meta_tracer = erts_tracer_nil;
 	MatchSetRef(match_spec);
 	bp->meta_ms = match_spec;
-	bmp = Alloc(sizeof(BpMetaPid));
-	erts_refc_init(&bmp->refc, 1);
-	erts_smp_atomic_init_nob(&bmp->pid, tracer_pid);
-	bp->meta_pid = bmp;
+	bmt = Alloc(sizeof(BpMetaTracer));
+	erts_refc_init(&bmt->refc, 1);
+        erts_tracer_update(&meta_tracer, tracer); /* copy tracer */
+	erts_smp_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer);
+	bp->meta_tracer = bmt;
     } else if (break_flags & ERTS_BPF_COUNT) {
 	BpCount* bcp;
 
@@ -1544,7 +1561,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags)
     }
     if (common & ERTS_BPF_META_TRACE) {
 	MatchSetUnref(bp->meta_ms);
-	bp_meta_unref(bp->meta_pid);
+	bp_meta_unref(bp->meta_tracer);
     }
     if (common & ERTS_BPF_COUNT) {
 	ASSERT((bp->flags & ERTS_BPF_COUNT_ACTIVE) == 0);
@@ -1560,10 +1577,12 @@ clear_function_break(BeamInstr *pc, Uint break_flags)
 }
 
 static void
-bp_meta_unref(BpMetaPid* bmp)
+bp_meta_unref(BpMetaTracer* bmt)
 {
-    if (erts_refc_dectest(&bmp->refc, 0) <= 0) {
-	Free(bmp);
+    if (erts_refc_dectest(&bmt->refc, 0) <= 0) {
+        ErtsTracer trc = erts_smp_atomic_read_nob(&bmt->tracer);
+        ERTS_TRACER_CLEAR(&trc);
+	Free(bmt);
     }
 }
 
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index f9eca94e2a..bb171be8a6 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -55,15 +55,15 @@ typedef struct {
 } BpCount;
 
 typedef struct {
-    erts_smp_atomic_t pid;
+    erts_smp_atomic_t tracer;
     erts_refc_t refc;
-} BpMetaPid;
+} BpMetaTracer;
 
 typedef struct generic_bp_data {
     Uint flags;
     Binary* local_ms;		/* Match spec for local call trace */
     Binary* meta_ms;		/* Match spec for meta trace */
-    BpMetaPid* meta_pid;	/* Meta trace pid */
+    BpMetaTracer* meta_tracer;	/* Meta tracer */
     BpCount* count;		/* For call count */
     BpDataTime* time;		/* For time trace */
 } GenericBpData;
@@ -132,10 +132,10 @@ void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local);
 void erts_clear_call_trace_bif(BeamInstr *pc, int local);
 
 void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec,
-			  Eterm tracer_pid);
+			  ErtsTracer tracer);
 void erts_clear_mtrace_break(BpFunctions *f);
 void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec,
-			 Eterm tracer_pid);
+			 ErtsTracer tracer);
 void erts_clear_mtrace_bif(BeamInstr *pc);
 
 void erts_set_debug_break(BpFunctions *f);
@@ -150,13 +150,13 @@ void erts_clear_export_break(Module *modp, BeamInstr* pc);
 
 BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg);
 BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args,
-		      Uint32 *ret_flags, Eterm *tracer_pid);
+                           Uint32 *ret_flags, ErtsTracer *tracer);
 
 int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local);
 int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
-			 Eterm *tracer_pid_rte);
+			 ErtsTracer *tracer_ret);
 int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret,
-		       Eterm *tracer_pid_ret);
+		       ErtsTracer *tracer_ret);
 int erts_is_native_break(BeamInstr *pc);
 int erts_is_count_break(BeamInstr *pc, Uint *count_ret);
 int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time);
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 901419c989..c356863b60 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4588,7 +4588,7 @@ do {						\
      
      SWAPOUT;		/* Needed for shared heap */
      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-     erts_trace_return(c_p, code, r(0), E+1/*Process tracer*/);
+     erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */);
      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
      SWAPIN;
      c_p->cp = NULL;
@@ -5228,7 +5228,8 @@ next_catch(Process* c_p, Eterm *reg) {
 	BeamInstr *cpp = c_p->cp;
 	if (cpp == beam_exception_trace) {
 	    erts_trace_exception(c_p, cp_val(ptr[0]),
-				 reg[1], reg[2], ptr+1);
+				 reg[1], reg[2],
+                                 ERTS_TRACER_FROM_ETERM(ptr+1));
 	    /* Skip return_trace parameters */
 	    ptr += 2;
 	} else if (cpp == beam_return_trace) {
@@ -5255,7 +5256,8 @@ next_catch(Process* c_p, Eterm *reg) {
 		}
 		if (cp_val(*prev) == beam_exception_trace) {
 		    erts_trace_exception(c_p, cp_val(ptr[0]),
-					 reg[1], reg[2], ptr+1);
+					 reg[1], reg[2],
+                                         ERTS_TRACER_FROM_ETERM(ptr+1));
 		}
 		/* Skip return_trace parameters */
 		ptr += 2;
@@ -6049,7 +6051,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
 	    return -1;
 	}
 #else /* ERTS_SMP */
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+        ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (!c_p->msg.len)
 #endif
 	    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 754e11f047..d633a982ca 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -123,15 +123,12 @@ static int insert_internal_link(Process* p, Eterm rpid)
 	erts_add_link(&ERTS_P_LINKS(p), LINK_PID, rp->common.id);
 	erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, p->common.id);
 
-	ASSERT(is_nil(ERTS_TRACER_PROC(p))
-	       || is_internal_pid(ERTS_TRACER_PROC(p))
-	       || is_internal_port(ERTS_TRACER_PROC(p)));
+	ASSERT(IS_TRACER_VALID(ERTS_TRACER(p)));
 
 	if (IS_TRACED(p)) {
 	    if (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1))  {
 		ERTS_TRACE_FLAGS(rp) |= (ERTS_TRACE_FLAGS(p) & TRACEE_FLAGS);
-		ERTS_TRACER_PROC(rp) = ERTS_TRACER_PROC(p); /* maybe steal */
-
+                erts_tracer_replace(&rp->common, ERTS_TRACER(p));
 		if (ERTS_TRACE_FLAGS(p) & F_TRACE_SOL1) { /* maybe override */
 		    ERTS_TRACE_FLAGS(rp) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
 		    ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
@@ -140,7 +137,8 @@ static int insert_internal_link(Process* p, Eterm rpid)
 	}
     }
     if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-	trace_proc(p, rp, am_getting_linked, p->common.id);
+	trace_proc(p, p == rp ? rp_locks : ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
+                   rp, am_getting_linked, p->common.id);
 
     if (p == rp)
 	erts_smp_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN);
@@ -159,7 +157,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
     DistEntry *dep;
 
     if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) {
-	trace_proc(BIF_P, BIF_P, am_link, BIF_ARG_1);
+	trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_P, am_link, BIF_ARG_1);
     }
     /* check that the pid or port which is our argument is OK */
 
@@ -613,7 +611,7 @@ erts_queue_monitor_message(Process *p,
     ref_copy    = copy_struct(ref, ref_size, &hp, ohp);
 
     tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy);
-    erts_queue_message(p, p_locksp, msgp, tup, NIL);
+    erts_queue_message(p, p_locksp, msgp, tup);
 }
 
 static BIF_RETTYPE
@@ -1001,6 +999,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
     Process *rp;
     DistEntry *dep;
     ErtsLink *l = NULL, *rl = NULL;
+    ErtsProcLocks cp_locks = ERTS_PROC_LOCK_MAIN;
 
     /*
      * SMP specific note concerning incoming exit signals:
@@ -1015,7 +1014,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
      */
 
     if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) {
-	trace_proc(BIF_P, BIF_P, am_unlink, BIF_ARG_1);
+	trace_proc(BIF_P, cp_locks, BIF_P, am_unlink, BIF_ARG_1);
     }
 
     if (is_internal_port(BIF_ARG_1)) {
@@ -1120,10 +1119,10 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
 
+    cp_locks |= ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS;
+
     /* get process struct */
-    rp = erts_pid2proc_opt(BIF_P, (ERTS_PROC_LOCK_MAIN
-				   | ERTS_PROC_LOCK_LINK
-				   | ERTS_PROC_LOCK_STATUS),
+    rp = erts_pid2proc_opt(BIF_P, cp_locks,
 			   BIF_ARG_1, ERTS_PROC_LOCK_LINK,
 			   ERTS_P2P_FLG_ALLOW_OTHER_X);
 
@@ -1149,14 +1148,17 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	    erts_destroy_link(rl);
 
 	if (IS_TRACED_FL(rp, F_TRACE_PROCS) && rl != NULL) {
-	    trace_proc(BIF_P, rp, am_getting_unlinked, BIF_P->common.id);
+            erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
+            cp_locks &= ~ERTS_PROC_LOCK_STATUS;
+	    trace_proc(BIF_P, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
+                       rp, am_getting_unlinked, BIF_P->common.id);
 	}
 
 	if (rp != BIF_P)
 	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     }
  
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+    erts_smp_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN);
 
     BIF_RET(am_true);
 
@@ -4337,12 +4339,34 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
     } else if (BIF_ARG_1 == am_trace_control_word) {
 	BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_2));
     } else if (BIF_ARG_1 == am_sequential_tracer) {
-        Eterm old_value = erts_set_system_seq_tracer(BIF_P,
-						     ERTS_PROC_LOCK_MAIN,
-						     BIF_ARG_2);
-	if (old_value != THE_NON_VALUE) {
-	    BIF_RET(old_value);
-	}
+        ErtsTracer new_seq_tracer, old_seq_tracer;
+        Eterm ret;
+
+        if (BIF_ARG_2 == am_false)
+            new_seq_tracer = erts_tracer_nil;
+        else
+            new_seq_tracer = erts_term_to_tracer(THE_NON_VALUE, BIF_ARG_2);
+
+        if (new_seq_tracer == THE_NON_VALUE)
+            goto error;
+
+        old_seq_tracer = erts_set_system_seq_tracer(BIF_P,
+                                                    ERTS_PROC_LOCK_MAIN,
+                                                    new_seq_tracer);
+
+        ERTS_TRACER_CLEAR(&new_seq_tracer);
+
+        if (old_seq_tracer == THE_NON_VALUE)
+            goto error;
+
+        if (ERTS_TRACER_IS_NIL(old_seq_tracer))
+            BIF_RET(am_false);
+
+        ret = erts_tracer_to_term(BIF_P, old_seq_tracer);
+
+        ERTS_TRACER_CLEAR(&old_seq_tracer);
+
+        BIF_RET(ret);
     } else if (BIF_ARG_1 == make_small(1)) {
 	int i, max;
 	ErtsMessage* mp;
@@ -4679,7 +4703,7 @@ skip_current_msgq(Process *c_p)
 	res = 0;
     }
     else {
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+        ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	c_p->msg.save = c_p->msg.last;
 	res = 1;
     }
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 9bb77190d6..58cd31cee9 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -181,9 +181,8 @@ bif erlang:port_set_data/2
 bif erlang:port_get_data/1
 
 # Tracing & debugging.
-bif erlang:trace_pattern/2
-bif erlang:trace_pattern/3
-bif erlang:trace/3
+bif erts_internal:trace_pattern/3
+bif erts_internal:trace/3
 bif erlang:trace_info/2
 bif erlang:trace_delivered/1
 bif erlang:seq_trace/2
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 9da750e366..ec6267711b 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -94,6 +94,7 @@ void erts_commit_staging_code_ix(void)
     ix = (ix + 1) % ERTS_NUM_CODE_IX;
     erts_smp_atomic32_set_nob(&the_staging_code_index, ix);
     export_staging_unlock();
+    erts_tracer_nif_clear();
     CIX_TRACE("activate");
 }
 
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 88449adb8e..8cd3f5fbe6 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -337,7 +337,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp)
 	    erts_destroy_link(rlnk);
 	    if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
 		/* We didn't exit the process and it is traced */
-		trace_proc(NULL, rp, am_getting_unlinked, sublnk->pid);
+		trace_proc(NULL, 0, rp, am_getting_unlinked, sublnk->pid);
 	    }
 	}
 	erts_smp_proc_unlock(rp, rp_locks);
@@ -397,7 +397,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	    msgp = erts_alloc_message_heap(rp, &rp_locks,
 					   3, &hp, &ohp);
 	    tup = TUPLE2(hp, am_nodedown, name);
-	    erts_queue_message(rp, &rp_locks, msgp, tup, NIL);
+	    erts_queue_message(rp, &rp_locks, msgp, tup);
 	}
 	erts_smp_proc_unlock(rp, rp_locks);
     }
@@ -1275,7 +1275,7 @@ int erts_net_message(Port *prt,
 	erts_smp_de_links_unlock(dep);
 
 	if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-	    trace_proc(NULL, rp, am_getting_linked, from);
+	    trace_proc(NULL, 0, rp, am_getting_linked, from);
 
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	break;
@@ -1300,7 +1300,7 @@ int erts_net_message(Port *prt,
 	lnk = erts_remove_link(&ERTS_P_LINKS(rp), from);
 
 	if (IS_TRACED_FL(rp, F_TRACE_PROCS) && lnk != NULL) {
-	    trace_proc(NULL, rp, am_getting_unlinked, from);
+	    trace_proc(NULL, 0, rp, am_getting_unlinked, from);
 	}
 
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
@@ -1628,7 +1628,11 @@ int erts_net_message(Port *prt,
 					     ERTS_XSIG_FLG_IGN_KILL);
 		if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
 		    /* We didn't exit the process and it is traced */
-		    trace_proc(NULL, rp, am_getting_unlinked, from);
+                    if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
+                        erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
+                        rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
+                    }
+		    trace_proc(NULL, 0, rp, am_getting_unlinked, from);
 		}
 	    }
 	    erts_smp_proc_unlock(rp, rp_locks);
@@ -3313,7 +3317,7 @@ send_nodes_mon_msg(Process *rp,
     }
 
     ASSERT(hend == hp);
-    erts_queue_message(rp, rp_locksp, mp, msg, NIL);
+    erts_queue_message(rp, rp_locksp, mp, msg);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index cfe0bc3205..a6519bd9e4 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3210,7 +3210,7 @@ reply_alloc_info(void *vair)
     if (hp != hp_end)
 	erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1);
 
-    erts_queue_message(rp, &rp_locks, mp, msg, NIL);
+    erts_queue_message(rp, &rp_locks, mp, msg);
 
     if (air->req_sched == sched_id)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 9482ab9265..d2fe440d47 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -270,6 +270,9 @@ type	PROC_SYS_TSK	SHORT_LIVED	PROCESSES	proc_sys_task
 type	PROC_SYS_TSK_QS	SHORT_LIVED	PROCESSES	proc_sys_task_queues
 type    NEW_TIME_OFFSET	SHORT_LIVED	SYSTEM		new_time_offset
 type	IOB_REQ		SHORT_LIVED	SYSTEM		io_bytes_request
+type    TRACER_NIF      LONG_LIVED      SYSTEM          tracer_nif
+type    TRACE_MSG_QUEUE SHORT_LIVED     SYSTEM          trace_message_queue
+type    SCHED_ASYNC_JOB SHORT_LIVED     SYSTEM          async_calls
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index a79ce11563..1e2db38442 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1737,7 +1737,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	hp += REF_THING_SIZE;
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
     }
-    erts_queue_message(proc, &rp_locks, mp, mess, am_undefined);
+    erts_queue_message(proc, &rp_locks, mp, mess);
     erts_smp_proc_unlock(proc, rp_locks);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 }
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 68f6abfcdc..7a72b0d8cc 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -846,6 +846,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
 
 	if (unlock_locks)
 	    erts_smp_proc_unlock(rp, unlock_locks);
+
     }
 
     /*
@@ -2162,8 +2163,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	res = build_snifs_term(&hp, NULL, NIL);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_sequential_tracer) {
-	val = erts_get_system_seq_tracer();
-	ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false);
+	ErtsTracer seq_tracer = erts_get_system_seq_tracer();
+        val = erts_tracer_to_term(BIF_P, seq_tracer);
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, am_sequential_tracer, val);
 	BIF_RET(res);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 71b6d78094..eb55bd585e 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -79,6 +79,7 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
     }
 
     if (port->drv_ptr->flags & ERL_DRV_FLAG_USE_INIT_ACK) {
+
         /* Copied from erl_port_task.c */
         port->async_open_port = erts_alloc(ERTS_ALC_T_PRTSD,
                                            sizeof(*port->async_open_port));
@@ -911,7 +912,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
     }
     
     if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
-        trace_virtual_sched(p, am_out);
+        trace_sched(p, ERTS_PROC_LOCK_MAIN, am_out);
     }
     
 
@@ -935,13 +936,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 		err_typep ? *err_typep : 4711,
 		err_nump ? *err_nump : 4711));
     	if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
-            trace_virtual_sched(p, am_in);
+            trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
     	}
 	goto do_return;
     }
     
     if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
-        trace_virtual_sched(p, am_in);
+        trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
     }
 
     if (linebuf && port->linebuf == NULL){
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index a9443ee8df..049899211f 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -52,7 +52,7 @@ static int                         erts_default_trace_pattern_is_on;
 static Binary                     *erts_default_match_spec;
 static Binary                     *erts_default_meta_match_spec;
 static struct trace_pattern_flags  erts_default_trace_pattern_flags;
-static Eterm                       erts_default_meta_tracer_pid;
+static ErtsTracer                  erts_default_meta_tracer;
 
 static struct {			/* Protected by code write permission */
     int current;
@@ -75,8 +75,6 @@ static BIF_RETTYPE
 system_monitor(Process *p, Eterm monitor_pid, Eterm list);
 
 static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/
-static int already_traced(Process *p, Process *tracee_p, Eterm tracer);
-static int port_already_traced(Process *p, Port *tracee_port, Eterm tracer);
 static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key);
 static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key);
 static Eterm trace_info_on_load(Process* p, Eterm key);
@@ -94,21 +92,15 @@ erts_bif_trace_init(void)
     erts_default_match_spec = NULL;
     erts_default_meta_match_spec = NULL;
     erts_default_trace_pattern_flags = erts_trace_pattern_flags_off;
-    erts_default_meta_tracer_pid = NIL;
+    erts_default_meta_tracer = erts_tracer_nil;
 }
 
 /*
  * Turn on/off call tracing for the given function(s).
  */
-  
-Eterm
-trace_pattern_2(BIF_ALIST_2)
-{
-    return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL);
-}
 
 Eterm
-trace_pattern_3(BIF_ALIST_3)
+erts_internal_trace_pattern_3(BIF_ALIST_3)
 {
     return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 }
@@ -125,11 +117,10 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     Eterm l;
     struct trace_pattern_flags flags = erts_trace_pattern_flags_off;
     int is_global;
-    Process *meta_tracer_proc = p;
-    Eterm meta_tracer_pid = p->common.id;
+    ErtsTracer meta_tracer = erts_tracer_nil;
 
     if (!erts_try_seize_code_write_permission(p)) {
-	ERTS_BIF_YIELD3(bif_export[BIF_trace_pattern_3], p, MFA, Pattern, flaglist);
+	ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_pattern_3], p, MFA, Pattern, flaglist);
     }
     finish_bp.current = -1;
 
@@ -160,31 +151,11 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     is_global = 0;
     for(l = flaglist; is_list(l); l = CDR(list_val(l))) {
 	if (is_tuple(CAR(list_val(l)))) {
-	    Eterm *tp = tuple_val(CAR(list_val(l)));
-	    
-	    if (arityval(tp[0]) != 2 || tp[1] != am_meta) {
-		goto error;
-	    }
-	    meta_tracer_pid = tp[2];
-	    if (is_internal_pid(meta_tracer_pid)) {
-		meta_tracer_proc = erts_pid2proc(NULL, 0, meta_tracer_pid, 0);
-		if (!meta_tracer_proc) {
-		    goto error;
-		}
-	    } else if (is_internal_port(meta_tracer_pid)) {
-		Port *meta_tracer_port;
-		meta_tracer_proc = NULL;
-		meta_tracer_port = (erts_port_lookup(
-					meta_tracer_pid,
-					ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP));
-		if (!meta_tracer_port)
-		    goto error;
-	    } else {
-		goto error;
-	    }
-	    if (is_global) {
-		goto error;
-	    }
+            meta_tracer = erts_term_to_tracer(am_meta, CAR(list_val(l)));
+            if (meta_tracer == THE_NON_VALUE) {
+                meta_tracer = erts_tracer_nil;
+                goto error;
+            }
 	    flags.breakpoint = 1;
 	    flags.meta       = 1;
 	} else {
@@ -202,6 +173,8 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		}
 		flags.breakpoint = 1;
 		flags.meta       = 1;
+                if (ERTS_TRACER_IS_NIL(meta_tracer))
+                    meta_tracer = erts_term_to_tracer(THE_NON_VALUE, p->common.id);
 		break;
 	    case am_global:
 		if (flags.breakpoint) {
@@ -252,14 +225,11 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	    MatchSetUnref(erts_default_meta_match_spec);
 	    erts_default_meta_match_spec = match_prog_set;
 	    MatchSetRef(erts_default_meta_match_spec);
-	    erts_default_meta_tracer_pid = meta_tracer_pid;
-	    if (meta_tracer_proc) {
-		ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER;
-	    }
+            erts_tracer_update(&erts_default_meta_tracer, meta_tracer);
 	} else if (! flags.breakpoint) {
 	    MatchSetUnref(erts_default_meta_match_spec);
 	    erts_default_meta_match_spec = NULL;
-	    erts_default_meta_tracer_pid = NIL;
+	    ERTS_TRACER_CLEAR(&erts_default_meta_tracer);
 	}
 	if (erts_default_trace_pattern_flags.breakpoint &&
 	    flags.breakpoint) { 
@@ -340,20 +310,18 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	if (is_small(mfa[2])) {
 	    mfa[2] = signed_val(mfa[2]);
 	}
-    
-	if (meta_tracer_proc) {
-	    ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER;
-	}
 
 	matches = erts_set_trace_pattern(p, mfa, specified,
 					 match_prog_set, match_prog_set,
-					 on, flags, meta_tracer_pid, 0);
+					 on, flags, meta_tracer, 0);
     }
 
  error:
     MatchSetUnref(match_prog_set);
     UnUseTmpHeap(3,p);
 
+    ERTS_TRACER_CLEAR(&meta_tracer);
+
 #ifdef ERTS_SMP
     if (finish_bp.current >= 0) {
 	ASSERT(matches >= 0);
@@ -404,7 +372,7 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 			       Binary **match_spec,
 			       Binary **meta_match_spec,
 			       struct trace_pattern_flags *trace_pattern_flags,
-			       Eterm *meta_tracer_pid)
+			       ErtsTracer *meta_tracer)
 {
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() ||
 		       erts_smp_thr_progress_is_blocking());
@@ -416,8 +384,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 	*meta_match_spec = erts_default_meta_match_spec;
     if (trace_pattern_flags)
 	*trace_pattern_flags = erts_default_trace_pattern_flags;
-    if (meta_tracer_pid)
-	*meta_tracer_pid = erts_default_meta_tracer_pid;
+    if (meta_tracer)
+	*meta_tracer = erts_default_meta_tracer;
 }
 
 int erts_is_default_trace_enabled(void)
@@ -465,12 +433,12 @@ erts_trace_flag2bit(Eterm flag)
 ** occurred in the argument list.
 */
 int
-erts_trace_flags(Eterm List, 
-		 Uint *pMask, Eterm *pTracer, int *pCpuTimestamp)
+erts_trace_flags(Eterm List,
+                 Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp)
 {
     Eterm list = List;
     Uint mask = 0;
-    Eterm tracer = NIL;
+    ErtsTracer tracer = erts_tracer_nil;
     int cpu_timestamp = 0;
     
     while (is_list(list)) {
@@ -483,33 +451,72 @@ erts_trace_flags(Eterm List,
 	    cpu_timestamp = !0;
 #endif
 	} else if (is_tuple(item)) {
-	    Eterm* tp = tuple_val(item);
-	    
-	    if (arityval(tp[0]) != 2 || tp[1] != am_tracer) goto error;
-	    if (is_internal_pid(tp[2]) || is_internal_port(tp[2])) {
-		tracer = tp[2];
-	    } else goto error;
+            tracer = erts_term_to_tracer(am_tracer, item);
+            if (tracer == THE_NON_VALUE)
+                goto error;
 	} else goto error;
 	list = CDR(list_val(list));
     }
     if (is_not_nil(list)) goto error;
     
-    if (pMask && mask)                  *pMask         = mask;
-    if (pTracer && tracer != NIL)       *pTracer       = tracer;
-    if (pCpuTimestamp && cpu_timestamp) *pCpuTimestamp = cpu_timestamp;
+    if (pMask && mask)                           *pMask         = mask;
+    if (pTracer && !ERTS_TRACER_IS_NIL(tracer))  *pTracer       = tracer;
+    if (pCpuTimestamp && cpu_timestamp)          *pCpuTimestamp = cpu_timestamp;
     return !0;
  error:
     return 0;
 }
 
-Eterm trace_3(BIF_ALIST_3)
+static ERTS_INLINE int
+start_trace(Process *c_p, ErtsTracer tracer,
+            ErtsPTabElementCommon *common,
+            int on, int mask)
+{
+    /* We can use the common part of both port+proc without checking what it is
+       In the code below port is used for both proc and port */
+    Port *port = (Port*)common;
+
+    /*
+     * SMP build assumes that either system is blocked or:
+     * * main lock is held on c_p
+     * * all locks are held on port common
+     */
+
+    if (!ERTS_TRACER_IS_NIL(tracer)) {
+        if ((ERTS_TRACE_FLAGS(port) & TRACEE_FLAGS)
+            && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) {
+            /* This tracee is already being traced, and not by the
+             * tracer to be */
+            if (erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCKS_ALL, common)) {
+                /* The tracer is still in use */
+                return 1;
+            }
+            /* Current tracer now invalid */
+        }
+    }
+
+    if (on)
+        ERTS_TRACE_FLAGS(port) |= mask;
+    else
+        ERTS_TRACE_FLAGS(port) &= ~mask;
+
+    if ((ERTS_TRACE_FLAGS(port) & TRACEE_FLAGS) == 0) {
+        tracer = erts_tracer_nil;
+        erts_tracer_replace(common, erts_tracer_nil);
+    } else if (!ERTS_TRACER_IS_NIL(tracer))
+        erts_tracer_replace(common, tracer);
+
+    return 0;
+}
+
+Eterm erts_internal_trace_3(BIF_ALIST_3)
 {
     Process* p = BIF_P;
     Eterm pid_spec = BIF_ARG_1;
     Eterm how = BIF_ARG_2;
     Eterm list = BIF_ARG_3;
     int on;
-    Eterm tracer = NIL;
+    ErtsTracer tracer = erts_tracer_nil;
     int matches = 0;
     Uint mask = 0;
     int cpu_ts = 0;
@@ -522,41 +529,24 @@ Eterm trace_3(BIF_ALIST_3)
     }
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
-	ERTS_BIF_YIELD3(bif_export[BIF_trace_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+	ERTS_BIF_YIELD3(bif_export[BIF_erts_internal_trace_3],
+                        BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
     }
 
-    if (is_nil(tracer) || is_internal_pid(tracer)) {
-	Process *tracer_proc = erts_pid2proc(p,
-					     ERTS_PROC_LOCK_MAIN,
-					     is_nil(tracer) ? p->common.id : tracer,
-					     ERTS_PROC_LOCKS_ALL);
-	if (!tracer_proc)
-	    goto error;
-	ERTS_TRACE_FLAGS(tracer_proc) |= F_TRACER;
-	erts_smp_proc_unlock(tracer_proc,
-			     (tracer_proc == p
-			      ? ERTS_PROC_LOCKS_ALL_MINOR
-			      : ERTS_PROC_LOCKS_ALL));
-    } else if (is_internal_port(tracer)) {
-	Port *tracer_port = erts_id2port_sflgs(tracer,
-					       p,
-					       ERTS_PROC_LOCK_MAIN,
-					       ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
-	if (!tracer_port)
-	    goto error;
-	ERTS_TRACE_FLAGS(tracer_port) |= F_TRACER;
-	erts_port_release(tracer_port);
-    } else
-	goto error;
-
     switch (how) {
     case am_false: 
 	on = 0; 
 	break;
     case am_true: 
 	on = 1;
-	if (is_nil(tracer))
-	    tracer = p->common.id;
+        if (ERTS_TRACER_IS_NIL(tracer))
+            tracer = erts_term_to_tracer(am_tracer, p->common.id);
+
+        if (tracer == THE_NON_VALUE) {
+            tracer = erts_tracer_nil;
+            goto error;
+        }
+
 	break;
     default: 
 	goto error;
@@ -575,34 +565,20 @@ Eterm trace_3(BIF_ALIST_3)
 	}
 #endif
 
-	if (pid_spec == tracer)
-	    goto error;
-
 	tracee_port = erts_id2port_sflgs(pid_spec,
 					 p,
 					 ERTS_PROC_LOCK_MAIN,
 					 ERTS_PORT_SFLGS_INVALID_LOOKUP);
+
 	if (!tracee_port)
 	    goto error;
-	
-	if (tracer != NIL && port_already_traced(p, tracee_port, tracer)) {
+
+        if (start_trace(p, tracer, &tracee_port->common, on, mask)) {
 	    erts_port_release(tracee_port);
 	    goto already_traced;
-	}
-
-	if (on)
-	    ERTS_TRACE_FLAGS(tracee_port) |= mask;
-	else
-	    ERTS_TRACE_FLAGS(tracee_port) &= ~mask;
-
-	if (!ERTS_TRACE_FLAGS(tracee_port))
-	    ERTS_TRACER_PROC(tracee_port) = NIL;
-	else if (tracer != NIL)
-	    ERTS_TRACER_PROC(tracee_port) = tracer;
-
-	erts_port_release(tracee_port);
-
-	matches = 1;
+        }
+        erts_port_release(tracee_port);
+        matches = 1;
     } else if (is_pid(pid_spec)) {
 	Process *tracee_p;
 
@@ -615,33 +591,19 @@ Eterm trace_3(BIF_ALIST_3)
 	 * and not about to be tracing.
 	 */
 
-	if (pid_spec == tracer)
-	    goto error;
-
 	tracee_p = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
 				 pid_spec, ERTS_PROC_LOCKS_ALL);
 	if (!tracee_p)
 	    goto error;
 
-	if (tracer != NIL && already_traced(p, tracee_p, tracer)) {
+        if (start_trace(tracee_p, tracer, &tracee_p->common, on, mask)) {
 	    erts_smp_proc_unlock(tracee_p,
 				 (tracee_p == p
 				  ? ERTS_PROC_LOCKS_ALL_MINOR
 				  : ERTS_PROC_LOCKS_ALL));
 	    goto already_traced;
-	}
-
-	if (on)
-	    ERTS_TRACE_FLAGS(tracee_p) |= mask;
-	else
-	    ERTS_TRACE_FLAGS(tracee_p) &= ~mask;
-
-	if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS) == 0)
-	    ERTS_TRACER_PROC(tracee_p) = NIL;
-	else if (tracer != NIL)
-	    ERTS_TRACER_PROC(tracee_p) = tracer;
-
-	erts_smp_proc_unlock(tracee_p,
+        }
+        erts_smp_proc_unlock(tracee_p,
 			     (tracee_p == p
 			      ? ERTS_PROC_LOCKS_ALL_MINOR
 			      : ERTS_PROC_LOCKS_ALL));
@@ -719,22 +681,7 @@ Eterm trace_3(BIF_ALIST_3)
 		    Process* tracee_p = erts_pix2proc(i);
 		    if (! tracee_p) 
 			continue;
-		    if (tracer != NIL) {
-			if (tracee_p->common.id == tracer)
-			    continue;
-			if (already_traced(NULL, tracee_p, tracer))
-			    continue;
-		    }
-		    if (on) {
-			ERTS_TRACE_FLAGS(tracee_p) |= mask;
-		    } else {
-			ERTS_TRACE_FLAGS(tracee_p) &= ~mask;
-		    }
-		    if(!(ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS)) {
-			ERTS_TRACER_PROC(tracee_p) = NIL;
-		    } else if (tracer != NIL) {
-			ERTS_TRACER_PROC(tracee_p) = tracer;
-		    }
+                    start_trace(p, tracer, &tracee_p->common, on, mask);
 		    matches++;
 		}
 	    }
@@ -749,21 +696,7 @@ Eterm trace_3(BIF_ALIST_3)
 		    state = erts_atomic32_read_nob(&tracee_port->state);
 		    if (state & ERTS_PORT_SFLGS_DEAD)
 			continue;
-		    if (tracer != NIL) {
-			if (tracee_port->common.id == tracer)
-			    continue;
-			if (port_already_traced(NULL, tracee_port, tracer))
-			    continue;
-		    }
-
-		    if (on) ERTS_TRACE_FLAGS(tracee_port) |= mask;
-		    else ERTS_TRACE_FLAGS(tracee_port) &= ~mask;
-		
-		    if (!(ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS)) {
-			ERTS_TRACER_PROC(tracee_port) = NIL;
-		    } else if (tracer != NIL) {
-			ERTS_TRACER_PROC(tracee_port) = tracer;
-		    }
+                    start_trace(p, tracer, &tracee_port->common, on, mask);
 		    /* matches are not counted for ports since it would violate compatibility */
 		    /* This could be a reason to modify this function or make a new one. */
 		}
@@ -772,10 +705,9 @@ Eterm trace_3(BIF_ALIST_3)
 
 	if (pid_spec == am_all || pid_spec == am_new) {
 	    Uint def_flags = mask;
-	    Eterm def_tracer = tracer;
 
 	    ok = 1;
-	    erts_change_default_tracing(on, &def_flags, &def_tracer);
+	    erts_change_default_tracing(on, &def_flags, tracer);
 
 #ifdef HAVE_ERTS_NOW_CPU
 	    if (cpu_ts && !on) {
@@ -801,6 +733,7 @@ Eterm trace_3(BIF_ALIST_3)
     }
 #endif
     erts_release_code_write_permission();
+    ERTS_TRACER_CLEAR(&tracer);
 
     BIF_RET(make_small(matches));
 
@@ -810,6 +743,8 @@ Eterm trace_3(BIF_ALIST_3)
 
  error:
 
+    ERTS_TRACER_CLEAR(&tracer);
+
 #ifdef ERTS_SMP
     if (system_blocked) {
 	erts_smp_thr_progress_unblock();
@@ -821,88 +756,6 @@ Eterm trace_3(BIF_ALIST_3)
     BIF_ERROR(p, BADARG);
 }
 
-/* Check that the process to be traced is not already traced
- * by a valid other tracer than the tracer to be.
- */
-static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer)
-{
-    /*
-     * SMP build assumes that either system is blocked or:
-     * * main lock is held on c_p
-     * * all locks are held on port tracee_p
-     */
-    if ((ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS)
-	&& ERTS_TRACER_PROC(tracee_port) != tracer) {
-	/* This tracee is already being traced, and not by the 
-	 * tracer to be */
-	if (is_internal_port(ERTS_TRACER_PROC(tracee_port))) {
-	    if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_port))) {
-		/* Current trace port now invalid 
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else if(is_internal_pid(ERTS_TRACER_PROC(tracee_port))) {
-	    Process *tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_port));
-	    if (!tracer_p) {
-		/* Current trace process now invalid
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else {
-	remove_tracer:
-	    ERTS_TRACE_FLAGS(tracee_port) &= ~TRACEE_FLAGS;
-	    ERTS_TRACER_PROC(tracee_port) = NIL;
-	}
-    }
-    return 0;
-}
-
-/* Check that the process to be traced is not already traced
- * by a valid other tracer than the tracer to be.
- */
-static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer)
-{
-    /*
-     * SMP build assumes that either system is blocked or:
-     * * main lock is held on c_p
-     * * all locks multiple are held on tracee_p
-     */
-    if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS)
-	&& ERTS_TRACER_PROC(tracee_p) != tracer) {
-	/* This tracee is already being traced, and not by the 
-	 * tracer to be */
-	if (is_internal_port(ERTS_TRACER_PROC(tracee_p))) {
-	    if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_p))) {
-		/* Current trace port now invalid 
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else if(is_internal_pid(ERTS_TRACER_PROC(tracee_p))) {
-	    Process *tracer_p;
-
-	    tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_p));
-	    if (!tracer_p) {
-		/* Current trace process now invalid
-		 * - discard it and approve the new. */
-		goto remove_tracer;
-	    } else
-		return 1;
-	}
-	else {
-	remove_tracer:
-	    ERTS_TRACE_FLAGS(tracee_p) &= ~TRACEE_FLAGS;
-	    ERTS_TRACER_PROC(tracee_p) = NIL;
-	}
-    }
-    return 0;
-}
-
 /*
  * Return information about a process or an external function being traced.
  */
@@ -936,41 +789,30 @@ static Eterm
 trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 {
     Eterm tracer;
-    Uint trace_flags;
+    Uint trace_flags = am_false;
     Eterm* hp;
 
     if (pid_spec == am_new) {
-	erts_get_default_tracing(&trace_flags, &tracer);
+        ErtsTracer def_tracer;
+	erts_get_default_tracing(&trace_flags, &def_tracer);
+        tracer = erts_tracer_to_term(p, def_tracer);
+        ERTS_TRACER_CLEAR(&def_tracer);
     } else if (is_internal_pid(pid_spec)) {
 	Process *tracee;
 	tracee = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
-			       pid_spec, ERTS_PROC_LOCKS_ALL);
+			       pid_spec, ERTS_PROC_LOCK_MAIN);
 
-	if (!tracee) {
+	if (!tracee)
 	    return am_undefined;
-	} else {
-	    tracer = ERTS_TRACER_PROC(tracee);
-	    trace_flags = ERTS_TRACE_FLAGS(tracee);
-	}
 
-	if (is_internal_pid(tracer)) {
-	    if (!erts_proc_lookup(tracer)) {
-	    reset_tracer:
-		ERTS_TRACE_FLAGS(tracee) &= ~TRACEE_FLAGS;
-		trace_flags = ERTS_TRACE_FLAGS(tracee);
-		tracer = ERTS_TRACER_PROC(tracee) = NIL;
-	    }
-	}
-	else if (is_internal_port(tracer)) {
-	    if (!erts_is_valid_tracer_port(tracer))
-		goto reset_tracer;
-	}
-#ifdef ERTS_SMP
-	erts_smp_proc_unlock(tracee,
-			     (tracee == p
-			      ? ERTS_PROC_LOCKS_ALL_MINOR
-			      : ERTS_PROC_LOCKS_ALL));
-#endif
+        if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee)))
+            erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN, &tracee->common);
+
+        tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee));
+        trace_flags = ERTS_TRACE_FLAGS(tracee);
+
+        if (tracee != p)
+            erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN);
     } else if (is_external_pid(pid_spec)
 	       && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) {
 	    return am_undefined;
@@ -1024,8 +866,10 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 	HRelease(p,limit,hp+3);
 	return TUPLE2(hp, key, flag_list);
     } else if (key == am_tracer) {
-	hp = HAlloc(p, 3);
-	return TUPLE2(hp, key, tracer); /* Local pid or port */
+        if (tracer == am_false)
+            tracer = NIL;
+        hp = HAlloc(p, 3);
+        return TUPLE2(hp, key, tracer);
     } else {
 	goto error;
     }
@@ -1054,11 +898,11 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
  */
 static int function_is_traced(Process *p,
 			      Eterm mfa[3],
-			      Binary **ms,              /* out */
-			      Binary **ms_meta,         /* out */
-			      Eterm   *tracer_pid_meta, /* out */
-			      Uint    *count,           /* out */
-			      Eterm   *call_time)       /* out */
+			      Binary    **ms,              /* out */
+			      Binary    **ms_meta,         /* out */
+			      ErtsTracer *tracer_pid_meta, /* out */
+			      Uint       *count,           /* out */
+			      Eterm      *call_time)       /* out */
 {
     Export e;
     Export* ep;
@@ -1123,7 +967,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     Eterm traced = am_false;
     Eterm match_spec = am_false;
     Eterm retval = am_false;
-    Eterm meta = am_false;
+    ErtsTracer meta = erts_tracer_nil;
     Eterm call_time = NIL;
     int r;
 
@@ -1193,7 +1037,10 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	retval = match_spec;
 	break;
     case am_meta:
-	retval = meta;
+        retval = erts_tracer_to_term(p, meta);
+        if (retval == am_false)
+            /* backwards compatibility */
+            retval = NIL;
 	break;
     case am_meta_match_spec:
 	if (r & FUNC_TRACE_META_TRACE) {
@@ -1216,7 +1063,8 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	}
 	break;
     case am_all: {
-	Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false;
+	Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false,
+            m = am_false;
 	
 	if (ms) {
 	    match_spec = MatchSetGetSource(ms);
@@ -1235,6 +1083,9 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	if (r & FUNC_TRACE_TIME_TRACE) {
 	    ct = call_time;
 	}
+
+        m = erts_tracer_to_term(p, meta);
+
 	hp = HAlloc(p, (3+2)*6);
 	retval = NIL;
 	t = TUPLE2(hp, am_call_count, c); hp += 3;
@@ -1243,7 +1094,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
-	t = TUPLE2(hp, am_meta, meta); hp += 3;
+	t = TUPLE2(hp, am_meta, m); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_match_spec, match_spec); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
@@ -1306,7 +1157,8 @@ trace_info_on_load(Process* p, Eterm key)
     case am_meta:
 	hp = HAlloc(p, 3);
 	if (erts_default_trace_pattern_flags.meta) {
-	    return TUPLE2(hp, key, erts_default_meta_tracer_pid);
+            ASSERT(!ERTS_TRACER_IS_NIL(erts_default_meta_tracer));
+	    return TUPLE2(hp, key, erts_tracer_to_term(p, erts_default_meta_tracer));
 	} else {
 	    return TUPLE2(hp, key, am_false);
 	}
@@ -1345,7 +1197,7 @@ trace_info_on_load(Process* p, Eterm key)
 	}
     case am_all:
 	{
-	    Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t;
+	    Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t, m;
 	    
 	    if (erts_default_trace_pattern_flags.local ||
 		(! erts_default_trace_pattern_flags.breakpoint)) {
@@ -1363,6 +1215,8 @@ trace_info_on_load(Process* p, Eterm key)
 		    MatchSetGetSource(erts_default_meta_match_spec);
 		meta_match_spec = copy_object(meta_match_spec, p);
 	    }
+            m = (erts_default_trace_pattern_flags.meta
+                 ? erts_tracer_to_term(p, erts_default_meta_tracer) : am_false);
 	    hp = HAlloc(p, (3+2)*5 + 3);
 	    t = TUPLE2(hp, am_call_count, 
 		       (erts_default_trace_pattern_flags.call_count
@@ -1370,9 +1224,7 @@ trace_info_on_load(Process* p, Eterm key)
 	    r = CONS(hp, t, r); hp += 2;
 	    t = TUPLE2(hp, am_meta_match_spec, meta_match_spec); hp += 3;
 	    r = CONS(hp, t, r); hp += 2;
-	    t = TUPLE2(hp, am_meta, 
-		       (erts_default_trace_pattern_flags.meta
-			? erts_default_meta_tracer_pid : am_false)); hp += 3;
+	    t = TUPLE2(hp, am_meta, m); hp += 3;
 	    r = CONS(hp, t, r); hp += 2;
 	    t = TUPLE2(hp, am_match_spec, match_spec); hp += 3;
 	    r = CONS(hp, t, r); hp += 2;
@@ -1397,7 +1249,7 @@ int
 erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 		       Binary* match_prog_set, Binary *meta_match_prog_set,
 		       int on, struct trace_pattern_flags flags,
-		       Eterm meta_tracer_pid, int is_blocking)
+		       ErtsTracer meta_tracer, int is_blocking)
 {
     const ErtsCodeIndex code_ix = erts_active_code_ix();
     int matches = 0;
@@ -1487,7 +1339,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 		    }
 		    if (flags.meta) {
 			erts_set_mtrace_bif(pc, meta_match_prog_set,
-					    meta_tracer_pid);
+					    meta_tracer);
 			m = 1;
 		    }
 		    if (flags.call_time) {
@@ -1527,7 +1379,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 	    }
 	    if (flags.meta) {
 		erts_set_mtrace_break(&finish_bp.f, meta_match_prog_set,
-				      meta_tracer_pid);
+				      meta_tracer);
 	    }
 	    if (flags.call_count) {
 		erts_set_count_break(&finish_bp.f, on);
@@ -2336,50 +2188,79 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 }
 /* End: Trace for System Profiling */
 
-BIF_RETTYPE
-trace_delivered_1(BIF_ALIST_1)
+/* Trace delivered send an aux work message to all schedulers
+   and when all schedulers have acknowledged that they have seen
+   the message the message is sent to the requesting process.
+
+   IMPORTANT: We have to make sure that the all messages sent
+   using enif_send have been delivered before we send the message
+   to the caller.
+
+   There used to be a separate implementation for when only a pid
+   is passed in, but since this is not performance critical code
+   we now use the same approach for both.
+*/
+
+typedef struct {
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    Eterm target;
+    erts_smp_atomic32_t refc;
+} ErtsTraceDeliveredAll;
+
+static void
+reply_trace_delivered_all(void *vtdarp)
 {
-    DECL_AM(trace_delivered);
-#ifdef ERTS_SMP
-    ErlHeapFragment *bp;
-#else
-    ErtsProcLocks locks = 0;
-#endif
-    Eterm *hp;
-    Eterm msg, ref, msg_ref;
-    Process *p;
-    if (BIF_ARG_1 == am_all) {
-	p = NULL;
-    } else if (! (p = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-				    BIF_ARG_1, ERTS_PROC_LOCKS_ALL))) {
-	if (is_not_internal_pid(BIF_ARG_1)) {
-	    BIF_ERROR(BIF_P, BADARG);
-	}
-    }
-    
-    ref = erts_make_ref(BIF_P);
+    ErtsTraceDeliveredAll *tdarp = (ErtsTraceDeliveredAll *) vtdarp;
+    ErtsProcLocks rp_locks = 0;
 
-#ifdef ERTS_SMP
-    bp = new_message_buffer(REF_THING_SIZE + 4);
-    hp = &bp->mem[0];
-    msg_ref = STORE_NC(&hp, &bp->off_heap, ref);
-#else
-    hp = HAlloc(BIF_P, 4);
-    msg_ref = ref;
-#endif
+    if (erts_smp_atomic32_dec_read_nob(&tdarp->refc) == 0) {
+        Process *rp = tdarp->proc;
+        Eterm *hp = NULL;
+        ErlOffHeap *ohp = NULL;
+        ErtsMessage *mp = NULL;
+        Eterm ref_copy, msg;
 
-    msg = TUPLE3(hp, AM_trace_delivered, BIF_ARG_1, msg_ref);
+        mp = erts_alloc_message_heap(
+            rp, &rp_locks, 4 + NC_HEAP_SIZE(tdarp->ref), &hp, &ohp);
 
-#ifdef ERTS_SMP
-    erts_send_sys_msg_proc(BIF_P->common.id, BIF_P->common.id, msg, bp);
-    if (p)
-	erts_smp_proc_unlock(p,
-			     (BIF_P == p
-			      ? ERTS_PROC_LOCKS_ALL_MINOR
-			      : ERTS_PROC_LOCKS_ALL));
-#else
-    erts_send_message(BIF_P, BIF_P, &locks, msg, ERTS_SND_FLG_NO_SEQ_TRACE);
-#endif
+        ref_copy = STORE_NC(&hp, ohp, tdarp->ref);
+        msg = TUPLE3(hp, am_trace_delivered, tdarp->target, ref_copy);
+
+        erts_queue_message(rp, &rp_locks, mp, msg);
+
+        if (rp_locks)
+            erts_smp_proc_unlock(rp, rp_locks);
 
-    BIF_RET(ref);
+	erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp);
+        erts_proc_dec_refc(rp);
+    }
+}
+
+BIF_RETTYPE
+trace_delivered_1(BIF_ALIST_1)
+{
+
+    if (BIF_ARG_1 == am_all || is_internal_pid(BIF_ARG_1)) {
+        Eterm *hp, ref;
+        ErtsTraceDeliveredAll *tdarp =
+            erts_alloc(ERTS_ALC_T_MISC_AUX_WORK, sizeof(ErtsTraceDeliveredAll));
+
+        tdarp->proc = BIF_P;
+        ref = erts_make_ref(BIF_P);
+        hp = &tdarp->ref_heap[0];
+        tdarp->ref = STORE_NC(&hp, NULL, ref);
+        tdarp->target = BIF_ARG_1;
+        erts_smp_atomic32_init_nob(&tdarp->refc,
+                                   (erts_aint32_t) erts_no_schedulers);
+        erts_proc_add_refc(BIF_P, 1);
+        erts_schedule_multi_misc_aux_work(0,
+                                          erts_no_schedulers,
+                                          reply_trace_delivered_all,
+                                          (void *) tdarp);
+        BIF_RET(ref);
+    } else {
+        BIF_ERROR(BIF_P, BADARG);
+    }
 }
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index be14386b14..76b96637ae 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -135,21 +135,22 @@ get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks)
 
 
 static Eterm
-set_tracee_flags(Process *tracee_p, Eterm tracer, Uint d_flags, Uint e_flags) {
+set_tracee_flags(Process *tracee_p, ErtsTracer tracer,
+                 Uint d_flags, Uint e_flags) {
     Eterm ret;
     Uint flags;
 
-    if (tracer == NIL) {
+    if (ERTS_TRACER_IS_NIL(tracer)) {
 	flags = ERTS_TRACE_FLAGS(tracee_p) & ~TRACEE_FLAGS;
     }  else {
 	flags = ((ERTS_TRACE_FLAGS(tracee_p) & ~d_flags) | e_flags);
-	if (! flags) tracer = NIL;
+	if (! flags) tracer = erts_tracer_nil;
     }
-    ret = ((ERTS_TRACER_PROC(tracee_p) != tracer
+    ret = ((!ERTS_TRACER_COMPARE(ERTS_TRACER(tracee_p),tracer)
 	    || ERTS_TRACE_FLAGS(tracee_p) != flags)
 	   ? am_true
 	   : am_false);
-    ERTS_TRACER_PROC(tracee_p) = tracer;
+    erts_tracer_replace(&tracee_p->common, tracer);
     ERTS_TRACE_FLAGS(tracee_p) = flags;
     return ret;
 }
@@ -163,40 +164,16 @@ set_tracee_flags(Process *tracee_p, Eterm tracer, Uint d_flags, Uint e_flags) {
 ** returns fail_term on failure. Fails if tracer pid or port is invalid.
 */
 static Eterm 
-set_match_trace(Process *tracee_p, Eterm fail_term, Eterm tracer,
+set_match_trace(Process *tracee_p, Eterm fail_term, ErtsTracer tracer,
 		Uint d_flags, Uint e_flags) {
-    Eterm ret = fail_term;
-    Process *tracer_p;
-    
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCKS_ALL == 
-		       erts_proc_lc_my_proc_locks(tracee_p));
-
-    if (is_internal_pid(tracer)
-	&& (tracer_p = 
-	    erts_pid2proc(tracee_p, ERTS_PROC_LOCKS_ALL,
-			  tracer, ERTS_PROC_LOCKS_ALL))) {
-	if (tracee_p != tracer_p) {
-	    ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags);
-	    ERTS_TRACE_FLAGS(tracer_p) |= (ERTS_TRACE_FLAGS(tracee_p)
-					   ? F_TRACER
-					   : 0);
-	    erts_smp_proc_unlock(tracer_p, ERTS_PROC_LOCKS_ALL);
-	}
-    } else if (is_internal_port(tracer)) {
-	Port *tracer_port = 
-	    erts_id2port_sflgs(tracer,
-			       tracee_p,
-			       ERTS_PROC_LOCKS_ALL,
-			       ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
-	if (tracer_port) {
-	    ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags);
-	    erts_port_release(tracer_port);
-	}
-    } else {
-	ASSERT(is_nil(tracer));
-	ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags);
-    }
-    return ret;
+
+    ERTS_SMP_LC_ASSERT(
+        ERTS_PROC_LOCKS_ALL == erts_proc_lc_my_proc_locks(tracee_p)
+        || erts_thr_progress_is_blocking());
+
+    if (ERTS_TRACER_IS_NIL(tracer) || erts_is_tracer_enabled(tracee_p, tracer))
+        return set_tracee_flags(tracee_p, tracer, d_flags, e_flags);
+    return fail_term;
 }
 
 /*
@@ -2358,7 +2335,7 @@ restart:
 	case matchEnableTrace:
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
 		BEGIN_ATOMIC_TRACE(c_p);
-		set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), 0, n);
+		set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n);
 		esp[-1] = am_true;
 	    } else {
 		esp[-1] = FAIL_TERM;
@@ -2371,7 +2348,7 @@ restart:
 		BEGIN_ATOMIC_TRACE(c_p);
 		if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) {
 		    /* Always take over the tracer of the current process */
-		    set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), 0, n);
+		    set_tracee_flags(tmpp, ERTS_TRACER(c_p), 0, n);
 		    esp[-1] = am_true;
 		}
 	    }
@@ -2379,7 +2356,7 @@ restart:
 	case matchDisableTrace:
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
 		BEGIN_ATOMIC_TRACE(c_p);
-		set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), n, 0);
+		set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0);
 		esp[-1] = am_true;
 	    } else {
 		esp[-1] = FAIL_TERM;
@@ -2392,7 +2369,7 @@ restart:
 		BEGIN_ATOMIC_TRACE(c_p);
 		if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) {
 		    /* Always take over the tracer of the current process */
-		    set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), n, 0);
+		    set_tracee_flags(tmpp, ERTS_TRACER(c_p), n, 0);
 		    esp[-1] = am_true;
 		}
 	    }
@@ -2428,7 +2405,7 @@ restart:
 	    {
 		/*    disable         enable                                */
 		Uint  d_flags  = 0,   e_flags  = 0;  /* process trace flags */
-		Eterm tracer = ERTS_TRACER_PROC(c_p);
+		ErtsTracer tracer = erts_tracer_nil;
 		/* XXX Atomicity note: Not fully atomic. Default tracer
 		 * is sampled from current process but applied to
 		 * tracee and tracer later after releasing main
@@ -2440,29 +2417,34 @@ restart:
 		 * {trace,[],[{{tracer,Tracer}}]} is much, much older.
 		 */
 		int   cputs = 0;
+                erts_tracer_update(&tracer, ERTS_TRACER(c_p));
 		
 		if (! erts_trace_flags(esp[-1], &d_flags, &tracer, &cputs) ||
 		    ! erts_trace_flags(esp[-2], &e_flags, &tracer, &cputs) ||
 		    cputs ) {
 		    (--esp)[-1] = FAIL_TERM;
+                    ERTS_TRACER_CLEAR(&tracer);
 		    break;
 		}
 		erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		(--esp)[-1] = set_match_trace(c_p, FAIL_TERM, tracer,
 					      d_flags, e_flags);
 		erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                ERTS_TRACER_CLEAR(&tracer);
 	    }
 	    break;
 	case matchTrace3:
 	    {
 		/*    disable         enable                                */
 		Uint  d_flags  = 0,   e_flags  = 0;  /* process trace flags */
-		Eterm tracer = ERTS_TRACER_PROC(c_p);
+		ErtsTracer tracer = erts_tracer_nil;
 		/* XXX Atomicity note. Not fully atomic. See above. 
 		 * Above it could possibly be solved, but not here.
 		 */
 		int   cputs = 0;
 		Eterm tracee = (--esp)[0];
+
+                erts_tracer_update(&tracer, ERTS_TRACER(c_p));
 		
 		if (! erts_trace_flags(esp[-1], &d_flags, &tracer, &cputs) ||
 		    ! erts_trace_flags(esp[-2], &e_flags, &tracer, &cputs) ||
@@ -2470,6 +2452,7 @@ restart:
 		    ! (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, 
 				       tracee, ERTS_PROC_LOCKS_ALL))) {
 		    (--esp)[-1] = FAIL_TERM;
+                    ERTS_TRACER_CLEAR(&tracer);
 		    break;
 		}
 		if (tmpp == c_p) {
@@ -2483,6 +2466,7 @@ restart:
 		    erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
 		    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 		}
+                ERTS_TRACER_CLEAR(&tracer);
 	    }
 	    break;
 	case matchCatch:  /* Match success, now build result */
@@ -5084,7 +5068,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 	lint_res = db_match_set_lint(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE);
 	mps = db_match_set_compile(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE);
     }
-    
+
     if (mps == NULL) {
 	hp = HAlloc(p,3);
 	ret = TUPLE2(hp, am_error, lint_res);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 62417fa05c..f33ade27f3 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2233,9 +2233,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	n++;
     }
 #endif
-    ASSERT(is_nil(ERTS_TRACER_PROC(p)) ||
-	   is_internal_pid(ERTS_TRACER_PROC(p)) ||
-	   is_internal_port(ERTS_TRACER_PROC(p)));
+    ASSERT(IS_TRACER_VALID(ERTS_TRACER(p)));
 
     ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0)));
     if (is_not_immed(p->group_leader)) {
@@ -2905,7 +2903,7 @@ reply_gc_info(void *vgcirp)
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, mp, msg, NIL);
+    erts_queue_message(rp, &rp_locks, mp, msg);
 
     if (gcirp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 1a621863bb..8e201d5711 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1248,7 +1248,7 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
 	    ErtsMessage *mp = erts_alloc_message(0, NULL);
 	    mp->data.heap_frag = tmr->btm.bp;
 	    erts_queue_message(proc, &proc_locks, mp,
-			       tmr->btm.message, NIL);
+			       tmr->btm.message);
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND);
 	    queued_message = 1;
 	    proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND;
@@ -1980,7 +1980,7 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 
 	ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end);
 
-	erts_queue_message(proc, &proc_locks, mp, msg, NIL);
+	erts_queue_message(proc, &proc_locks, mp, msg);
 
 	if (c_p)
 	    proc_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -2111,7 +2111,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 
 	msg = TUPLE3(hp, tag, tref, res);
 
-	erts_queue_message(c_p, &proc_locks, mp, msg, NIL);
+	erts_queue_message(c_p, &proc_locks, mp, msg);
 
 	proc_locks &= ~ERTS_PROC_LOCK_MAIN;
 	if (proc_locks)
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 9231fb1b34..39c0617143 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -97,6 +97,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"dist_entry_links",			"address"		},
     {   "code_write_permission",                NULL                    },
     {	"proc_status",				"pid"			},
+    {	"proc_trace",				"pid"			},
     {   "ports_snapshot",                       NULL                    },
     {	"meta_name_tab",	         	"address"		},
     {	"meta_main_tab_slot",			"address"		},
@@ -148,6 +149,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"dist_entry_out_queue",			"address"		},
     {	"port_sched_lock",			"port_id"		},
     {	"sys_msg_q", 				NULL			},
+    {	"tracer_mtx", 				NULL			},
     {   "port_table",                           NULL                    },
 #endif
     {	"mtrace_op",				NULL			},
@@ -160,9 +162,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"proclist_pre_alloc_lock",		"address"		},
     {	"xports_list_pre_alloc_lock",		"address"		},
     {	"inet_buffer_stack_lock",		NULL			},
-    {	"gc_info",				NULL			},
-    {	"io_wake",				NULL			},
-    {	"timer_wheel",				NULL			},
     {	"system_block",				NULL			},
     {	"timeofday",				NULL			},
     {	"get_time",				NULL			},
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 7596747b91..1cb02ec274 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -324,7 +324,7 @@ erts_queue_dist_message(Process *rcvr,
 			tok_label, tok_lastcnt, tok_serial);
 	    }
 #endif
-	    erts_queue_message(rcvr, rcvr_locks, mp, msg, token);
+	    erts_queue_message(rcvr, rcvr_locks, mp, msg);
 	}
     }
     else {
@@ -349,7 +349,7 @@ erts_queue_dist_message(Process *rcvr,
         }
 #endif
 
-	LINK_MESSAGE(rcvr, mp);
+	LINK_MESSAGE(rcvr, mp, &mp->next, 1);
 
 	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
@@ -364,31 +364,34 @@ erts_queue_dist_message(Process *rcvr,
     }
 }
 
-/* Add a message last in message queue */
+/* Add messages last in message queue */
 static Sint
-queue_message(Process *c_p,
-	      Process* receiver,
-	      erts_aint32_t *receiver_state,
-	      ErtsProcLocks *receiver_locks,
-	      ErtsMessage* mp,
-	      Eterm message,
-	      Eterm seq_trace_token
-#ifdef USE_VM_PROBES
-		   , Eterm dt_utag
-#endif
-    )
+queue_messages(Process *c_p,
+               Process* receiver,
+               erts_aint32_t *receiver_state,
+               ErtsProcLocks *receiver_locks,
+               ErtsMessage* first,
+               ErtsMessage** last,
+               Uint len)
 {
     Sint res;
     int locked_msgq = 0;
     erts_aint32_t state;
 
-    ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver));
+    ASSERT(is_value(ERL_MESSAGE_TERM(first)));
+    ASSERT(ERL_MESSAGE_TOKEN(first) == am_undefined ||
+           ERL_MESSAGE_TOKEN(first) == NIL ||
+           is_tuple(ERL_MESSAGE_TOKEN(first)));
 
 #ifdef ERTS_SMP
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
+                       *receiver_locks == erts_proc_lc_my_proc_locks(receiver));
+#endif
 
     if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
 	if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
-	    ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
+            ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
 
 	    if (receiver_state)
 		state = *receiver_state;
@@ -417,16 +420,10 @@ queue_message(Process *c_p,
 	/* Drop message if receiver is exiting or has a pending exit... */
 	if (locked_msgq)
 	    erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
-	erts_cleanup_messages(mp);
+	erts_cleanup_messages(first);
 	return 0;
     }
 
-    ERL_MESSAGE_TERM(mp) = message;
-    ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
-#ifdef USE_VM_PROBES
-    ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
-#endif
-
     res = receiver->msg.len;
 #ifdef ERTS_SMP
     if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
@@ -440,75 +437,83 @@ queue_message(Process *c_p,
 	 */
 	res += receiver->msg_inq.len;
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
-	LINK_MESSAGE_PRIVQ(receiver, mp);
+        LINK_MESSAGE_PRIVQ(receiver, first, last, len);
     }
     else
 #endif
     {
-	LINK_MESSAGE(receiver, mp);
+	LINK_MESSAGE(receiver, first, last, len);
     }
 
+    if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
+        ErtsMessage *msg = first;
+
 #ifdef USE_VM_PROBES
-    if (DTRACE_ENABLED(message_queued)) {
-        DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
-        Sint tok_label = 0;
-        Sint tok_lastcnt = 0;
-        Sint tok_serial = 0;
-
-        dtrace_proc_str(receiver, receiver_name);
-        if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
-            tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
-            tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
-            tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
+        if (DTRACE_ENABLED(message_queued)) {
+            DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
+            Sint tok_label = 0;
+            Sint tok_lastcnt = 0;
+            Sint tok_serial = 0;
+            Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg);
+
+            dtrace_proc_str(receiver, receiver_name);
+            if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
+                tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
+                tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
+                tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
+            }
+            DTRACE6(message_queued,
+                    receiver_name, size_object(ERL_MESSAGE_TERM(msg)),
+                    receiver->msg.len,
+                    tok_label, tok_lastcnt, tok_serial);
         }
-        DTRACE6(message_queued,
-                receiver_name, size_object(message), receiver->msg.len,
-                tok_label, tok_lastcnt, tok_serial);
-    }
 #endif
 
-    if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE))
-	trace_receive(receiver, message);
+        while (msg) {
+            trace_receive(receiver, ERL_MESSAGE_TERM(msg));
+            msg = msg->next;
+        }
+
+    }
 
     if (locked_msgq)
 	erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
 
-    erts_proc_notify_new_message(receiver,
 #ifdef ERTS_SMP
-				 *receiver_locks
+    erts_proc_notify_new_message(receiver, *receiver_locks);
 #else
-				 0
-#endif
-	);
-
-#ifndef ERTS_SMP
+    erts_proc_notify_new_message(receiver, 0);
     ERTS_HOLE_CHECK(receiver);
 #endif
     return res;
 }
 
-void
-#ifdef USE_VM_PROBES
-erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks,
-                         ErtsMessage* mp,
-                         Eterm message, Eterm seq_trace_token, Eterm dt_utag)
-#else
+static Sint
+queue_message(Process *c_p,
+              Process* receiver,
+              erts_aint32_t *receiver_state,
+              ErtsProcLocks *receiver_locks,
+              ErtsMessage* mp, Eterm msg)
+{
+    ERL_MESSAGE_TERM(mp) = msg;
+    return queue_messages(c_p, receiver, receiver_state, receiver_locks,
+                          mp, &mp->next, 1 );
+}
+
+Sint
 erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks,
-                   ErtsMessage* mp,
-                   Eterm message, Eterm seq_trace_token)
-#endif
+                   ErtsMessage* mp, Eterm msg)
 {
-    queue_message(NULL,
-		  receiver,
-		  NULL,
-		  receiver_locks,
-		  mp,
-		  message,
-		  seq_trace_token
-#ifdef USE_VM_PROBES
-		  , dt_utag
-#endif
-	);
+    return queue_message(NULL, receiver, NULL, receiver_locks, mp, msg);
+}
+
+
+Sint
+erts_queue_messages(Process* receiver, ErtsProcLocks *receiver_locks,
+                    ErtsMessage* first, ErtsMessage** last, Uint len)
+{
+    return queue_messages(NULL, receiver, NULL, receiver_locks,
+                          first, last, len);
 }
 
 void
@@ -821,17 +826,15 @@ erts_send_message(Process* sender,
 #endif
     }
 
+    ERL_MESSAGE_TOKEN(mp) = token;
+#ifdef USE_VM_PROBES
+    ERL_MESSAGE_DT_UTAG(mp) = utag;
+#endif
     res = queue_message(sender,
 			receiver,
 			&receiver_state,
 			receiver_locks,
-			mp,
-			message,
-			token
-#ifdef USE_VM_PROBES
-			, utag
-#endif
-	);
+			mp, message);
 
     BM_SWAP_TIMER(send,system);
     
@@ -887,7 +890,8 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL);
 	temptoken = copy_struct(token, sz_token, &hp, ohp);
-	erts_queue_message(to, to_locksp, mp, save, temptoken);
+        ERL_MESSAGE_TOKEN(mp) = temptoken;
+	erts_queue_message(to, to_locksp, mp, save);
     } else {
 	sz_from = IS_CONST(from) ? 0 : size_object(from);
 #ifdef SHCOPY_SEND
@@ -909,7 +913,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 		     ? from
 		     : copy_struct(from, sz_from, &hp, ohp));
 	save = TUPLE3(hp, am_EXIT, from_copy, mess);
-	erts_queue_message(to, to_locksp, mp, save, NIL);
+	erts_queue_message(to, to_locksp, mp, save);
     }
 }
 
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index ad4e65274c..608cf552a2 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -182,48 +182,64 @@ typedef struct {
     Sint len;            /* queue length */
 } ErlMessageInQueue;
 
+typedef struct erl_trace_message_queue__ {
+    struct erl_trace_message_queue__ *next; /* point to the next receiver */
+    Eterm receiver;
+    ErtsMessage* first;
+    ErtsMessage** last;  /* point to the last next pointer */
+    Sint len;            /* queue length */
+} ErlTraceMessageQueue;
+
 #endif
 
 /* Get "current" message */
 #define PEEK_MESSAGE(p)  (*(p)->msg.save)
 
+#ifdef USE_VM_PROBES
+#define LINK_MESSAGE_DTAG(mp, dt) ERL_MESSAGE_DT_UTAG(mp) = dt
+#else
+#define LINK_MESSAGE_DTAG(mp, dt)
+#endif
 
-/* Add message last in private message queue */
-#define LINK_MESSAGE_PRIVQ(p, mp) do { \
-    *(p)->msg.last = (mp); \
-    (p)->msg.last = &(mp)->next; \
-    (p)->msg.len++; \
-} while (0)
-
+#define LINK_MESSAGE_IMPL(p, first_msg, last_msg, num_msgs, where) do { \
+        *(p)->where.last = (first_msg);                                 \
+        (p)->where.last = (last_msg);                                   \
+        (p)->where.len += (num_msgs);                                   \
+    } while(0)
 
 #ifdef ERTS_SMP
 
-/* Move in message queue to end of private message queue */
-#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P)					\
-do {									\
-    if ((P)->msg_inq.first) {						\
-	*(P)->msg.last = (P)->msg_inq.first;				\
-	(P)->msg.last = (P)->msg_inq.last;				\
-	(P)->msg.len += (P)->msg_inq.len;				\
-	(P)->msg_inq.first = NULL;					\
-	(P)->msg_inq.last = &(P)->msg_inq.first;			\
-	(P)->msg_inq.len = 0;						\
-    }									\
-} while (0)
-
-/* Add message last in message queue */
-#define LINK_MESSAGE(p, mp) do { \
-    *(p)->msg_inq.last = (mp); \
-    (p)->msg_inq.last = &(mp)->next; \
-    (p)->msg_inq.len++; \
-} while(0)
+/* Add message last in private message queue */
+#define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, len)                 \
+    do {                                                                \
+        LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg);            \
+    } while (0)
+
+/* Add message last_msg in message queue */
+#define LINK_MESSAGE(p, first_msg, last_msg, len)       \
+    LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg_inq)
+
+#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p)                   \
+    do {                                                \
+        if (p->msg_inq.first) {                         \
+            *p->msg.last = p->msg_inq.first;            \
+            p->msg.last = p->msg_inq.last;              \
+            p->msg.len += p->msg_inq.len;               \
+            p->msg_inq.first = NULL;                    \
+            p->msg_inq.last = &p->msg_inq.first;        \
+            p->msg_inq.len = 0;                         \
+        }                                               \
+    } while (0)
 
 #else
 
-#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P)
+#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p)
 
-/* Add message last in message queue */
-#define LINK_MESSAGE(p, mp) LINK_MESSAGE_PRIVQ((p), (mp))
+/* Add message last_msg in message queue */
+#define LINK_MESSAGE(p, first_msg, last_msg, len)                       \
+    do {                                                                \
+        LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg);            \
+    } while(0)
 
 #endif
 
@@ -259,23 +275,30 @@ do {									\
 	(HEAP_FRAG_P)->off_heap.overhead = 0;				\
     } while (0)
 
+#ifdef USE_VM_PROBES
+#define ERL_MESSAGE_DT_UTAG_INIT(MP) ERL_MESSAGE_DT_UTAG(MP) = NIL
+#else
+#define ERL_MESSAGE_DT_UTAG_INIT(MP) do{ } while (0)
+#endif
+
+#define ERTS_INIT_MESSAGE(MP)                           \
+    do {                                                \
+        (MP)->next = NULL;                              \
+        ERL_MESSAGE_TERM(MP) = THE_NON_VALUE;           \
+        ERL_MESSAGE_TOKEN(MP) = NIL;                    \
+        ERL_MESSAGE_DT_UTAG_INIT(MP);                   \
+        MP->data.attached = NULL;                       \
+    } while (0)
+
 void init_message(void);
 ErlHeapFragment* new_message_buffer(Uint);
 ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
 					    Eterm *, Uint);
 void free_message_buffer(ErlHeapFragment *);
 void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm);
-#ifdef USE_VM_PROBES
-void erts_queue_message_probe(Process*, ErtsProcLocks*, ErtsMessage*,
-                              Eterm message, Eterm seq_trace_token, Eterm dt_utag);
-#define erts_queue_message(RP,RL,BP,Msg,SEQ) \
-    erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL)
-#else
-void erts_queue_message(Process*, ErtsProcLocks*, ErtsMessage*,
-                        Eterm message, Eterm seq_trace_token);
-#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \
-    erts_queue_message((RP),(RL),(BP),(Msg),(SEQ))
-#endif
+Sint erts_queue_message(Process*, ErtsProcLocks*,ErtsMessage*, Eterm);
+Sint erts_queue_messages(Process*, ErtsProcLocks*,
+                         ErtsMessage*, ErtsMessage**, Uint);
 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
 Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
@@ -354,9 +377,7 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp)
 
     if (sz == 0) {
 	mp = erts_alloc_message_ref();
-	mp->next = NULL;
-	ERL_MESSAGE_TERM(mp) = NIL;
-	mp->data.attached = NULL;
+        ERTS_INIT_MESSAGE(mp);
 	if (hpp)
 	    *hpp = NULL;
 	return mp;
@@ -365,8 +386,7 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp)
     mp = erts_alloc(ERTS_ALC_T_MSG,
 		    sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm));
 
-    mp->next = NULL;
-    ERL_MESSAGE_TERM(mp) = NIL;
+    ERTS_INIT_MESSAGE(mp);
     mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
     ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz);
 
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 71e3fd8b6e..d0f305900a 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -257,7 +257,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
 
     if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
 
-    erts_queue_message(rp, &rp_locks, msgp, msg, NIL);
+    erts_queue_message(rp, &rp_locks, msgp, msg);
 
     if (esdp && msaccrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 61dba4ef82..f35c08450c 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -130,17 +130,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
     env->fpe_was_unmasked = erts_block_fpe();
     env->tmp_obj_list = NULL;
     env->exception_thrown = 0;
-}
-
-static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif)
-{
-    env->mod_nif = mod_nif;
-    env->proc = NULL;
-    env->hp = NULL;
-    env->hp_end = NULL;
-    env->heap_frag = NULL;
-    env->fpe_was_unmasked = erts_block_fpe();
-    env->tmp_obj_list = NULL;
+    env->tracee = NULL;
 }
 
 /* Temporary object header, auto-deallocated when NIF returns
@@ -180,13 +170,6 @@ void erts_post_nif(ErlNifEnv* env)
     free_tmp_objs(env);
 }
 
-static void post_nif_noproc(ErlNifEnv* env)
-{
-    erts_unblock_fpe(env->fpe_was_unmasked);
-    free_tmp_objs(env);
-}
-
-
 /* Flush out our cached heap pointers to allow an ordinary HAlloc
 */
 static void flush_env(ErlNifEnv* env)
@@ -247,18 +230,20 @@ struct enif_msg_environment_t
     Process phony_proc;
 };
 
-ErlNifEnv* enif_alloc_env(void)
+static ERTS_INLINE void
+setup_nif_env(struct enif_msg_environment_t* msg_env,
+              struct erl_module_nif* mod)
 {
-    struct enif_msg_environment_t* msg_env =
-	erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t));
     Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */
-	
-    msg_env->env.hp = phony_heap; 
+
+    msg_env->env.hp = phony_heap;
     msg_env->env.hp_end = phony_heap;
     msg_env->env.heap_frag = NULL;
-    msg_env->env.mod_nif = NULL;
+    msg_env->env.mod_nif = mod;
     msg_env->env.tmp_obj_list = NULL;
+    msg_env->env.fpe_was_unmasked = erts_block_fpe();
     msg_env->env.proc = &msg_env->phony_proc;
+    msg_env->env.exception_thrown = 0;
     memset(&msg_env->phony_proc, 0, sizeof(Process));
     HEAP_START(&msg_env->phony_proc) = phony_heap;
     HEAP_TOP(&msg_env->phony_proc) = phony_heap;
@@ -270,6 +255,13 @@ ErlNifEnv* enif_alloc_env(void)
     msg_env->phony_proc.space_verified = 0;
     msg_env->phony_proc.space_verified_from = NULL;
 #endif
+}
+
+ErlNifEnv* enif_alloc_env(void)
+{
+    struct enif_msg_environment_t* msg_env =
+	erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t));
+    setup_nif_env(msg_env, NULL);
     return &msg_env->env;
 }
 void enif_free_env(ErlNifEnv* env)
@@ -302,24 +294,114 @@ void enif_clear_env(ErlNifEnv* env)
     menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
     
     ASSERT(!is_offheap(&MSO(p)));
+    erts_unblock_fpe(env->fpe_was_unmasked);
     free_tmp_objs(env);
 }
-int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
-	      ErlNifEnv* msg_env, ERL_NIF_TERM msg)
+
+#ifdef ERTS_SMP
+#ifdef DEBUG
+static int enif_send_delay = 0;
+#define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 2 == 0)
+#else
+#ifdef ERTS_PROC_LOCK_OWN_IMPL
+#define ERTS_FORCE_ENIF_SEND_DELAY() 0
+#else
+/*
+ * We always schedule messages if we do not use our own
+ * process lock implementation, as if we try to do a trylock on
+ * a lock that might already be locked by the same thread.
+ * And what happens then with different mutex implementations
+ * is not always guaranteed.
+ */
+#define ERTS_FORCE_ENIF_SEND_DELAY() 1
+#endif
+#endif
+
+int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
+{
+    ErlTraceMessageQueue *msgq, **last_msgq;
+    int reds = 0;
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE);
+
+    msgq = c_p->trace_msg_q;
+
+    if (!msgq)
+        goto error;
+
+    do {
+        Process* rp;
+        ErtsProcLocks rp_locks;
+        ErtsMessage *first, **last;
+        Uint len;
+
+        first = msgq->first;
+        last = msgq->last;
+        len = msgq->len;
+        msgq->first = NULL;
+        msgq->last = &msgq->first;
+        msgq->len = 0;
+        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE);
+
+        ASSERT(len != 0);
+
+        rp = erts_proc_lookup(msgq->receiver);
+        if (rp) {
+            rp_locks = 0;
+            if (rp->common.id == c_p->common.id)
+                rp_locks = c_p_locks;
+            erts_queue_messages(rp, &rp_locks, first, last, len);
+            if (rp->common.id == c_p->common.id)
+                rp_locks &= ~c_p_locks;
+            if (rp_locks)
+                erts_smp_proc_unlock(rp, rp_locks);
+            reds += len;
+        } else {
+            erts_cleanup_messages(first);
+        }
+        reds += 1;
+        erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE);
+        msgq = msgq->next;
+    } while (msgq);
+
+    last_msgq = &c_p->trace_msg_q;
+
+    while (*last_msgq) {
+        msgq = *last_msgq;
+        if (msgq->len == 0) {
+            *last_msgq = msgq->next;
+            erts_free(ERTS_ALC_T_TRACE_MSG_QUEUE, msgq);
+        } else {
+            last_msgq = &msgq->next;
+        }
+    }
+
+error:
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE);
+
+    return reds;
+}
+
+#endif
+
+int
+enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
+          ErlNifEnv* msg_env, ERL_NIF_TERM msg)
 {
     struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
-    ErtsProcLocks rp_locks = 0;
+    ErtsProcLocks rp_locks = 0, lc_locks = 0, c_p_locks = ERTS_PROC_LOCK_MAIN;
     Process* rp;
     Process* c_p;
     ErtsMessage *mp;
     Eterm receiver = to_pid->pid;
     int flush_me = 0;
-    int scheduler = erts_get_scheduler_id() != 0;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int scheduler = esdp ? esdp->no : 0;
 
     if (env != NULL) {
 	c_p = env->proc;
 	if (receiver == c_p->common.id) {
-	    rp_locks = ERTS_PROC_LOCK_MAIN;
+	    rp_locks = c_p_locks;
 	    flush_me = 1;
 	}
     }
@@ -329,7 +411,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 #else
 	erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
 #endif
-    }    
+    }
 
     rp = (scheduler
 	  ? erts_proc_lookup(receiver)
@@ -353,7 +435,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
             menv->env.heap_frag = NULL;
             MBUF(&menv->phony_proc) = NULL;
         }
-        ASSERT(!is_offheap(&MSO(&menv->phony_proc)));
     } else {
         Uint sz = size_object(msg);
         Eterm *hp;
@@ -362,14 +443,83 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         ASSERT(hp == mp->hfrag.mem+mp->hfrag.used_size);
     }
 
-    if (flush_me) {	
-	flush_env(env); /* Needed for ERTS_HOLE_CHECK */ 
+    ERL_MESSAGE_TERM(mp) = msg;
+
+    if (flush_me) {
+	flush_env(env); /* Needed for ERTS_HOLE_CHECK */
+    }
+
+    if (!env || !env->tracee) {
+        erts_queue_message(rp, &rp_locks, mp, msg);
+    }
+#ifdef ERTS_SMP
+    else {
+        /* This clause is taken when the nif is called in the context
+           of a traced process. We do not know which locks we have
+           so we have to do a try lock and if that fails we en queue
+           the message in a special trace message output queue of the
+           tracee */
+        ErlTraceMessageQueue *msgq;
+        Process *t_p = env->tracee;
+
+
+        erts_smp_proc_lock(t_p, ERTS_PROC_LOCK_TRACE);
+
+        msgq = t_p->trace_msg_q;
+
+        while (msgq != NULL) {
+            if (msgq->receiver == receiver) {
+                break;
+            }
+            msgq = msgq->next;
+        }
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+        lc_locks = erts_proc_lc_my_proc_locks(rp);
+        rp_locks |= lc_locks;
+        if (receiver == c_p->common.id)
+            c_p_locks |= lc_locks;
+#endif
+        if (ERTS_FORCE_ENIF_SEND_DELAY() || msgq ||
+            rp_locks & ERTS_PROC_LOCK_MSGQ ||
+            erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
+
+            if (!msgq) {
+
+                msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE,
+                                  sizeof(ErlTraceMessageQueue));
+                msgq->receiver = receiver;
+                msgq->first = mp;
+                msgq->last = &mp->next;
+                msgq->len = 1;
+
+                /* Insert in linked list */
+                msgq->next = t_p->trace_msg_q;
+                t_p->trace_msg_q = msgq;
+
+                erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
+
+                erts_schedule_flush_trace_messages(t_p->common.id);
+
+            } else {
+                msgq->len++;
+                *msgq->last = mp;
+                msgq->last = &mp->next;
+                erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
+            }
+        } else {
+            erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
+            rp_locks &= ~ERTS_PROC_LOCK_TRACE;
+            rp_locks |= ERTS_PROC_LOCK_MSGQ;
+            erts_queue_message(rp, &rp_locks, mp, msg);
+        }
     }
-    erts_queue_message(rp, &rp_locks, mp, msg, am_undefined);
+#endif
+
     if (c_p == rp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-    if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+    if (rp_locks & ~lc_locks)
+	erts_smp_proc_unlock(rp, rp_locks & ~lc_locks);
     if (!scheduler)
 	erts_proc_dec_refc(rp);
     if (flush_me) {
@@ -1485,10 +1635,10 @@ static void close_lib(struct erl_module_nif* lib)
     ASSERT(erts_refc_read(&lib->rt_dtor_cnt,0) == 0);
 
     if (lib->entry != NULL && lib->entry->unload != NULL) {
-	ErlNifEnv env;
-	pre_nif_noproc(&env, lib);
-	lib->entry->unload(&env, lib->priv_data);
-	post_nif_noproc(&env);
+	struct enif_msg_environment_t msg_env;
+        setup_nif_env(&msg_env, lib);
+	lib->entry->unload(&msg_env.env, lib->priv_data);
+        enif_clear_env(&msg_env.env);
     }
     if (!erts_is_static_nif(lib->handle))
       erts_sys_ddll_close(lib->handle);
@@ -1634,10 +1784,10 @@ static void nif_resource_dtor(Binary* bin)
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
 
     if (type->dtor != NULL) {
-	ErlNifEnv env;
-	pre_nif_noproc(&env, type->owner);
-	type->dtor(&env,resource->data);
-	post_nif_noproc(&env);
+        struct enif_msg_environment_t msg_env;
+        setup_nif_env(&msg_env, type->owner);
+	type->dtor(&msg_env.env, resource->data);
+        enif_clear_env(&msg_env.env);
     }
     if (erts_refc_dectest(&type->refc, 0) == 0) {
 	ASSERT(type->next == NULL);
@@ -2916,6 +3066,9 @@ erts_unload_nif(struct erl_module_nif* lib)
     ASSERT(erts_smp_thr_progress_is_blocking());
     ASSERT(lib != NULL);
     ASSERT(lib->mod != NULL);
+
+    erts_tracer_nif_clear();
+
     for (rt = resource_type_list.next;
 	 rt != &resource_type_list;
 	 rt = next) {
@@ -2958,6 +3111,7 @@ void erl_nif_init()
     resource_type_list.owner = NULL;
     resource_type_list.module = THE_NON_VALUE;
     resource_type_list.name = THE_NON_VALUE;
+
 }
 
 int erts_nif_get_funcs(struct erl_module_nif* mod,
@@ -2967,7 +3121,8 @@ int erts_nif_get_funcs(struct erl_module_nif* mod,
     return mod->entry->num_of_funcs;
 }
 
-Eterm erts_nif_call_function(Process *p, struct erl_module_nif* mod,
+Eterm erts_nif_call_function(Process *p, Process *tracee,
+                             struct erl_module_nif* mod,
                              ErlNifFunc *fun, int argc, Eterm *argv)
 {
     Eterm nif_result;
@@ -2979,13 +3134,14 @@ Eterm erts_nif_call_function(Process *p, struct erl_module_nif* mod,
             break;
     ASSERT(i < mod->entry->num_of_funcs);
     if (p)
-        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN);
+        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN
+                           || erts_smp_thr_progress_is_blocking());
 #endif
     if (p) {
         struct enif_environment_t env;
         ASSERT(is_internal_pid(p->common.id));
         erts_pre_nif(&env, p, mod);
-        env.may_delay_enif_send = 1;
+        env.tracee = tracee;
         nif_result = (*fun->fptr)(&env, argc, argv);
         if (env.exception_thrown)
             nif_result = THE_NON_VALUE;
@@ -2995,7 +3151,7 @@ Eterm erts_nif_call_function(Process *p, struct erl_module_nif* mod,
            so we create a phony one. */
         struct enif_msg_environment_t msg_env;
         setup_nif_env(&msg_env, mod);
-        msg_env.env.may_delay_enif_send = 1;
+        msg_env.env.tracee = tracee;
         nif_result = (*fun->fptr)(&msg_env.env, argc, argv);
         if (msg_env.env.exception_thrown)
             nif_result = THE_NON_VALUE;
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 33785ecaca..3964f7f679 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -87,10 +87,10 @@ typedef ErlNifSInt64 ErlNifTime;
 #define ERL_NIF_TIME_ERROR ((ErlNifSInt64) ERTS_NAPI_TIME_ERROR__)
 
 typedef enum {
-    ERL_NIF_SEC = ERTS_NAPI_SEC__,
-    ERL_NIF_MSEC = ERTS_NAPI_MSEC__,
-    ERL_NIF_USEC = ERTS_NAPI_USEC__,
-    ERL_NIF_NSEC = ERTS_NAPI_NSEC__
+    ERL_NIF_SEC    = ERTS_NAPI_SEC__,
+    ERL_NIF_MSEC   = ERTS_NAPI_MSEC__,
+    ERL_NIF_USEC   = ERTS_NAPI_USEC__,
+    ERL_NIF_NSEC   = ERTS_NAPI_NSEC__
 } ErlNifTimeUnit;
 
 struct enif_environment_t;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d222e79f56..e7d0d127e2 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -444,7 +444,8 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP;
 typedef enum {
     ERTS_PSTT_GC,	/* Garbage Collect */
     ERTS_PSTT_CPC,	/* Check Process Code */
-    ERTS_PSTT_COHMQ     /* Change off heap message queue */
+    ERTS_PSTT_COHMQ,    /* Change off heap message queue */
+    ERTS_PSTT_FTMQ      /* Flush trace msg queue */
 } ErtsProcSysTaskType;
 
 #define ERTS_MAX_PROC_SYS_TASK_ARGS 2
@@ -1138,7 +1139,7 @@ reply_sched_wall_time(void *vswtrp)
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, mp, msg, NIL);
+    erts_queue_message(rp, &rp_locks, mp, msg);
 
     if (swtrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -1217,7 +1218,7 @@ reply_system_check(void *vscrp)
     hpp = &hp;
     msg = STORE_NC(hpp, ohp, scrp->ref);
 
-    erts_queue_message(rp, &rp_locks, mp, msg, NIL);
+    erts_queue_message(rp, &rp_locks, mp, msg);
 
     if (scrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -9315,6 +9316,7 @@ Process *schedule(Process *p, int calls)
 	esdp = erts_scheduler_data;
 	ASSERT(esdp->current_process == p);
 #endif
+
 	reds = actual_reds = calls - esdp->virtual_reds;
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
@@ -9327,27 +9329,39 @@ Process *schedule(Process *p, int calls)
 
 	p->reds += actual_reds;
 
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+#ifdef ERTS_SMP
+	erts_smp_proc_lock(p, ERTS_PROC_LOCK_TRACE);
+        if (p->trace_msg_q) {
+            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE);
+            erts_schedule_flush_trace_messages(p->common.id);
+        } else
+            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE);
+#endif
 
-	state = erts_smp_atomic32_read_acqb(&p->state);
+	state = erts_smp_atomic32_read_nob(&p->state);
 
 	if (IS_TRACED(p)) {
 	    if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE))
 		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
-	    if (state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) {
+	    if ((state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) {
 		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
-		    trace_sched(p, ((state & ERTS_PSFLG_FREE)
-				    ? am_out_exited
-				    : am_out_exiting));
+		    trace_sched(p, ERTS_PROC_LOCK_MAIN,
+                                ((state & ERTS_PSFLG_FREE)
+                                 ? am_out_exited
+                                 : am_out_exiting));
 	    }
 	    else {
-		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED))
-		    trace_sched(p, am_out);
-		else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
-		    trace_virtual_sched(p, am_out);
+		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) ||
+                    ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
+		    trace_sched(p, ERTS_PROC_LOCK_MAIN, am_out);
 	    }
 	}
 
+	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+
+        /* have to re-read state after taking lock */
+        state = erts_smp_atomic32_read_nob(&p->state);
+
 #ifdef ERTS_SMP
 	if (state & ERTS_PSFLG_PENDING_EXIT)
 	    erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN
@@ -9844,27 +9858,27 @@ Process *schedule(Process *p, int calls)
 
 	p->fcalls = reds;
 
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+
 	if (IS_TRACED(p)) {
 	    if (state & ERTS_PSFLG_EXITING) {
 		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
-		    trace_sched(p, am_in_exiting);
+		    trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in_exiting);
 	    }
 	    else {
-		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED))
-		    trace_sched(p, am_in);
-		else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
-		    trace_virtual_sched(p, am_in);
+		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) ||
+                    ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
+		    trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
 	    }
 	    if (IS_TRACED_FL(p, F_TRACE_CALLS)) {
 		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN);
 	    }
 	}
 
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
 #ifdef ERTS_SMP
-	if (is_not_nil(ERTS_TRACER_PROC(p)))
-	    erts_check_my_tracer_proc(p);
+        /* Clears tracer if it has been removed */
+        (void)ERTS_TRACER_PROC_IS_ENABLED(p);
 #endif
 
 	if (state & ERTS_PSFLG_RUNNING_SYS) {
@@ -9996,7 +10010,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 	ASSERT(hp_start + hsz == hp);
 #endif
 
-	erts_queue_message(rp, &rp_locks, mp, msg, NIL);
+	erts_queue_message(rp, &rp_locks, mp, msg);
 
 	if (c_p == rp)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -10164,8 +10178,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 {
     int garbage_collected = 0;
     erts_aint32_t state = *statep;
-    int max_reds = in_reds;
-    int reds = 0;
+    int reds = in_reds;
     int qmask = 0;
 
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
@@ -10193,12 +10206,12 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    if (c_p->flags & F_DISABLE_GC) {
 		save_gc_task(c_p, st, st_prio);
 		st = NULL;
-		reds++;
+		reds--;
 	    }
 	    else {
 		if (!garbage_collected) {
 		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
-		    reds += erts_garbage_collect_nobump(c_p,
+		    reds -= erts_garbage_collect_nobump(c_p,
 							0,
 							c_p->arg_reg,
 							c_p->arity);
@@ -10207,21 +10220,30 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 		st_res = am_true;
 	    }
 	    break;
-	case ERTS_PSTT_CPC:
+	case ERTS_PSTT_CPC: {
+            int cpc_reds = 0;
 	    st_res = erts_check_process_code(c_p,
 					     st->arg[0],
                                              unsigned_val(st->arg[1]),
-					     &reds);
+					     &cpc_reds);
+            reds -= cpc_reds;
 	    if (is_non_value(st_res)) {
 		/* Needed gc, but gc was disabled */
 		save_gc_task(c_p, st, st_prio);
 		st = NULL;
 	    }
 	    break;
+        }
 	case ERTS_PSTT_COHMQ:
-	    reds += erts_complete_off_heap_message_queue_change(c_p);
+	    reds -= erts_complete_off_heap_message_queue_change(c_p);
 	    st_res = am_true;
 	    break;
+#ifdef ERTS_SMP
+        case ERTS_PSTT_FTMQ:
+	    reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN);
+	    st_res = am_true;
+	    break;
+#endif
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
 	    st_res = am_false;
@@ -10231,11 +10253,11 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    reds += notify_sys_task_executed(c_p, st, st_res);
 
 	state = erts_smp_atomic32_read_acqb(&c_p->state);
-    } while (qmask && reds < max_reds);
+    } while (qmask && reds > 0);
 
     *statep = state;
 
-    return reds;
+    return in_reds - reds;
 }
 
 static int
@@ -10267,6 +10289,12 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	case ERTS_PSTT_COHMQ:
 	    st_res = am_false;
 	    break;
+#ifdef ERTS_SMP
+        case ERTS_PSTT_FTMQ:
+	    reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN);
+	    st_res = am_true;
+	    break;
+#endif
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
 	    st_res = am_false;
@@ -10402,8 +10430,8 @@ badarg:
     BIF_ERROR(BIF_P, BADARG);
 }
 
-void
-erts_schedule_complete_off_heap_message_queue_change(Eterm pid)
+static void
+erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type)
 {
     Process *rp = erts_proc_lookup(pid);
     if (rp) {
@@ -10413,7 +10441,7 @@ erts_schedule_complete_off_heap_message_queue_change(Eterm pid)
 
 	st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK,
 			ERTS_PROC_SYS_TASK_SIZE(0));
-	st->type = ERTS_PSTT_COHMQ;
+	st->type = type;
 	st->requester = NIL;
 	st->reply_tag = NIL;
 	st->req_id = NIL;
@@ -10428,6 +10456,18 @@ erts_schedule_complete_off_heap_message_queue_change(Eterm pid)
     }
 }
 
+void
+erts_schedule_complete_off_heap_message_queue_change(Eterm pid)
+{
+    erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ);
+}
+
+void
+erts_schedule_flush_trace_messages(Eterm pid)
+{
+    erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ);
+}
+
 static void
 save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
 {
@@ -10886,6 +10926,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     Eterm res = THE_NON_VALUE;
     erts_aint32_t state = 0;
     erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL;
+    ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL;
 #ifdef SHCOPY_SPAWN
     erts_shcopy_t info;
     INITIALIZE_SHCOPY(info);
@@ -11062,7 +11103,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	    : STORE_NC(&p->htop, &p->off_heap, parent->group_leader);
     }
 
-    erts_get_default_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER_PROC(p));
+    erts_get_default_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER(p));
 
     p->msg.first = NULL;
     p->msg.last = &p->msg.first;
@@ -11098,21 +11139,47 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->last_old_htop = NULL;
 #endif
 
+#ifdef ERTS_SMP
+    p->trace_msg_q = NULL;
+    p->scheduler_data = NULL;
+    p->suspendee = NIL;
+    p->pending_suspenders = NULL;
+    p->pending_exit.reason = THE_NON_VALUE;
+    p->pending_exit.bp = NULL;
+#endif
+
+#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
+    p->fp_exception = 0;
+#endif
+
     if (IS_TRACED(parent)) {
 	if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) {
 	    ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS);
-	    ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent);
-	}
-	if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) {
-	    trace_proc_spawn(parent, p->common.id, mod, func, args);
+            erts_tracer_replace(&p->common, ERTS_TRACER(parent));
 	}
-	if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) {
+        if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) {
 	    /* Overrides TRACE_CHILDREN */
 	    ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS);
-	    ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent);
+            erts_tracer_replace(&p->common, ERTS_TRACER(parent));
 	    ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOS1 | F_TRACE_SOS);
 	    ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOS1 | F_TRACE_SOS);
 	}
+        if (so->flags & SPO_LINK && ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) {
+		ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS);
+                erts_tracer_replace(&p->common, ERTS_TRACER(parent));
+		if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/
+		    ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
+		    ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
+		}
+        }
+        if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) {
+            locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            trace_proc_spawn(parent, p->common.id, mod, func, args);
+            if (so->flags & SPO_LINK)
+                trace_proc(parent, locks, parent, am_link, p->common.id);
+        }
     }
 
     /*
@@ -11123,10 +11190,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef DEBUG
 	int ret;
 #endif
-	if (IS_TRACED_FL(parent, F_TRACE_PROCS)) {
-	    trace_proc(parent, parent, am_link, p->common.id);
-	}
-
 #ifdef DEBUG
 	ret = erts_add_link(&ERTS_P_LINKS(parent),  LINK_PID, p->common.id);
 	ASSERT(ret == 0);
@@ -11137,17 +11200,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id);
 #endif
 
-	if (IS_TRACED(parent)) {
-	    if (ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) {
-		ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS);
-		ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); /*maybe steal*/
-
-		if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/
-		    ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
-		    ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
-		}
-	    }
-	}
     }
 
     /*
@@ -11162,19 +11214,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	so->mref = mref;
     }
 
-#ifdef ERTS_SMP
-    p->scheduler_data = NULL;
-    p->suspendee = NIL;
-    p->pending_suspenders = NULL;
-    p->pending_exit.reason = THE_NON_VALUE;
-    p->pending_exit.bp = NULL;
-#endif
-
-#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
-    p->fp_exception = 0;
-#endif
-
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+    erts_smp_proc_unlock(p, locks);
 
     res = p->common.id;
 
@@ -11198,7 +11238,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
  error:
 
-    erts_smp_proc_unlock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR);
 
     return res;
 }
@@ -11223,7 +11263,7 @@ void erts_init_empty_process(Process *p)
     p->rcount = 0;
     p->common.id = ERTS_INVALID_PID;
     p->reds = 0;
-    ERTS_TRACER_PROC(p) = NIL;
+    ERTS_TRACER(p) = erts_tracer_nil;
     ERTS_TRACE_FLAGS(p) = F_INITIAL_TRACE_FLAGS;
     p->group_leader = ERTS_INVALID_PID;
     p->flags = 0;
@@ -11343,7 +11383,7 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->live_hf_end == ERTS_INVALID_HFRAG_PTR);
     ASSERT(p->heap == NULL);
     ASSERT(p->common.id == ERTS_INVALID_PID);
-    ASSERT(ERTS_TRACER_PROC(p) == NIL);
+    ASSERT(ERTS_TRACER_IS_NIL(ERTS_TRACER(p)));
     ASSERT(ERTS_TRACE_FLAGS(p) == F_INITIAL_TRACE_FLAGS);
     ASSERT(p->group_leader == ERTS_INVALID_PID);
     ASSERT(p->next == NULL);
@@ -11690,7 +11730,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
 #endif
-	erts_queue_message(to, to_locksp, mp, mess, NIL);
+	erts_queue_message(to, to_locksp, mp, mess);
     } else {
 	Eterm temp_token;
 	Uint sz_token;
@@ -11708,9 +11748,10 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
 #endif
 	/* the trace token must in this case be updated by the caller */
-	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL);
+	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, to);
 	temp_token = copy_struct(token, sz_token, &hp, ohp);
-	erts_queue_message(to, to_locksp, mp, mess, temp_token);
+        ERL_MESSAGE_TOKEN(mp) = temp_token;
+	erts_queue_message(to, to_locksp, mp, mess);
     }
 }
 
@@ -11819,6 +11860,9 @@ send_exit_signal(Process *c_p,		/* current process if and only
 
     if ((state & ERTS_PSFLG_TRAP_EXIT)
 	&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
+        /* have to release the status lock in order to send the exit message */
+        erts_smp_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND);
+        *rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
         if (have_seqtrace(token) && token_update)
 	    seq_trace_update_send(token_update);
 	if (is_value(exit_tuple))
@@ -12116,6 +12160,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
     DistEntry *dep;
     Process *rp;
 
+
     switch(lnk->type) {
     case LINK_PID:
 	if(is_internal_port(item)) {
@@ -12163,7 +12208,11 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 		    if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
 			/* We didn't exit the process and it is traced */
 			if (IS_TRACED_FL(rp, F_TRACE_PROCS)) {
-			    trace_proc(p, rp, am_getting_unlinked, p->common.id);
+                            if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
+                                erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
+                                rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
+                            }
+			    trace_proc(NULL, 0, rp, am_getting_unlinked, p->common.id);
 			}
 		    }
 		}
@@ -12280,8 +12329,6 @@ erts_do_exit_process(Process* p, Eterm reason)
 	if (IS_TRACED_FL(p, F_TRACE_CALLS))
 	    erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING);
 
-	if (IS_TRACED_FL(p,F_TRACE_PROCS))
-	    trace_proc(p, p, am_exit, reason);
     }
 
     erts_trace_check_exiting(p->common.id);
@@ -12297,6 +12344,10 @@ erts_do_exit_process(Process* p, Eterm reason)
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 
+    if (IS_TRACED_FL(p,F_TRACE_PROCS))
+        trace_proc(p, ERTS_PROC_LOCK_MAIN, p, am_exit, reason);
+
+
     /*
      * p->u.initial of this process can *not* be used anymore;
      * will be overwritten by misc termination data.
@@ -12436,6 +12487,9 @@ erts_continue_exit_process(Process *p)
 	ASSERT(!p->common.u.alive.reg);
     }
 
+    if (IS_TRACED_FL(p, F_TRACE_SCHED_EXIT))
+        trace_sched(p, curr_locks, am_out_exited);
+
     erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
     curr_locks = ERTS_PROC_LOCKS_ALL;
 
@@ -12559,6 +12613,12 @@ erts_continue_exit_process(Process *p)
     if (nif_export)
 	erts_destroy_nif_export(nif_export);
 
+#ifdef ERTS_SMP
+    erts_flush_trace_messages(p, 0);
+#endif
+
+    ERTS_TRACER_CLEAR(&ERTS_TRACER(p));
+
     delete_process(p);
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c2303eea49..8261ae28f8 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1041,6 +1041,7 @@ struct process {
 
 #ifdef ERTS_SMP
     ErlMessageInQueue msg_inq;
+    ErlTraceMessageQueue *trace_msg_q;
     ErtsPendExit pending_exit;
     erts_proc_lock_t lock;
     ErtsSchedulerData *scheduler_data;
@@ -1358,7 +1359,6 @@ extern int erts_system_profile_ts_type;
 #define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))
 
 /* process trace_flags */
-
 #define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
 			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
 #define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
@@ -1380,8 +1380,7 @@ extern int erts_system_profile_ts_type;
 #define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
 #define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
 #define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
-#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
-#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */
+#define F_EXCEPTION_TRACE    F_TRACE_FLAG(15) /* May have exception trace on stack */
 
 /* port trace flags, currently the same as process trace flags */
 #define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
@@ -1717,6 +1716,8 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
 					     ErtsThrPrgrLaterOp *,
 					     UWord);
 void erts_schedule_complete_off_heap_message_queue_change(Eterm pid);
+void erts_schedule_flush_trace_messages(Eterm pid);
+int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks);
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_dbg_check_halloc_lock(Process *p);
@@ -1996,7 +1997,6 @@ erts_psd_set(Process *p, int ix, void *data)
 #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \
     erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
 
-
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
 
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index e1f470d381..a69185bc5c 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -106,6 +106,7 @@ static struct {
     Sint16 proc_lock_msgq;
     Sint16 proc_lock_btm;
     Sint16 proc_lock_status;
+    Sint16 proc_lock_trace;
 } lc_id;
 #endif
 
@@ -152,6 +153,7 @@ erts_init_proc_lock(int cpus)
     lc_id.proc_lock_msgq	= erts_lc_get_lock_order_id("proc_msgq");
     lc_id.proc_lock_btm		= erts_lc_get_lock_order_id("proc_btm");
     lc_id.proc_lock_status	= erts_lc_get_lock_order_id("proc_status");
+    lc_id.proc_lock_trace	= erts_lc_get_lock_order_id("proc_trace");
 #endif
 }
 
@@ -1056,6 +1058,11 @@ erts_proc_lock_init(Process *p)
     ethr_mutex_lock(&p->lock.status.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.status.lc);
+#endif
+    erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count);
+    ethr_mutex_lock(&p->lock.trace.mtx);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_trylock(1, &p->lock.trace.lc);
 #endif
 #endif
 #ifdef ERTS_PROC_LOCK_DEBUG
@@ -1078,6 +1085,7 @@ erts_proc_lock_fin(Process *p)
     erts_mtx_destroy(&p->lock.msgq);
     erts_mtx_destroy(&p->lock.btm);
     erts_mtx_destroy(&p->lock.status);
+    erts_mtx_destroy(&p->lock.trace);
 #endif
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
     erts_lcnt_proc_lock_destroy(p);
@@ -1100,26 +1108,29 @@ void erts_lcnt_enable_proc_lock_count(int enable) {
 void erts_lcnt_proc_lock_init(Process *p) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) {
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_main));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_link));
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq));
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm));
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_link));
         erts_lcnt_init_lock_empty(&(p->lock.lcnt_status));
+        erts_lcnt_init_lock_empty(&(p->lock.lcnt_trace));
     } else { /* now the common case */
         Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL;
         erts_lcnt_init_lock_x(&(p->lock.lcnt_main),  "proc_main",  ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_link),  "proc_link",  ERTS_LCNT_LT_PROCLOCK, pid);
         erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq),  "proc_msgq",  ERTS_LCNT_LT_PROCLOCK, pid);
         erts_lcnt_init_lock_x(&(p->lock.lcnt_btm),   "proc_btm",   ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_link),  "proc_link",  ERTS_LCNT_LT_PROCLOCK, pid);
         erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_x(&(p->lock.lcnt_trace), "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid);
     } /* the lock names should really be aligned to four characters */
 } /* logic reversed */
 
 void erts_lcnt_proc_lock_destroy(Process *p) {
     erts_lcnt_destroy_lock(&(p->lock.lcnt_main));
+    erts_lcnt_destroy_lock(&(p->lock.lcnt_link));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_btm));
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_link));
     erts_lcnt_destroy_lock(&(p->lock.lcnt_status));
+    erts_lcnt_destroy_lock(&(p->lock.lcnt_trace));
 }
 
 static void lcnt_enable_proc_lock_count(Process *proc, int enable) {
@@ -1138,10 +1149,11 @@ static void lcnt_enable_proc_lock_count(Process *proc, int enable) {
 void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock(&(lock->lcnt_trace)); }
 }
 
 void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
@@ -1150,44 +1162,50 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line);
     }
+    if (locks & ERTS_PROC_LOCK_LINK) {
+	erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line);
+    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
         erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line);
     }
     if (locks & ERTS_PROC_LOCK_BTM) {
         erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line);
-    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+        erts_lcnt_lock_post_x(&(lock->lcnt_trace), file, line);
+    }
 }
 
 void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock_unaquire(&(lock->lcnt_trace)); }
 }
 
 void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_unlock(&(lock->lcnt_trace)); }
 }
 void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) {
     if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
     if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); }
+    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); }
     if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); }
     if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); }
     if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); }
+    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_trylock(&(lock->lcnt_trace), res); }
 } /* reversed logic */
 #endif /* ERTS_ENABLE_LOCK_COUNT */
 
@@ -1224,6 +1242,10 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_lock_x(&lck,file,line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_lock_x(&lck,file,line);
+    }
 }
 
 void
@@ -1253,6 +1275,10 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_trylock_x(locked, &lck, file, line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_trylock_x(locked, &lck, file, line);
+    }
 }
 
 void
@@ -1261,6 +1287,10 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_unlock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_unlock(&lck);
@@ -1292,6 +1322,10 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_might_unlock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_might_unlock(&lck);
@@ -1323,6 +1357,8 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 	erts_lc_might_unlock(&p->lock.btm.lc);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_might_unlock(&p->lock.status.lc);
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_lc_might_unlock(&p->lock.trace.lc);
 #endif
 }
 
@@ -1354,6 +1390,10 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_require_lock(&lck, file, line);
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_require_lock(&lck, file, line);
+    }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_lc_require_lock(&p->lock.main.lc, file, line);
@@ -1365,6 +1405,8 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 	erts_lc_require_lock(&p->lock.btm.lc, file, line);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_require_lock(&p->lock.status.lc, file, line);
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_lc_require_lock(&p->lock.trace.lc, file, line);
 #endif
 }
 
@@ -1375,6 +1417,10 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
 					   ERTS_LC_FLG_LT_PROCLOCK);
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	lck.id = lc_id.proc_lock_trace;
+	erts_lc_unrequire_lock(&lck);
+    }
     if (locks & ERTS_PROC_LOCK_STATUS) {
 	lck.id = lc_id.proc_lock_status;
 	erts_lc_unrequire_lock(&lck);
@@ -1406,6 +1452,8 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
 	erts_lc_unrequire_lock(&p->lock.btm.lc);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_lc_unrequire_lock(&p->lock.status.lc);
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_lc_unrequire_lock(&p->lock.trace.lc);
 #endif
 }
 
@@ -1429,6 +1477,8 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
 	    lck.id = lc_id.proc_lock_btm;
 	else if (locks & ERTS_PROC_LOCK_STATUS)
 	    lck.id = lc_id.proc_lock_status;
+	else if (locks & ERTS_PROC_LOCK_TRACE)
+	    lck.id = lc_id.proc_lock_trace;
 	else
 	    erts_lc_fail("Unknown proc lock found");
 
@@ -1441,14 +1491,7 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
 
 void erts_proc_lc_chk_only_proc_main(Process *p)
 {
-#if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t proc_main = ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
-						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK);
-    erts_lc_check_exact(&proc_main, 1);
-#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_check_exact(&p->lock.main.lc, 1);
-#endif
+    erts_proc_lc_chk_only_proc(p, ERTS_PROC_LOCK_MAIN);
 }
 
 #if ERTS_PROC_LOCK_OWN_IMPL
@@ -1456,14 +1499,67 @@ void erts_proc_lc_chk_only_proc_main(Process *p)
   ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK)
 #endif /* ERTS_PROC_LOCK_OWN_IMPL */
 
+void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks)
+{
+    int have_locks_len = 0;
+#if ERTS_PROC_LOCK_OWN_IMPL
+    erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                    ERTS_PROC_LC_EMPTY_LOCK_INIT};
+    if (locks & ERTS_PROC_LOCK_MAIN) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_main;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_LINK) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_link;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_MSGQ) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_BTM) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_btm;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_STATUS) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_status;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+    erts_lc_lock_t have_locks[6];
+    if (locks & ERTS_PROC_LOCK_MAIN)
+	have_locks[have_locks_len++] = p->lock.main.lc;
+    if (locks & ERTS_PROC_LOCK_LINK)
+	have_locks[have_locks_len++] = p->lock.link.lc;
+    if (locks & ERTS_PROC_LOCK_MSGQ)
+	have_locks[have_locks_len++] = p->lock.msgq.lc;
+    if (locks & ERTS_PROC_LOCK_BTM)
+	have_locks[have_locks_len++] = p->lock.btm.lc;
+    if (locks & ERTS_PROC_LOCK_STATUS)
+	have_locks[have_locks_len++] = p->lock.status.lc;
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	have_locks[have_locks_len++] = p->lock.trace.lc;
+#endif
+    erts_lc_check_exact(have_locks, have_locks_len);
+}
+
 void
 erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 {
     int have_locks_len = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
     if (locks & ERTS_PROC_LOCK_MAIN) {
@@ -1486,8 +1582,12 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len].id = lc_id.proc_lock_status;
 	have_locks[have_locks_len++].extra = p->common.id;
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t have_locks[5];
+    erts_lc_lock_t have_locks[6];
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
     if (locks & ERTS_PROC_LOCK_LINK)
@@ -1498,6 +1598,8 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len++] = p->lock.btm.lc;
     if (locks & ERTS_PROC_LOCK_STATUS)
 	have_locks[have_locks_len++] = p->lock.status.lc;
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	have_locks[have_locks_len++] = p->lock.trace.lc;
 #endif
     erts_lc_check(have_locks, have_locks_len, NULL, 0);
 }
@@ -1508,12 +1610,14 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
     int have_locks_len = 0;
     int have_not_locks_len = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
 				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
-    erts_lc_lock_t have_not_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
+    erts_lc_lock_t have_not_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT,
+                                        ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT,
 					ERTS_PROC_LC_EMPTY_LOCK_INIT};
 
@@ -1557,9 +1661,17 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_status;
 	have_not_locks[have_not_locks_len++].extra = p->common.id;
     }
+    if (locks & ERTS_PROC_LOCK_TRACE) {
+	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
+	have_locks[have_locks_len++].extra = p->common.id;
+    }
+    else {
+	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_trace;
+	have_not_locks[have_not_locks_len++].extra = p->common.id;
+    }
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t have_locks[5];
-    erts_lc_lock_t have_not_locks[5];
+    erts_lc_lock_t have_locks[6];
+    erts_lc_lock_t have_not_locks[6];
 
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
@@ -1581,6 +1693,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len++] = p->lock.status.lc;
     else
 	have_not_locks[have_not_locks_len++] = p->lock.status.lc;
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	have_locks[have_locks_len++] = p->lock.trace.lc;
+    else
+	have_not_locks[have_not_locks_len++] = p->lock.trace.lc;
 #endif
 
     erts_lc_check(have_locks, have_locks_len,
@@ -1590,10 +1706,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 ErtsProcLocks
 erts_proc_lc_my_proc_locks(Process *p)
 {
-    int resv[5];
+    int resv[6];
     ErtsProcLocks res = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
+    erts_lc_lock_t locks[6] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
 						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_link,
@@ -1606,17 +1722,21 @@ erts_proc_lc_my_proc_locks(Process *p)
 						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_status,
+						 p->common.id,
+						 ERTS_LC_FLG_LT_PROCLOCK),
+			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_trace,
 						 p->common.id,
 						 ERTS_LC_FLG_LT_PROCLOCK)};
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t locks[5] = {p->lock.main.lc,
+    erts_lc_lock_t locks[6] = {p->lock.main.lc,
 			       p->lock.link.lc,
 			       p->lock.msgq.lc,
 			       p->lock.btm.lc,
-			       p->lock.status.lc};
+			       p->lock.status.lc,
+			       p->lock.trace.lc};
 #endif
 
-    erts_lc_have_locks(resv, locks, 5);
+    erts_lc_have_locks(resv, locks, 6);
     if (resv[0])
 	res |= ERTS_PROC_LOCK_MAIN;
     if (resv[1])
@@ -1627,6 +1747,8 @@ erts_proc_lc_my_proc_locks(Process *p)
 	res |= ERTS_PROC_LOCK_BTM;
     if (resv[4])
 	res |= ERTS_PROC_LOCK_STATUS;
+    if (resv[5])
+	res |= ERTS_PROC_LOCK_TRACE;
 
     return res;
 }
@@ -1634,14 +1756,15 @@ erts_proc_lc_my_proc_locks(Process *p)
 void
 erts_proc_lc_chk_no_proc_locks(char *file, int line)
 {
-    int resv[5];
-    int ids[5] = {lc_id.proc_lock_main,
+    int resv[6];
+    int ids[6] = {lc_id.proc_lock_main,
 		  lc_id.proc_lock_link,
 		  lc_id.proc_lock_msgq,
 		  lc_id.proc_lock_btm,
-		  lc_id.proc_lock_status};
-    erts_lc_have_lock_ids(resv, ids, 5);
-    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) {
+		  lc_id.proc_lock_status,
+		  lc_id.proc_lock_trace};
+    erts_lc_have_lock_ids(resv, ids, 6);
+    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4] || resv[5])) {
 	erts_lc_fail("%s:%d: Thread has process locks locked when expected "
 		     "not to have any process locks locked",
 		     file, line);
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index ac324c6f3e..d81ef63af2 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -66,7 +66,7 @@
 
 #endif
 
-#define ERTS_PROC_LOCK_MAX_BIT 4
+#define ERTS_PROC_LOCK_MAX_BIT 5
 
 typedef erts_aint32_t ErtsProcLocks;
 
@@ -84,6 +84,7 @@ typedef struct erts_proc_lock_t_ {
     erts_lcnt_lock_t lcnt_msgq;
     erts_lcnt_lock_t lcnt_btm;
     erts_lcnt_lock_t lcnt_status;
+    erts_lcnt_lock_t lcnt_trace;
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     erts_mtx_t main;
@@ -91,6 +92,7 @@ typedef struct erts_proc_lock_t_ {
     erts_mtx_t msgq;
     erts_mtx_t btm;
     erts_mtx_t status;
+    erts_mtx_t trace;
 #else
 #  error "no implementation"
 #endif
@@ -137,9 +139,18 @@ typedef struct erts_proc_lock_t_ {
  *   Protects the following fields in the process structure:
  *   * pending_suspenders
  *   * suspendee
+ *   * sys_tasks
  *   * ...
  */
-#define ERTS_PROC_LOCK_STATUS		(((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT)
+#define ERTS_PROC_LOCK_STATUS		(((ErtsProcLocks) 1) << 4)
+
+/*
+ * Trace message lock:
+ *   Protects the order in which messages are sent
+ *   from trace nifs. This lock is taken inside enif_send.
+ *
+ */
+#define ERTS_PROC_LOCK_TRACE            (((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT)
 
 /*
  * Special fields:
@@ -154,8 +165,8 @@ typedef struct erts_proc_lock_t_ {
  *   all process locks are held, and are allowed to be read if
  *   at least one process lock (whichever one doesn't matter)
  *   is held:
- *     * tracer_proc
- *     * tracer_flags
+ *     * common.tracer
+ *     * common.trace_flags
  *
  *   The following fields are only allowed to be accessed if
  *   both the schedule queue lock and at least one process lock
@@ -208,7 +219,7 @@ typedef struct erts_proc_lock_t_ {
   ((((ErtsProcLocks) 1) << (ERTS_PROC_LOCK_MAX_BIT + 1)) - 1)
 
 #define ERTS_PROC_LOCKS_ALL_MINOR	(ERTS_PROC_LOCKS_ALL \
-					 & ~ERTS_PROC_LOCK_MAIN)
+                                         & ~ERTS_PROC_LOCK_MAIN)
 
 
 #define ERTS_PIX_LOCKS_BITS		10
@@ -260,6 +271,7 @@ void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks);
 void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks);
 void erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks);
 void erts_proc_lc_chk_only_proc_main(Process *p);
+void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks);
 void erts_proc_lc_chk_no_proc_locks(char *file, int line);
 ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p);
 int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks);
@@ -477,9 +489,15 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
     if (locks & ERTS_PROC_LOCK_STATUS)
 	if (erts_mtx_trylock(&p->lock.status) == EBUSY)
 	    goto busy_status;
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	if (erts_mtx_trylock(&p->lock.trace) == EBUSY)
+	    goto busy_trace;
 
     return 0;
 
+busy_trace:
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_mtx_unlock(&p->lock.trace);
 busy_status:
     if (locks & ERTS_PROC_LOCK_BTM)
 	erts_mtx_unlock(&p->lock.btm);
@@ -568,6 +586,8 @@ erts_smp_proc_lock__(Process *p,
 	erts_mtx_lock(&p->lock.btm);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_mtx_lock(&p->lock.status);
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_mtx_lock(&p->lock.trace);
 
 #ifdef ERTS_PROC_LOCK_DEBUG
     erts_proc_lock_op_debug(p, locks, 1);
@@ -653,6 +673,8 @@ erts_smp_proc_unlock__(Process *p,
     erts_proc_lock_op_debug(p, locks, 0);
 #endif
 
+    if (locks & ERTS_PROC_LOCK_TRACE)
+	erts_mtx_unlock(&p->lock.trace);
     if (locks & ERTS_PROC_LOCK_STATUS)
 	erts_mtx_unlock(&p->lock.status);
     if (locks & ERTS_PROC_LOCK_BTM)
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index 97df8bc3fc..a5931ffc25 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -37,14 +37,16 @@
 #include "erl_alloc.h"
 #include "erl_monitors.h"
 
-#define ERTS_TRACER_PROC(P) 	((P)->common.tracer_proc)
+#define ERTS_TRACER(P)          ((P)->common.tracer)
+#define ERTS_TRACER_MODULE(T) 	(CAR(list_val(T)))
+#define ERTS_TRACER_STATE(T) 	(CDR(list_val(T)))
 #define ERTS_TRACE_FLAGS(P)	((P)->common.trace_flags)
 
 #define ERTS_P_LINKS(P)		((P)->common.u.alive.links)
 #define ERTS_P_MONITORS(P)	((P)->common.u.alive.monitors)
 
 #define IS_TRACED(p) \
-    (ERTS_TRACER_PROC((p)) != NIL)
+    (ERTS_TRACER(p) != NIL)
 #define ARE_TRACE_FLAGS_ON(p,tf) \
     ((ERTS_TRACE_FLAGS((p)) & (tf|F_SENSITIVE)) == (tf))
 #define IS_TRACED_FL(p,tf) \
@@ -56,7 +58,7 @@ typedef struct {
 	erts_atomic_t atmc;
 	Sint sint;
     } refc;
-    Eterm tracer_proc;
+    ErtsTracer tracer;
     Uint trace_flags;
     erts_smp_atomic_t timer;
     union {
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 947def7398..346404fe2a 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1966,7 +1966,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 		    *patch_refp = ref;
 		    ASSERT(hsz == size_object(message_template));
 		    message = copy_struct(message_template, hsz, &hp, ohp);
-		    erts_queue_message(rp, &rp_locks, mp, message, NIL);
+		    erts_queue_message(rp, &rp_locks, mp, message);
 		}
 		erts_smp_proc_unlock(rp, rp_locks);
 	    }
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 1654ea58d9..1e6ae8c82e 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -39,6 +39,7 @@
 #include "erl_bits.h"
 #include "erl_thr_progress.h"
 #include "erl_bif_unique.h"
+#include "erl_map.h"
 
 #if 0
 #define DEBUG_PRINTOUTS
@@ -46,17 +47,13 @@
 #undef DEBUG_PRINTOUTS
 #endif
 
-extern BeamInstr beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
-extern BeamInstr beam_return_trace[1];      /* OpCode(i_return_trace) */
-extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
-
 /* Pseudo export entries. Never filled in with data, only used to
    yield unique pointers of the correct type. */
 Export exp_send, exp_receive, exp_timeout;
 
-static Eterm system_seq_tracer;
+static ErtsTracer system_seq_tracer;
 static Uint default_trace_flags;
-static Eterm default_tracer;
+static ErtsTracer default_tracer;
 
 static Eterm system_monitor;
 static Eterm system_profile;
@@ -70,11 +67,8 @@ static erts_smp_rwmtx_t sys_trace_rwmtx;
 
 enum ErtsSysMsgType {
     SYS_MSG_TYPE_UNDEFINED,
-    SYS_MSG_TYPE_TRACE,
-    SYS_MSG_TYPE_SEQTRACE,
     SYS_MSG_TYPE_SYSMON,
     SYS_MSG_TYPE_ERRLGR,
-    SYS_MSG_TYPE_PROC_MSG,
     SYS_MSG_TYPE_SYSPROF
 };
 
@@ -298,43 +292,6 @@ write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer)
     return res;
 }
 
-/*
- * Patch a timestamp into a tuple. The tuple MUST be the last thing
- * built on the heap before the call, and the timestamp MUST be
- * the last thing after the call. This since patch_ts() might adjust
- * the size of the used area.
- */
-
-#define PATCH_TS__(Type, Tuple, Hp, Bp, Tracer)			\
-    do {							\
-	int ts_type__ = (Type);					\
-	if (ts_type__)						\
-	    patch_ts(ts_type__, (Tuple), (Hp), (Bp), (Tracer));	\
-    } while (0)
-
-#ifdef ERTS_SMP
-#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \
-    PATCH_TS__((Type), (Tuple), (Hp), (Bp), NULL)
-#else
-#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \
-    PATCH_TS__((Type), (Tuple), (Hp), (Bp), (Tracer))
-#endif
-
-static ERTS_INLINE void
-patch_ts(int ts_type, Eterm tuple, Eterm* hp, ErlHeapFragment *bp, Process *tracer)
-{
-    Eterm *tptr = tuple_val(tuple);
-    int arity = arityval(*tptr);
-
-    ASSERT(ts_type);
-    ASSERT((tptr+arity+1) == hp);
-
-    tptr[0] = make_arityval(arity+1);
-    tptr[1] = am_trace_ts;
-
-    *hp = write_ts(ts_type, hp+1, bp, tracer);
-}
-
 #ifdef ERTS_SMP
 static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type,
 				     Eterm from,
@@ -349,6 +306,14 @@ static void enqueue_sys_msg(enum ErtsSysMsgType type,
 static void init_sys_msg_dispatcher(void);
 #endif
 
+static void init_tracer_nif(void);
+static int tracer_cmp_fun(void*, void*);
+static HashValue tracer_hash_fun(void*);
+static void *tracer_alloc_fun(void*);
+static void tracer_free_fun(void*);
+
+typedef struct ErtsTracerNif_ ErtsTracerNif;
+
 void erts_init_trace(void) {
     erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
@@ -363,79 +328,38 @@ void erts_init_trace(void) {
     erts_system_monitor_clear(NULL);
     erts_system_profile_clear(NULL);
     default_trace_flags = F_INITIAL_TRACE_FLAGS;
-    default_tracer = NIL;
-    system_seq_tracer = am_false;
+    default_tracer = erts_tracer_nil;
+    system_seq_tracer = erts_tracer_nil;
 #ifdef ERTS_SMP
     init_sys_msg_dispatcher();
 #endif
+    init_tracer_nif();
 }
 
-static Eterm system_seq_tracer;
-
 #define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, UNUSED) \
   (*(BPP) = new_message_buffer((SZ)), \
    *(OHPP) = &(*(BPP))->off_heap, \
    (*(BPP))->mem)
 
-#ifdef ERTS_SMP
-#define ERTS_ENQ_TRACE_MSG(FPID, TPID, MSG, BP) \
-do { \
-    ERTS_LC_ASSERT(erts_smp_lc_mtx_is_locked(&smq_mtx)); \
-    enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \
-} while(0)
-#else
-#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP)		\
-    do {							\
-	ErtsMessage *mp__ = erts_alloc_message(0, NULL);		\
-	mp__->data.heap_frag = (BP);				\
-	erts_queue_message((TPROC), NULL, mp__, (MSG), NIL);	\
-    } while (0)
-#endif
-
-/*
- * NOTE that the ERTS_GET_TRACER_REF() returns from the function  (!!!)
- * using it, and resets the parameters used if the tracer is invalid, i.e.,
- * use it with extreme care!
- */
-#ifdef ERTS_SMP
-#define ERTS_NULL_TRACER_REF NIL
-#define ERTS_TRACER_REF_TYPE Eterm
- /* In the smp case, we never find the tracer invalid here (the sys
-    message dispatcher thread takes care of that). */
-#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \
-do { (RES) = (TPID); } while(0)
-int
-erts_is_tracer_proc_valid(Process* p)
-{
-    return 1;
-}
-#else
-#define ERTS_NULL_TRACER_REF NULL
-#define ERTS_TRACER_REF_TYPE Process *
-#define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \
-do { \
-    (RES) = erts_proc_lookup((TPID)); \
-    if (!(RES) || !(ERTS_TRACE_FLAGS((RES)) & F_TRACER)) { \
-	(TPID) = NIL; \
-	(TRACEE_FLGS) &= ~TRACEE_FLAGS; \
-	return; \
-    } \
-} while (0)
-int
-erts_is_tracer_proc_valid(Process* p)
-{
-    Process* tracer;
+static ERTS_INLINE int
+send_to_tracer_nif_raw(Process *c_p, Process *tracee, const ErtsTracer tracer,
+                       Uint trace_flags, Eterm t_p_id, ErtsTracerNif *tnif,
+                       Eterm tag, Eterm msg, Eterm extra, Eterm pam_result);
+static ERTS_INLINE int
+send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
+                   Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag,
+                   Eterm msg, Eterm extra);
+static ERTS_INLINE Eterm
+call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
+                    ErtsTracerNif **tnif_ref, Eterm tag, Eterm t_p_id);
+static int
+is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
+                       ErtsPTabElementCommon *t_p,
+                       ErtsTracerNif **tnif_ret, Eterm tag);
 
-    tracer = erts_proc_lookup(ERTS_TRACER_PROC(p));
-    if (tracer && ERTS_TRACE_FLAGS(tracer) & F_TRACER) {
-	return 1;
-    } else {
-	ERTS_TRACER_PROC(p) = NIL;
-	ERTS_TRACE_FLAGS(p) = ~TRACEE_FLAGS;
-	return 0;
-    }
-}
-#endif
+#define SEND_TO_TRACER(c_p, tag, msg)                                   \
+    send_to_tracer_nif(c_p, &(c_p)->common, (c_p)->common.id, NULL, tag, \
+                       msg, THE_NON_VALUE)
 
 static Uint active_sched;
 
@@ -450,19 +374,6 @@ static void
 exiting_reset(Eterm exiting)
 {
     erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
-    if (exiting == default_tracer) {
-	default_tracer = NIL;
-	default_trace_flags &= TRACEE_FLAGS;
-#ifdef DEBUG
-	default_trace_flags |= F_INITIAL_TRACE_FLAGS;
-#endif
-    }
-    if (exiting == system_seq_tracer) {
-#ifdef DEBUG_PRINTOUTS
-	erts_fprintf(stderr, "seq tracer %T exited\n", exiting);
-#endif
-	system_seq_tracer = am_false;
-    }
     if (exiting == system_monitor) {
 #ifdef ERTS_SMP
 	system_monitor = NIL;
@@ -487,11 +398,7 @@ erts_trace_check_exiting(Eterm exiting)
 {
     int reset = 0;
     erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
-    if (exiting == default_tracer)
-	reset = 1;
-    else if (exiting == system_seq_tracer)
-	reset = 1;
-    else if (exiting == system_monitor)
+    if (exiting == system_monitor)
 	reset = 1;
     else if (exiting == system_profile)
 	reset = 1;
@@ -500,23 +407,26 @@ erts_trace_check_exiting(Eterm exiting)
 	exiting_reset(exiting);
 }
 
-static ERTS_INLINE int
-is_valid_tracer(Eterm tracer)
-{
-    return erts_proc_lookup(tracer) || erts_is_valid_tracer_port(tracer);
-}
-
-Eterm
-erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new)
+ErtsTracer
+erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new)
 {
-    Eterm old;
-
-    if (new != am_false && !is_valid_tracer(new))
-	return THE_NON_VALUE;
+    ErtsTracer old;
+
+    if (!ERTS_TRACER_IS_NIL(new)) {
+        Eterm nif_result = call_enabled_tracer(
+            NULL, new, NULL,
+            am_trace_status, am_undefined);
+        switch (nif_result) {
+        case am_trace: break;
+        default:
+            return THE_NON_VALUE;
+        }
+    }
 
     erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
     old = system_seq_tracer;
-    system_seq_tracer = new;
+    system_seq_tracer = erts_tracer_nil;
+    erts_tracer_update(&system_seq_tracer, new);
 
 #ifdef DEBUG_PRINTOUTS
     erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old);
@@ -525,47 +435,56 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new)
     return old;
 }
 
-Eterm
+ErtsTracer
 erts_get_system_seq_tracer(void)
 {
-    Eterm st;
+    ErtsTracer st;
     erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
     st = system_seq_tracer;
 #ifdef DEBUG_PRINTOUTS
     erts_fprintf(stderr, "get seq tracer %T\n", st);
 #endif
     erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+
+    if (st != erts_tracer_nil &&
+        call_enabled_tracer(NULL, st, NULL, am_trace_status, am_undefined) == am_remove) {
+        erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil);
+        st = erts_tracer_nil;
+    }
+
     return st;
 }
 
 static ERTS_INLINE void
-get_default_tracing(Uint *flagsp, Eterm *tracerp)
+get_default_tracing(Uint *flagsp, ErtsTracer *tracerp)
 {
     if (!(default_trace_flags & TRACEE_FLAGS))
-	default_tracer = NIL;
+	ERTS_TRACER_CLEAR(&default_tracer);
 
-    if (is_nil(default_tracer)) {
+    if (ERTS_TRACER_IS_NIL(default_tracer)) {
 	default_trace_flags &= ~TRACEE_FLAGS;
-    } else if (is_internal_pid(default_tracer)) {
-	if (!erts_proc_lookup(default_tracer)) {
-	reset_tracer:
-	    default_trace_flags &= ~TRACEE_FLAGS;
-	    default_tracer = NIL;
-	}
     } else {
-	ASSERT(is_internal_port(default_tracer));
-	if (!erts_is_valid_tracer_port(default_tracer))
-	    goto reset_tracer;
+        Eterm nif_result = call_enabled_tracer(
+            NULL, default_tracer, NULL,
+            am_trace_status, am_undefined);
+        switch (nif_result) {
+        case am_trace: break;
+        default:
+            default_trace_flags &= ~TRACEE_FLAGS;
+            ERTS_TRACER_CLEAR(&default_tracer);
+        }
     }
 
     if (flagsp)
 	*flagsp = default_trace_flags;
-    if (tracerp)
-	*tracerp = default_tracer;
+    if (tracerp) {
+	erts_tracer_update(tracerp,default_tracer);
+    }
 }
 
 void
-erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp)
+erts_change_default_tracing(int setflags, Uint *flagsp,
+                            const ErtsTracer tracer)
 {
     erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
     if (flagsp) {
@@ -574,16 +493,18 @@ erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp)
 	else
 	    default_trace_flags &= ~(*flagsp);
     }
-    if (tracerp)
-	default_tracer = *tracerp;
-    get_default_tracing(flagsp, tracerp);
+
+    erts_tracer_update(&default_tracer, tracer);
+
+    get_default_tracing(flagsp, NULL);
     erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
 
 void
-erts_get_default_tracing(Uint *flagsp, Eterm *tracerp)
+erts_get_default_tracing(Uint *flagsp, ErtsTracer *tracerp)
 {
     erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    *tracerp = erts_tracer_nil; /* initialize */
     get_default_tracing(flagsp, tracerp);
     erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
 }
@@ -622,29 +543,21 @@ erts_get_system_profile(void) {
     return profile;
 }
 
-#ifdef ERTS_SMP
-static void
-do_send_to_port(Eterm to,
-		Port* unused_port,
-		Eterm from,
-		enum ErtsSysMsgType type,
-		Eterm message)
-{
-    Uint sz = size_object(message);
-    ErlHeapFragment *bp = new_message_buffer(sz);
-    Uint *hp = bp->mem;
-    Eterm msg = copy_struct(message, sz, &hp, &bp->off_heap);
 
-    enqueue_sys_msg_unlocked(type, from, to, msg, bp);
-}
-
-#define WRITE_SYS_MSG_TO_PORT write_sys_msg_to_port
+#ifdef HAVE_ERTS_NOW_CPU
+#  define GET_NOW(m, s, u) \
+do { \
+    if (erts_cpu_timestamp) \
+	erts_get_now_cpu(m, s, u); \
+    else \
+	get_now(m, s, u); \
+} while (0)
 #else
-#define WRITE_SYS_MSG_TO_PORT do_send_to_port
+#  define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0)
 #endif
 
 static void
-WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
+write_sys_msg_to_port(Eterm unused_to,
 		      Port* trace_port,
 		      Eterm unused_from,
 		      enum ErtsSysMsgType unused_type,
@@ -671,150 +584,6 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
     erts_free(ERTS_ALC_T_TMP, (void *) buffer);
 }
 
-
-#ifndef ERTS_SMP
-/* Send        {trace_ts, Pid, out, 0, Timestamp}
- * followed by {trace_ts, Pid, in, 0, NewTimestamp}
- *
- * 'NewTimestamp' through patch_ts().
- */
-static void 
-do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp, int ts_type) {
-#define LOCAL_HEAP_SIZE (5+5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-    Eterm message;
-    Eterm *hp;
-    Eterm mfarity;
-
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-    ASSERT(is_pid(pid));
-    ASSERT(is_tuple(timestamp));
-    ASSERT(*tuple_val(timestamp) == make_arityval(3));
-    
-    hp = local_heap;
-    mfarity = make_small(0);
-    message = TUPLE5(hp, am_trace_ts, pid, am_out, mfarity, timestamp);
-    /* Note, hp is deliberately NOT incremented since it will be reused */
-
-    do_send_to_port(trace_port->common.id,
-		    trace_port,
-		    pid,
-		    SYS_MSG_TYPE_UNDEFINED,
-		    message);
-
-
-    message = TUPLE5(hp, am_trace_ts, pid, am_in, mfarity,
-		     NIL /* Will be overwritten by timestamp */);
-    hp += 6;
-    hp[-1] = write_ts(ts_type, hp, NULL, NULL);
-
-    do_send_to_port(trace_port->common.id,
-		    trace_port,
-		    pid,
-		    SYS_MSG_TYPE_UNDEFINED,
-		    message);
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-}
-#endif
-
-/* If (c_p != NULL), a fake schedule out/in message pair will be sent,
- * if the driver so requests. 
- * It is assumed that 'message' is not an 'out' message.
- *
- * 'c_p' is the currently executing process, "tracee" is the traced process
- * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP_MASK), 
- * 'message' must contain a timestamp.
- */
-static void
-send_to_port(Process *c_p, Eterm message, 
-	     Eterm *tracer_pid, Uint *tracee_flags) {
-    Port* trace_port;
-#ifndef ERTS_SMP
-    int ts_type;
-#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE
-    Eterm ts;
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-#endif
-
-    ASSERT(is_internal_port(*tracer_pid));
-#ifdef ERTS_SMP
-    if (is_not_internal_port(*tracer_pid))
-	return;
-
-    trace_port = NULL;
-#else
-
-    trace_port = erts_id2port_sflgs(*tracer_pid,
-				    NULL,
-				    0,
-				    ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
-
-    if (!trace_port) {
-	*tracee_flags &= ~TRACEE_FLAGS;
-	*tracer_pid = NIL;
-	return;
-    }
-
-    /*
-     * Make a fake schedule only if the current process is traced
-     * with 'running' and 'timestamp'.
-     */
-
-    if ( c_p == NULL || 
-	 (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { 
-#endif
-	do_send_to_port(*tracer_pid,
-			trace_port,
-			c_p ? c_p->common.id : NIL,
-			SYS_MSG_TYPE_TRACE,
-			message);
-#ifndef ERTS_SMP
-	erts_port_release(trace_port);
-	return;
-    }
-
-    /*
-     * Note that the process being traced for some type of trace messages
-     * (e.g. getting_linked) need not be the current process. That other
-     * process might not have timestamps enabled.
-     */
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-    /* A fake schedule might be needed.
-     * Create a dummy trace message with timestamp to be
-     * passed to do_send_schedfix_to_port().
-     */
-    ts_type = TFLGS_TS_TYPE(c_p);
-    ts = write_ts(ts_type, local_heap, NULL, NULL);
-
-    trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY;
-    do_send_to_port(*tracer_pid,
-		    trace_port,
-		    c_p ? c_p->common.id : NIL,
-		    SYS_MSG_TYPE_TRACE,
-		    message);
-
-    if (trace_port->control_flags & PORT_CONTROL_FLAG_HEAVY) {
-	/* The driver has just informed us that the last write took a 
-	 * non-neglectible amount of time.
-	 *
-	 * We need to fake some trace messages to compensate for the time the
-	 * current process had to sacrifice for the writing of the previous
-	 * trace message. We pretend that the process got scheduled out
-	 * just after writning the real trace message, and now gets scheduled
-	 * in again.
-	 */
-	do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type);
-    }
-
-    erts_port_release(trace_port);
-
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-#endif
-}
-
 #ifndef ERTS_SMP
 /* Profile send
  * Checks if profiler is port or process
@@ -843,11 +612,11 @@ profile_send(Eterm from, Eterm message) {
 					   0,
 					   ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
 	if (profiler_port) {
-	    do_send_to_port(profiler,
-			    profiler_port,
-			    NIL, /* or current process->common.id */
-			    SYS_MSG_TYPE_SYSPROF,
-			    message);
+	    write_sys_msg_to_port(profiler,
+                                  profiler_port,
+                                  NIL, /* or current process->common.id */
+                                  SYS_MSG_TYPE_SYSPROF,
+                                  message);
 	    erts_port_release(profiler_port);
 	}
     	
@@ -867,154 +636,26 @@ profile_send(Eterm from, Eterm message) {
 	else
 	    msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap);
 
-        erts_queue_message(profile_p, NULL, mp, msg, NIL);
+        erts_queue_message(profile_p, NULL, mp, msg);
     }
 }
 
 #endif
 
-
-/* A fake schedule out/in message pair will be sent,
- * if the driver so requests.
- *
- * 'c_p' is the currently executing process, may be NULL.
- */
-static void
-seq_trace_send_to_port(Process *c_p,
-		       Eterm seq_tracer,
-		       Eterm message)
-{
-    Port* trace_port;
-#ifndef ERTS_SMP
-    int ts_type;
-    Eterm ts;
-#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#endif
-
-    ASSERT(is_internal_port(seq_tracer));
-#ifdef ERTS_SMP
-    if (is_not_internal_port(seq_tracer))
-	return;
-
-    trace_port = NULL;
-#else
-    trace_port = erts_id2port_sflgs(seq_tracer,
-				    NULL,
-				    0,
-				    ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
-    if (!trace_port) {
-	system_seq_tracer = am_false;
-#ifndef ERTS_SMP
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#endif
-	return;
-    }
-
-    if (c_p == NULL
-	|| (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) {
-#endif
-	do_send_to_port(seq_tracer,
-			trace_port,
-			c_p ? c_p->common.id : NIL,
-			SYS_MSG_TYPE_SEQTRACE,
-			message);
-
-#ifndef ERTS_SMP
-	erts_port_release(trace_port);
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-	return;
-    }
-    /* Make a fake schedule only if the current process is traced
-     * with 'running' and 'timestamp'.
-     */
-
-    /* A fake schedule might be needed.
-     * Create a dummy trace message with timestamp to be
-     * passed to do_send_schedfix_to_port().
-     */
-    ts_type = TFLGS_TS_TYPE(c_p);
-    ts = write_ts(ts_type, local_heap, NULL, NULL);
-
-    trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY;
-    do_send_to_port(seq_tracer,
-		    trace_port,
-		    c_p ? c_p->common.id : NIL,
-		    SYS_MSG_TYPE_SEQTRACE,
-		    message);
-
-    if (trace_port->control_flags & PORT_CONTROL_FLAG_HEAVY) {
-	/* The driver has just informed us that the last write took a 
-	 * non-neglectible amount of time.
-	 *
-	 * We need to fake some trace messages to compensate for the time the
-	 * current process had to sacrifice for the writing of the previous
-	 * trace message. We pretend that the process got scheduled out
-	 * just after writing the real trace message, and now gets scheduled
-	 * in again.
-	 */
-	do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type);
-    }
-
-    erts_port_release(trace_port);
-
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-#endif
-}
-
-static ERTS_INLINE void
-send_to_tracer(Process *tracee,
-	       ERTS_TRACER_REF_TYPE tracer_ref,
-	       Eterm msg,
-	       Eterm **hpp,
-	       ErlHeapFragment *bp,
-	       int no_fake_sched)
-{
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(tracee));
-
-    erts_smp_mtx_lock(&smq_mtx);
-
-    if (is_internal_pid(ERTS_TRACER_PROC(tracee))) {
-	PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, bp, tracer_ref);
-	ERTS_ENQ_TRACE_MSG(tracee->common.id, tracer_ref, msg, bp);
-    }
-    else {
-	ASSERT(is_internal_port(ERTS_TRACER_PROC(tracee)));
-	PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, NULL, NULL);
-	send_to_port(no_fake_sched ? NULL : tracee,
-		     msg,
-		     &ERTS_TRACER_PROC(tracee),
-		     &ERTS_TRACE_FLAGS(tracee));
-    }
-
-    erts_smp_mtx_unlock(&smq_mtx);
-
-}
-
 static void
-trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
+trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 {
-#define LOCAL_HEAP_SIZE (5+4+1+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-    DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
-    Eterm tmp, mess, *hp;
-    ErlHeapFragment *bp = NULL;
-    ErlOffHeap *off_heap;
-    ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF;
-    int sched_no, curr_func, to_port, no_fake_sched;
+    Eterm tmp, *hp;
+    int curr_func;
+    ErtsTracerNif *tnif = NULL;
 
-    if (is_nil(ERTS_TRACER_PROC(p)))
+    if (ERTS_TRACER_IS_NIL(ERTS_TRACER(p)))
 	return;
 
-    no_fake_sched = never_fake_sched;
-
     switch (what) {
     case am_out:
     case am_out_exiting:
     case am_out_exited:
-	no_fake_sched = 1;
-	break;
     case am_in:
     case am_in_exiting:
 	break;
@@ -1023,16 +664,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 	break;
     }
 
-    sched_no = IS_TRACED_FL(p, F_TRACE_SCHED_NO);
-    to_port = is_internal_port(ERTS_TRACER_PROC(p));
-
-    if (!to_port) {
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(p),
-			    ERTS_TRACE_FLAGS(p));
-    }
+    if (!is_tracer_proc_enabled(p, locks, &p->common, &tnif, what))
+        return;
 
     if (ERTS_PROC_IS_EXITING(p))
 	curr_func = 0;
@@ -1042,44 +675,16 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 	curr_func = p->current != NULL;
     }
 
-    UseTmpHeap(LOCAL_HEAP_SIZE,p);
-
-    if (to_port)
-	hp = local_heap;
-    else {
-	Uint size = 5;
-	if (curr_func)
-	    size += 4;
-	if (sched_no)
-	    size += 1;
-	size += PATCH_TS_SIZE(p);
-	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
-    }
-
     if (!curr_func) {
 	tmp = make_small(0);
     } else {
+        hp = HAlloc(p, 4);
 	tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2]));
 	hp += 4;
     }
 
-    if (!sched_no) {
-	mess = TUPLE4(hp, am_trace, p->common.id, what, tmp);
-	hp += 5;
-    }
-    else {
-#ifdef ERTS_SMP
-	Eterm sched_id = make_small(p->scheduler_data->no);
-#else
-	Eterm sched_id = make_small(1);
-#endif
-	mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, tmp);
-	hp += 6;
-    }
-
-    send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched);
-    UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
-#undef LOCAL_HEAP_SIZE
+    send_to_tracer_nif(p, &p->common, p->common.id, tnif,
+                       what, tmp, THE_NON_VALUE);
 }
 
 /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
@@ -1089,9 +694,9 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
  * 'out_exiting', or 'out_exited'.
  */
 void
-trace_sched(Process *p, Eterm what)
+trace_sched(Process *p, ErtsProcLocks locks, Eterm what)
 {
-    trace_sched_aux(p, what, 0);
+    trace_sched_aux(p, locks, what);
 }
 
 /* Send {trace_ts, Pid, Send, Msg, DestPid, Timestamp}
@@ -1102,17 +707,13 @@ trace_sched(Process *p, Eterm what)
 void
 trace_send(Process *p, Eterm to, Eterm msg)
 {
-    Eterm operation;
-    unsigned sz_msg;
-    unsigned sz_to;
-    Eterm* hp;
-    Eterm mess;
-    
+    Eterm operation = am_send;
+    ErtsTracerNif *tnif = NULL;
+
     if (!ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)) {
 	return;
     }
 
-    operation = am_send;
     if (is_internal_pid(to)) {
 	if (!erts_proc_lookup(to))
 	    goto send_to_non_existing_process;
@@ -1125,117 +726,36 @@ trace_send(Process *p, Eterm to, Eterm msg)
 	operation = am_atom_put(s, sys_strlen(s));
     }
 
-    if (is_internal_port(ERTS_TRACER_PROC(p))) {
-#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-	hp = local_heap;
-	mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to);
-	hp += 6;
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
-	send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	Uint need;
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(p),
-			    ERTS_TRACE_FLAGS(p));
-
-	sz_msg = size_object(msg);
-	sz_to  = size_object(to);
-	need = sz_msg + sz_to + 6 + PATCH_TS_SIZE(p);
-	
-	hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
-
-	to = copy_struct(to,
-			 sz_to,
-			 &hp,
-			 off_heap);
-	msg = copy_struct(msg,
-			  sz_msg,
-			  &hp,
-			  off_heap);
-	mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to);
-	hp += 6;
-
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
-	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
+    if (is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, operation))
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, operation, msg, to);
 }
 
 /* Send {trace_ts, Pid, receive, Msg, Timestamp}
  * or   {trace, Pid, receive, Msg}
  */
 void
-trace_receive(Process *rp, Eterm msg)
+trace_receive(Process *c_p, Eterm msg)
 {
-    Eterm mess;
-    size_t sz_msg;
-    Eterm* hp;
-
-    if (is_internal_port(ERTS_TRACER_PROC(rp))) {
-#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-	hp = local_heap;
-	mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg);
-	hp += 5;
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, NULL, NULL);
-	send_to_port(rp, mess, &ERTS_TRACER_PROC(rp), &ERTS_TRACE_FLAGS(rp));
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	Uint hsz;
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(rp)));
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(rp),
-			    ERTS_TRACE_FLAGS(rp));
-
-	sz_msg = size_object(msg);
-
-	hsz = sz_msg + 5 + PATCH_TS_SIZE(rp);
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref);
-
-	msg = copy_struct(msg, sz_msg, &hp, off_heap);
-	mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg);
-	hp += 5;
-
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, bp, tracer_ref);
-	ERTS_ENQ_TRACE_MSG(rp->common.id, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
+    ErtsTracerNif *tnif = NULL;
+    if (is_tracer_proc_enabled(NULL, 0, &c_p->common,
+                               &tnif, am_receive))
+        send_to_tracer_nif(NULL, &c_p->common, c_p->common.id,
+                           tnif, am_receive, msg, THE_NON_VALUE);
 }
 
 int
 seq_trace_update_send(Process *p)
 {
-    Eterm seq_tracer = erts_get_system_seq_tracer();
+    ErtsTracer seq_tracer = erts_get_system_seq_tracer();
     ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p))));
-    if ((p->common.id == seq_tracer) || have_no_seqtrace(SEQ_TRACE_TOKEN(p))) {
+    if (have_no_seqtrace(SEQ_TRACE_TOKEN(p)) ||
+        (seq_tracer != NIL &&
+         call_enabled_tracer( NULL, seq_tracer, NULL, am_trace_status,
+                              p ? p->common.id : am_undefined) != am_trace)
+#ifdef USE_VM_PROBES
+	 || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag)
+#endif
+	 ) {
 	return 0;
     }
     SEQ_TRACE_TOKEN_SENDER(p) = p->common.id;
@@ -1263,20 +783,28 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 			 Eterm receiver, Process *process, Eterm exitfrom)
 {
     Eterm mess;
-    ErlHeapFragment* bp;
     Eterm* hp;
     Eterm label;
     Eterm lastcnt_serial;
     Eterm type_atom;
-    int sz_exit;
-    Eterm seq_tracer;
-    int ts_type;
+    ErtsTracer seq_tracer;
+    int seq_tracer_flags = 0;
+#define LOCAL_HEAP_SIZE (64)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 
     seq_tracer = erts_get_system_seq_tracer();
 
     ASSERT(is_tuple(token) || is_nil(token));
-    if (SEQ_TRACE_T_SENDER(token) == seq_tracer || token == NIL ||
-	(process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE)) {
+    if (token == NIL || (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE) ||
+        ERTS_TRACER_IS_NIL(seq_tracer) ||
+        call_enabled_tracer(
+            NULL, seq_tracer, NULL, am_trace_status,
+            process ? process->common.id : am_undefined) != am_trace) {
+	return;
+    }
+
+    if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & type) == 0) {
+	/* No flags set, nothing to do */
 	return;
     }
 
@@ -1289,151 +817,29 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	return;			/* To avoid warning */
     }
 
-    if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & type) == 0) {
-	/* No flags set, nothing to do */
-	return;
-    }
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
-    if (seq_tracer == am_false) {
-	return;			/* no need to send anything */
+    hp = local_heap;
+    label = SEQ_TRACE_T_LABEL(token);
+    lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token),
+                            SEQ_TRACE_T_SERIAL(token));
+    hp += 3;
+    if (exitfrom != NIL) {
+        msg = TUPLE3(hp, am_EXIT, exitfrom, msg);
+        hp += 4;
     }
+    mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token),
+                  receiver, msg);
+    hp += 6;
 
-    ts_type = ERTS_SEQTFLGS2TSTYPE(unsigned_val(SEQ_TRACE_T_FLAGS(token)));
-
-    if (is_internal_port(seq_tracer)) {
-#define LOCAL_HEAP_SIZE (60 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+    seq_tracer_flags |=  ERTS_SEQTFLGS2TFLGS(unsigned_val(SEQ_TRACE_T_FLAGS(token)));
 
-	hp = local_heap;
-	label = SEQ_TRACE_T_LABEL(token);
-	lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token),
-				SEQ_TRACE_T_SERIAL(token));
-	hp += 3;
-	if (exitfrom != NIL) {
-	    msg = TUPLE3(hp, am_EXIT, exitfrom, msg);
-	    hp += 4;
-	}
-	mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token),
-		      receiver, msg);
-	hp += 6;
+    send_to_tracer_nif_raw(NULL, process, seq_tracer, seq_tracer_flags,
+                           label, NULL, am_seq_trace, mess,
+                           THE_NON_VALUE, am_true);
 
-	erts_smp_mtx_lock(&smq_mtx);
-	if (!ts_type) {
-	    mess = TUPLE3(hp, am_seq_trace, label, mess);
-	    seq_trace_send_to_port(NULL, seq_tracer, mess);
-	} else {
-	    mess = TUPLE4(hp, am_seq_trace, label, mess,
-			  NIL /* Will be overwritten by timestamp */);
-	    hp += 5;
-	    hp[-1] = write_ts(ts_type, hp, NULL, NULL);
-	    seq_trace_send_to_port(process, seq_tracer, mess);
-	}
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-#ifndef ERTS_SMP
-	Process* tracer;
-#endif
-	Eterm sender_copy;
-	Eterm receiver_copy;
-	Eterm m2;
-	Uint sz_label, sz_lastcnt_serial, sz_msg, sz_ts, sz_sender,
-	    sz_exitfrom, sz_receiver;
-
-	ASSERT(is_internal_pid(seq_tracer));
-
-#ifndef ERTS_SMP
-
-	tracer = erts_proc_lookup(seq_tracer);
-	if (!tracer) {
-	    system_seq_tracer = am_false;
-	    return; /* no need to send anything */
-	}
-#endif
-	if (receiver == seq_tracer) {
-	    return; /* no need to send anything */
-	}
-
-	sz_label = size_object(SEQ_TRACE_T_LABEL(token));
-	sz_sender = size_object(SEQ_TRACE_T_SENDER(token));
-	sz_receiver = size_object(receiver);
-	sz_lastcnt_serial = 3; /* TUPLE2 */
-	sz_msg = size_object(msg);
-
-	sz_ts = patch_ts_size(ts_type);
-	if (exitfrom != NIL) {
-	    sz_exit = 4; /* create {'EXIT',exitfrom,msg} */
-	    sz_exitfrom = size_object(exitfrom);
-	}
-	else {
-	    sz_exit = 0;
-	    sz_exitfrom = 0;
-	}
-	bp = new_message_buffer(4 /* TUPLE3 */ + sz_ts + 6 /* TUPLE5 */ 
-				+ sz_lastcnt_serial + sz_label + sz_msg
-				+ sz_exit + sz_exitfrom
-				+ sz_sender + sz_receiver);
-	hp = bp->mem;
-	label = copy_struct(SEQ_TRACE_T_LABEL(token), sz_label, &hp, &bp->off_heap);
-	lastcnt_serial = TUPLE2(hp,SEQ_TRACE_T_LASTCNT(token),SEQ_TRACE_T_SERIAL(token));
-	hp += 3;
-	m2 = copy_struct(msg, sz_msg, &hp, &bp->off_heap);
-	if (sz_exit) {
-	    Eterm exitfrom_copy = copy_struct(exitfrom,
-					      sz_exitfrom,
-					      &hp,
-					      &bp->off_heap);
-	    m2 = TUPLE3(hp, am_EXIT, exitfrom_copy, m2);
-	    hp += 4;
-	}
-	sender_copy = copy_struct(SEQ_TRACE_T_SENDER(token),
-				  sz_sender,
-				  &hp,
-				  &bp->off_heap);
-	receiver_copy = copy_struct(receiver,
-				    sz_receiver,
-				    &hp,
-				    &bp->off_heap);
-	mess = TUPLE5(hp,
-		      type_atom,
-		      lastcnt_serial,
-		      sender_copy,
-		      receiver_copy,
-		      m2);
-	hp += 6;
-
-	erts_smp_mtx_lock(&smq_mtx);
-
-	if (!ts_type)
-	    mess = TUPLE3(hp, am_seq_trace, label, mess);
-	else {
-	    mess = TUPLE4(hp, am_seq_trace, label, mess,
-			  NIL /* Will be overwritten by timestamp */);
-	    hp += 5;
-	    /* Write timestamp in element 6 of the 'msg' tuple */
-	    hp[-1] = write_ts(ts_type, hp, bp,
-#ifndef ERTS_SMP
-			      tracer
-#else
-			      NULL
-#endif
-		);
-	}
-
-#ifdef ERTS_SMP
-	enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-#else
-        /* trace_token must be NIL here */
-	{
-	    ErtsMessage *mp = erts_alloc_message(0, NULL);
-	    mp->data.heap_frag = bp;
-	    erts_queue_message(tracer, NULL, mp, mess, NIL);
-	}
-#endif
-    }
 }
 
 /* Send {trace_ts, Pid, return_to, {Mod, Func, Arity}, Timestamp}
@@ -1442,63 +848,19 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 void 
 erts_trace_return_to(Process *p, BeamInstr *pc)
 {
-#define LOCAL_HEAP_SIZE (4+5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-    Eterm* hp;
     Eterm mfa;
-    Eterm mess;
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 
     BeamInstr *code_ptr = find_function_from_pc(pc);
 
 
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-    hp = local_heap;
-
     if (!code_ptr) {
 	mfa = am_undefined;
     } else {
+        Eterm *hp = HAlloc(p, 4);
 	mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2]));
-	hp += 4;
     }
-	
-    mess = TUPLE4(hp, am_trace, p->common.id, am_return_to, mfa);
-    hp += 5;
-
-    erts_smp_mtx_lock(&smq_mtx);
-
-    PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
-
-    if (is_internal_port(ERTS_TRACER_PROC(p))) {
-	send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
-    } else {
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-	unsigned size;
-
-	/*
-	 * Find the tracer.
-	 */
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(p),
-			    ERTS_TRACE_FLAGS(p));
-
-	size = size_object(mess);
 
-	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
-	
-	/*
-	 * Copy the trace message into the buffer and enqueue it.
-	 */
-	mess = copy_struct(mess, size, &hp, off_heap);
-	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
-    }
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-    erts_smp_mtx_unlock(&smq_mtx);
+    SEND_TO_TRACER(p, am_return_to, mfa);
 }
 
 
@@ -1506,114 +868,53 @@ erts_trace_return_to(Process *p, BeamInstr *pc)
  * or   {trace, Pid, return_from, {Mod, Name, Arity}, Retval}
  */
 void
-erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
+erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer)
 {
     Eterm* hp;
-    Eterm mfa;
-    Eterm mess;
-    Eterm mod, name;
+    Eterm mfa, mod, name;
     int arity;
     Uint meta_flags, *tracee_flags;
-    int ts_type;
-#ifdef ERTS_SMP
-    Eterm tracee;
-#endif
-    
-    ASSERT(tracer_pid);
-    if (*tracer_pid == am_true) {
+
+    ASSERT(tracer);
+    if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) {
 	/* Breakpoint trace enabled without specifying tracer =>
 	 *   use process tracer and flags
 	 */
-	tracer_pid = &ERTS_TRACER_PROC(p);
+	tracer = &ERTS_TRACER(p);
     }
-    if (is_nil(*tracer_pid)) {
+    if (ERTS_TRACER_IS_NIL(*tracer)) {
 	/* Trace disabled */
 	return;
     }
-    ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid));
-    if (*tracer_pid == p->common.id) {
-	/* Do not generate trace messages to oneself */
-	return;
-    }
-    if (tracer_pid == &ERTS_TRACER_PROC(p)) {
+    ASSERT(IS_TRACER_VALID(*tracer));
+    if (tracer == &ERTS_TRACER(p)) {
 	/* Tracer specified in process structure =>
 	 *   non-breakpoint trace =>
 	 *     use process flags
 	 */
 	tracee_flags = &ERTS_TRACE_FLAGS(p);
-#ifdef ERTS_SMP
-	tracee = p->common.id;
-#endif
+        if (! (*tracee_flags & F_TRACE_CALLS)) {
+            return;
+        }
     } else {
 	/* Tracer not specified in process structure =>
 	 *   tracer specified in breakpoint =>
 	 *     meta trace =>
 	 *       use fixed flag set instead of process flags
-	 */	    
+	 */
 	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
-#ifdef ERTS_SMP
-	tracee = NIL;
-#endif
-    }
-    if (! (*tracee_flags & F_TRACE_CALLS)) {
-	return;
     }
-    
+
     mod = fi[0];
     name = fi[1];
     arity = fi[2];
-    
-    ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags);
-
-    if (is_internal_port(*tracer_pid)) {
-#define LOCAL_HEAP_SIZE (4+6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-	hp = local_heap;
-	mfa = TUPLE3(hp, mod, name, make_small(arity));
-	hp += 4;
-	mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval);
-	hp += 6;
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(ts_type, mess, hp, NULL, NULL);
-	send_to_port(p, mess, tracer_pid, tracee_flags);
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-	unsigned size;
-	unsigned retval_size;
-
-	ASSERT(is_internal_pid(*tracer_pid));
-
-	ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags);
-
-	retval_size = size_object(retval);
-	size = 6 + 4 + retval_size + patch_ts_size(ts_type);
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
 
-	/*
-	 * Build the trace tuple and put it into receive queue of the tracer process.
-	 */
-	
-	mfa = TUPLE3(hp, mod, name, make_small(arity));
-	hp += 4;
-	retval = copy_struct(retval, retval_size, &hp, off_heap);
-	mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval);
-	hp += 6;
-
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(ts_type, mess, hp, bp, tracer_ref);
-
-	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
+    hp = HAlloc(p, 4);
+    mfa = TUPLE3(hp, mod, name, make_small(arity));
+    hp += 4;
+    send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
+                           NULL, am_return_from, mfa, retval, am_true);
 }
 
 /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, 
@@ -1625,116 +926,50 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
  */
 void
 erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
-		     Eterm *tracer_pid)
+		     ErtsTracer *tracer)
 {
     Eterm* hp;
-    Eterm mfa_tuple;
-    Eterm cv;
-    Eterm mess;
+    Eterm mfa_tuple, cv;
     Uint meta_flags, *tracee_flags;
-    int ts_type;
-#ifdef ERTS_SMP
-    Eterm tracee;
-#endif
-    
-    ASSERT(tracer_pid);
-    if (*tracer_pid == am_true) {
+
+    ASSERT(tracer);
+    if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) {
 	/* Breakpoint trace enabled without specifying tracer =>
 	 *   use process tracer and flags
 	 */
-	tracer_pid = &ERTS_TRACER_PROC(p);
+	tracer = &ERTS_TRACER(p);
     }
-    if (is_nil(*tracer_pid)) {
+    if (ERTS_TRACER_IS_NIL(*tracer)) {
 	/* Trace disabled */
 	return;
     }
-    ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid));
-    if (*tracer_pid == p->common.id) {
-	/* Do not generate trace messages to oneself */
-	return;
-    }
-    if (tracer_pid == &ERTS_TRACER_PROC(p)) {
+    ASSERT(IS_TRACER_VALID(*tracer));
+    if (tracer == &ERTS_TRACER(p)) {
 	/* Tracer specified in process structure =>
 	 *   non-breakpoint trace =>
 	 *     use process flags
 	 */
 	tracee_flags = &ERTS_TRACE_FLAGS(p);
-#ifdef ERTS_SMP
-	tracee = p->common.id;
-#endif
-	if (! (*tracee_flags & F_TRACE_CALLS)) {
-	    return;
-	}
+        if (! (*tracee_flags & F_TRACE_CALLS)) {
+            return;
+        }
     } else {
 	/* Tracer not specified in process structure =>
 	 *   tracer specified in breakpoint =>
 	 *     meta trace =>
 	 *       use fixed flag set instead of process flags
-	 */	    
+	 */
 	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
-#ifdef ERTS_SMP
-	tracee = NIL;
-#endif
     }
-    
-    ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags);
-
-    if (is_internal_port(*tracer_pid)) {
-#define LOCAL_HEAP_SIZE (4+3+6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-	hp = local_heap;
-	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2]));
-	hp += 4;
-	cv = TUPLE2(hp, class, value);
-	hp += 3;
-	mess = TUPLE5(hp, am_trace, p->common.id, am_exception_from, mfa_tuple, cv);
-	hp += 6;
-	ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE);
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(ts_type, mess, hp, NULL, NULL);
-	send_to_port(p, mess, tracer_pid, tracee_flags);
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-	unsigned size;
-	unsigned value_size;
-
-	ASSERT(is_internal_pid(*tracer_pid));
-
-	ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags);
-	
-	value_size = size_object(value);
-	size = 6 + 4 + 3 + value_size + patch_ts_size(ts_type);
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
-
-	/*
-	 * Build the trace tuple and put it into receive queue of the tracer process.
-	 */
-	
-	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm) mfa[2]));
-	hp += 4;
-	value = copy_struct(value, value_size, &hp, off_heap);
-	cv = TUPLE2(hp, class, value);
-	hp += 3;
-	mess = TUPLE5(hp, am_trace, p->common.id, 
-		      am_exception_from, mfa_tuple, cv);
-	hp += 6;
-
-	erts_smp_mtx_lock(&smq_mtx);
 
-	PATCH_TS(ts_type, mess, hp, bp, tracer_ref);
-
-	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
+    hp = HAlloc(p, 7);;
+    mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2]));
+    hp += 4;
+    cv = TUPLE2(hp, class, value);
+    hp += 3;
+    send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
+                           NULL, am_exception_from, mfa_tuple, cv, am_true);
 }
 
 /*
@@ -1753,7 +988,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
  */
 Uint32
 erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
-		Eterm* args, int local, Eterm *tracer_pid)
+		Eterm* args, int local, ErtsTracer *tracer)
 {
     Eterm* hp;
     Eterm mfa_tuple;
@@ -1761,55 +996,50 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
     int i;
     Uint32 return_flags;
     Eterm pam_result = am_true;
-    Eterm mess;
     Uint meta_flags, *tracee_flags;
-    int ts_type;
-#ifdef ERTS_SMP
-    Eterm tracee;
-#endif
+    ErtsTracerNif *tnif = NULL;
     Eterm transformed_args[MAX_ARG];
-    DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p);
+    ErtsTracer pre_ms_tracer = erts_tracer_nil;
 
-    ASSERT(tracer_pid);
-    if (*tracer_pid == am_true) {
-	/* Breakpoint trace enabled without specifying tracer =>
+    ASSERT(tracer);
+    if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) {
+        /* Breakpoint trace enabled without specifying tracer =>
 	 *   use process tracer and flags
 	 */
-	tracer_pid = &ERTS_TRACER_PROC(p);
-    } 
-    if (is_nil(*tracer_pid)) {
-	/* Trace disabled */
-	return 0;
+        tracer = &ERTS_TRACER(p);
     }
-    ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid));
-    if (*tracer_pid == p->common.id) {
-	/* Do not generate trace messages to oneself */
+    if (ERTS_TRACER_IS_NIL(*tracer)) {
+	/* Trace disabled */
 	return 0;
     }
-    if (tracer_pid == &ERTS_TRACER_PROC(p)) {
+    ASSERT(IS_TRACER_VALID(*tracer));
+    if (tracer == &ERTS_TRACER(p)) {
 	/* Tracer specified in process structure =>
 	 *   non-breakpoint trace =>
 	 *     use process flags
 	 */
 	tracee_flags = &ERTS_TRACE_FLAGS(p);
-#ifdef ERTS_SMP
-	tracee = p->common.id;
-#endif
+        if (!is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, am_call)) {
+            return 0;
+        }
     } else {
 	/* Tracer not specified in process structure =>
 	 *   tracer specified in breakpoint =>
 	 *     meta trace =>
 	 *       use fixed flag set instead of process flags
-	 */	    
-	if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) {
-	    /* No trace messages for sensitive processes. */
-	    return 0;
-	}
+	 */
+        if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) {
+            /* No trace messages for sensitive processes. */
+            return 0;
+        }
 	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
-#ifdef ERTS_SMP
-	tracee = NIL;
-#endif
+        switch (call_enabled_tracer(p, *tracer, &tnif, am_call, p->common.id)) {
+        default:
+        case am_remove: *tracer = erts_tracer_nil;
+        case am_discard: return 0;
+        case am_trace: break;
+        }
     }
 
     /*
@@ -1820,18 +1050,13 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
      * temporarily convert any match contexts to sub binaries.
      */
     arity = (Eterm) mfa[2];
-    UseTmpHeap(ERL_SUB_BIN_SIZE,p);
-#ifdef DEBUG
-    sub_bin_heap->thing_word = 0;
-#endif
     for (i = 0; i < arity; i++) {
 	Eterm arg = args[i];
 	if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) {
 	    ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg);
 	    ErlBinMatchBuffer* mb = &ms->mb;
 	    Uint bit_size;
-
-	    ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */
+            ErlSubBin *sub_bin_heap = (ErlSubBin *)HAlloc(p, ERL_SUB_BIN_SIZE);
 
 	    bit_size = mb->size - mb->offset;
 	    sub_bin_heap->thing_word = HEADER_SUB_BIN;
@@ -1848,275 +1073,98 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
     }
     args = transformed_args;
 
-    ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags);
-
-    if (is_internal_port(*tracer_pid)) {
-	Eterm local_heap[64+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG];
-	hp = local_heap;
+    /*
+     * If there is a PAM program, run it.  Return if it fails.
+     *
+     * Some precedence rules:
+     *
+     * - No proc flags, e.g 'silent' or 'return_to'
+     *   has any effect on meta trace.
+     * - The 'silent' process trace flag silences all call
+     *   related messages, e.g 'call', 'return_to' and 'return_from'.
+     * - The {message,_} PAM function does not affect {return_trace}.
+     * - The {message,false} PAM function shall give the same
+     *   'call' trace message as no PAM match.
+     * - The {message,true} PAM function shall give the same
+     *   'call' trace message as a nonexistent PAM program.
+     */
 
-	if (!erts_is_valid_tracer_port(*tracer_pid)) {
-#ifdef ERTS_SMP
-	    ASSERT(is_nil(tracee) || tracer_pid == &ERTS_TRACER_PROC(p));
-	    if (is_not_nil(tracee))
-		erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
-	    *tracee_flags &= ~TRACEE_FLAGS;
-	    *tracer_pid = NIL;
-#ifdef ERTS_SMP
-	    if (is_not_nil(tracee))
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
-	    UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-	    return 0;
-	}
-	
-	/*
-	 * If there is a PAM program, run it.  Return if it fails.
-	 *
-	 * Some precedence rules:
-	 * 
-	 * - No proc flags, e.g 'silent' or 'return_to' 
-	 *   has any effect on meta trace.
-	 * - The 'silent' process trace flag silences all call 
-	 *   related messages, e.g 'call', 'return_to' and 'return_from'.
-	 * - The {message,_} PAM function does not affect {return_trace}.
-	 * - The {message,false} PAM function shall give the same
-	 *   'call' trace message as no PAM match.
-	 * - The {message,true} PAM function shall give the same
-	 *   'call' trace message as a nonexistent PAM program.
-	 */
-	
-	/* BEGIN this code should be the same for port and pid trace */
-	return_flags = 0;
-	if (match_spec) {
-	    pam_result = erts_match_set_run(p, match_spec, args, arity,
-					    ERTS_PAM_TMP_RESULT, &return_flags);
-	    if (is_non_value(pam_result)) {
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return 0;
-	    }
-	}
-	if (tracee_flags == &meta_flags) {
-	    /* Meta trace */
-	    if (pam_result == am_false) {
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return return_flags;
-	    }
-	} else {
-	    /* Non-meta trace */
-	    if (*tracee_flags & F_TRACE_SILENT) { 
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return 0;
-	    }
-	    if (pam_result == am_false) {
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return return_flags;
-	    }
-	    if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
-		return_flags |= MATCH_SET_RETURN_TO_TRACE;
-	    }
-	}
-	/* END this code should be the same for port and pid trace */
-	
-	/*
-	 * Build the the {M,F,A} tuple in the local heap. 
-	 * (A is arguments or arity.)
-	 */
-	
-	if (*tracee_flags & F_TRACE_ARITY_ONLY) {
-	    mfa_tuple = make_small(arity);
-	} else {
-	    mfa_tuple = NIL;
-	    for (i = arity-1; i >= 0; i--) {
-		mfa_tuple = CONS(hp, args[i], mfa_tuple);
-		hp += 2;
-	    }
-	}
-	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
-	hp += 4;
-	
-	/*
-	 * Build the trace tuple and send it to the port.
-	 */
-	
-	mess = TUPLE4(hp, am_trace, p->common.id, am_call, mfa_tuple);
-	hp += 5;
-	if (pam_result != am_true) {
-	    hp[-5] = make_arityval(5);
-	    *hp++ = pam_result;
-	}
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(ts_type, mess, hp, NULL, NULL);
-	send_to_port(p, mess, tracer_pid, tracee_flags);
-	erts_smp_mtx_unlock(&smq_mtx);
-	erts_match_set_release_result(p);
-	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-	return *tracer_pid == NIL ? 0 : return_flags;
+    return_flags = 0;
+    if (match_spec) {
+        /* we have to make a copy of the tracer here as the match spec
+           may remove it, and we still want to generate a trace message */
+        erts_tracer_update(&pre_ms_tracer, *tracer);
+        tracer = &pre_ms_tracer;
+        pam_result = erts_match_set_run(p, match_spec, args, arity,
+                                        ERTS_PAM_TMP_RESULT, &return_flags);
+        if (is_non_value(pam_result)) {
+            erts_match_set_release_result(p);
+            UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
+            ERTS_TRACER_CLEAR(&pre_ms_tracer);
+            return 0;
+        }
+    }
 
+    if (tracee_flags == &meta_flags) {
+        /* Meta trace */
+        if (pam_result == am_false) {
+            erts_match_set_release_result(p);
+            UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
+            ERTS_TRACER_CLEAR(&pre_ms_tracer);
+            return return_flags;
+        }
     } else {
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	Process *tracer;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-#ifdef ERTS_SMP
-	Eterm tpid;
-#endif
-	unsigned size;
-	unsigned sizes[MAX_ARG];
-	unsigned pam_result_size = 0;
-	int invalid_tracer;
+        /* Non-meta trace */
+        if (*tracee_flags & F_TRACE_SILENT) {
+            erts_match_set_release_result(p);
+            UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
+            ERTS_TRACER_CLEAR(&pre_ms_tracer);
+            return 0;
+        }
+        if (pam_result == am_false) {
+            erts_match_set_release_result(p);
+            UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
+            ERTS_TRACER_CLEAR(&pre_ms_tracer);
+            return return_flags;
+        }
+        if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
+            return_flags |= MATCH_SET_RETURN_TO_TRACE;
+        }
+    }
 
-	ASSERT(is_internal_pid(*tracer_pid));
-	
-	tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
-			       *tracer_pid, ERTS_PROC_LOCK_STATUS);
-	if (!tracer)
-	    invalid_tracer = 1;
-	else {
-	    invalid_tracer = !(ERTS_TRACE_FLAGS(tracer) & F_TRACER);
-	    erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS);
-	}
+    ASSERT(!ERTS_TRACER_IS_NIL(*tracer));
 
-	if (invalid_tracer) {
-#ifdef ERTS_SMP
-	    ASSERT(is_nil(tracee)
-		   || tracer_pid == &ERTS_TRACER_PROC(p));
-	    if (is_not_nil(tracee))
-		erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
-	    *tracee_flags &= ~TRACEE_FLAGS;
-	    *tracer_pid = NIL;
-#ifdef ERTS_SMP
-	    if (is_not_nil(tracee))
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
-	    UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-	    return 0;
-	}
-
-#ifdef ERTS_SMP
-	tpid = *tracer_pid; /* Need to save tracer pid,
-			       since *tracer_pid might
-			       be reset by erts_match_set_run() */
-	tracer_ref = tpid;
-#else
-	tracer_ref = tracer;
-#endif
-	
-	/*
-	 * If there is a PAM program, run it.  Return if it fails.
-	 *
-	 * See the rules above in the port trace code.
-	 */
-	
-	/* BEGIN this code should be the same for port and pid trace */
-	return_flags = 0;
-	if (match_spec) {
-	    pam_result = erts_match_set_run(p, match_spec, args, arity,
-					    ERTS_PAM_TMP_RESULT, &return_flags);
-	    if (is_non_value(pam_result)) {
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return 0;
-	    }
-	}
-	if (tracee_flags == &meta_flags) {
-	    /* Meta trace */
-	    if (pam_result == am_false) {
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return return_flags;
-	    }
-	} else {
-	    /* Non-meta trace */
-	    if (*tracee_flags & F_TRACE_SILENT) { 
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return 0;
-	    }
-	    if (pam_result == am_false) {
-		erts_match_set_release_result(p);
-		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-		return return_flags;
-	    }
-	    if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
-		return_flags |= MATCH_SET_RETURN_TO_TRACE;
-	    }
-	}
-	/* END this code should be the same for port and pid trace */
-	
-	/*
-	 * Calculate number of words needed on heap.
-	 */
-	
-	size = 4 + 5;		/* Trace tuple + MFA tuple. */
-	if (! (*tracee_flags & F_TRACE_ARITY_ONLY)) {
-	    size += 2*arity;
-	    for (i = arity-1; i >= 0; i--) {
-		sizes[i] = size_object(args[i]);
-		size += sizes[i];
-	    }
-	}
-	size += patch_ts_size(ts_type);
-	if (pam_result != am_true) {
-	    pam_result_size = size_object(pam_result);
-	    size += 1 + pam_result_size;
-	    /* One element in trace tuple + term size. */
-	}
-	
-	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
+    /*
+     * Build the the {M,F,A} tuple in the local heap.
+     * (A is arguments or arity.)
+     */
 
-	/*
-	 * Build the the {M,F,A} tuple in the message buffer. 
-	 * (A is arguments or arity.)
-	 */
-	
-	if (*tracee_flags & F_TRACE_ARITY_ONLY) {
-	    mfa_tuple = make_small(arity);
-	} else {
-	    mfa_tuple = NIL;
-	    for (i = arity-1; i >= 0; i--) {
-		Eterm term = copy_struct(args[i], sizes[i], &hp, off_heap);
-		mfa_tuple = CONS(hp, term, mfa_tuple);
-		hp += 2;
-	    }
-	}
-	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
-	hp += 4;
-	
-	/*
-	 * Copy the PAM result (if any) onto the heap.
-	 */
-	
-	if (pam_result != am_true) {
-	    pam_result = copy_struct(pam_result, pam_result_size, &hp, off_heap);
-	}
 
-	erts_match_set_release_result(p);
+    if (*tracee_flags & F_TRACE_ARITY_ONLY) {
+        hp = HAlloc(p, 4);
+        mfa_tuple = make_small(arity);
+    } else {
+        hp = HAlloc(p, 4 + arity * 2);
+        mfa_tuple = NIL;
+        for (i = arity-1; i >= 0; i--) {
+            mfa_tuple = CONS(hp, args[i], mfa_tuple);
+            hp += 2;
+        }
+    }
+    mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
+    hp += 4;
 
-	/*
-	 * Build the trace tuple and enqueue it.
-	 */
-	
-	mess = TUPLE4(hp, am_trace, p->common.id, am_call, mfa_tuple);
-	hp += 5;
-	if (pam_result != am_true) {
-	    hp[-5] = make_arityval(5);
-	    *hp++ = pam_result;
-	}
+    /*
+     * Build the trace tuple and send it to the port.
+     */
+    send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
+                           tnif, am_call, mfa_tuple, THE_NON_VALUE, pam_result);
+    erts_match_set_release_result(p);
 
-	erts_smp_mtx_lock(&smq_mtx);
+    if (match_spec && tracer == &pre_ms_tracer)
+        ERTS_TRACER_CLEAR(&pre_ms_tracer);
 
-	PATCH_TS(ts_type, mess, hp, bp, tracer_ref);
-	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-	return return_flags;
-    }
+    return return_flags;
 }
 
 /* Sends trace message:
@@ -2128,69 +1176,13 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
  * 't_p' is the traced process.
  */
 void
-trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
+trace_proc(Process *c_p, ErtsProcLocks c_p_locks,
+           Process *t_p, Eterm what, Eterm data)
 {
-    Eterm mess;
-    Eterm* hp;
-    int need;
-
-    ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0)
-		       || erts_thr_progress_is_blocking());
-    if (is_internal_port(ERTS_TRACER_PROC(t_p))) {
-#define LOCAL_HEAP_SIZE (5+5)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-
-	hp = local_heap;
-	mess = TUPLE4(hp, am_trace, t_p->common.id, what, data);
-	hp += 5;
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL);
-	send_to_port(
-#ifndef ERTS_SMP
-	    /* No fake schedule out and in again after an exit */
-	    what == am_exit ? NULL : c_p,
-#else
-	    /* Fake schedule out and in are never sent when smp enabled */
-	    c_p,
-#endif
-	    mess,
-	    &ERTS_TRACER_PROC(t_p),
-	    &ERTS_TRACE_FLAGS(t_p));
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	Eterm tmp;
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-	size_t sz_data;
-
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p)));
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(t_p),
-			    ERTS_TRACE_FLAGS(t_p));
-
-	sz_data = size_object(data);
-
-	need = sz_data + 5 + PATCH_TS_SIZE(t_p);
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
-
-	tmp = copy_struct(data, sz_data, &hp, off_heap);
-	mess = TUPLE4(hp, am_trace, t_p->common.id, what, tmp);
-	hp += 5;
-
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref);
-
-	ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
+    ErtsTracerNif *tnif = NULL;
+    if (is_tracer_proc_enabled(c_p, c_p_locks, &t_p->common, &tnif, what))
+        send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif,
+                           what, data, THE_NON_VALUE);
 }
 
 
@@ -2202,62 +1194,20 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
  * and 'args' may be a deep term.
  */
 void
-trace_proc_spawn(Process *p, Eterm pid, 
+trace_proc_spawn(Process *p, Eterm pid,
 		 Eterm mod, Eterm func, Eterm args)
 {
-    Eterm mfa;
-    Eterm mess;
-    Eterm* hp;
-
-    if (is_internal_port(ERTS_TRACER_PROC(p))) {
-#define LOCAL_HEAP_SIZE (4+6+5)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-	hp = local_heap;
-	mfa = TUPLE3(hp, mod, func, args);
-	hp += 4;
-	mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, pid, mfa);
-	hp += 6;
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
-	send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	Eterm tmp;
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-	size_t sz_args, sz_pid;
-	Uint need;
-
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(p),
-			    ERTS_TRACE_FLAGS(p));
-
-	sz_args = size_object(args);
-	sz_pid = size_object(pid);
-	need = sz_args + 4 + 6 + PATCH_TS_SIZE(p);
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref);
-
-	tmp = copy_struct(args, sz_args, &hp, off_heap);
-	mfa = TUPLE3(hp, mod, func, tmp);
-	hp += 4;
-	tmp = copy_struct(pid, sz_pid, &hp, off_heap);
-	mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, tmp, mfa);
-	hp += 6;
+    ErtsTracerNif *tnif = NULL;
+    if (is_tracer_proc_enabled(p, ERTS_PROC_LOCKS_ALL &
+                               ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE),
+                               &p->common, &tnif, am_spawn)) {
+        Eterm mfa;
+        Eterm* hp;
 
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
-
-	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
+        hp = HAlloc(p, 4);
+        mfa = TUPLE3(hp, mod, func, args);
+        hp += 4;
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, am_spawn, pid, mfa);
     }
 }
 
@@ -2288,69 +1238,25 @@ void save_calls(Process *p, Export *e)
  * where 'HeapSize', 'OldHeapSize', 'StackSize', 'RecentSize and 'MbufSize'
  * are all small (atomic) integers.
  */
-void
-trace_gc(Process *p, Eterm what)
-{
-    ErlHeapFragment *bp = NULL;
-    ErlOffHeap *off_heap;
-    ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF;	/* Initialized
-								   to eliminate
-								   compiler
-								   warning */
-    Eterm* hp;
-    Eterm msg = NIL;
-    Uint size;
-
-#define LOCAL_HEAP_SIZE						\
-    (ERTS_PROCESS_GC_INFO_MAX_SIZE) +                           \
-	5/*4-tuple */ + ERTS_TRACE_PATCH_TS_MAX_SIZE
-    DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
-
-    UseTmpHeap(LOCAL_HEAP_SIZE,p);
-
-    if (is_internal_port(ERTS_TRACER_PROC(p))) {
-	hp = local_heap;
-#ifdef DEBUG
-	size = 0;
-        (void) erts_process_gc_info(p, &size, NULL);
-
-	size += 5/*4-tuple*/ + PATCH_TS_SIZE(p);
-#endif
-    } else {
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
+void
+trace_gc(Process *p, Eterm what)
+{
+    ErtsTracerNif *tnif = NULL;
+    Eterm* hp;
+    Eterm msg = NIL;
+    Uint size = 0;
 
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(p),
-			    ERTS_TRACE_FLAGS(p));
+    if (is_tracer_proc_enabled(
+            p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, what)) {
 
-	size = 0;
         (void) erts_process_gc_info(p, &size, NULL);
+        hp = HAlloc(p, size);
 
-	size += 5/*4-tuple*/ + PATCH_TS_SIZE(p);
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref);
-    }
-
-    ASSERT(size <= LOCAL_HEAP_SIZE);
+        msg = erts_process_gc_info(p, NULL, &hp);
 
-    msg = erts_process_gc_info(p, NULL, &hp);
-
-    msg = TUPLE4(hp, am_trace, p->common.id, what, msg);
-    hp += 5;
-
-    erts_smp_mtx_lock(&smq_mtx);
-
-    if (is_internal_port(ERTS_TRACER_PROC(p))) {
-	PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, NULL, NULL);
-	send_to_port(p, msg, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
-    }
-    else {
-	PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, bp, tracer_ref);
-	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, msg, bp);
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, what,
+                           msg, THE_NON_VALUE);
     }
-    erts_smp_mtx_unlock(&smq_mtx);
-    UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
-#undef LOCAL_HEAP_SIZE
 }
 
 void 
@@ -2412,7 +1318,7 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+	erts_queue_message(monitor_p, NULL, mp, msg);
     }
 #endif
 }
@@ -2477,7 +1383,7 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time)
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+	erts_queue_message(monitor_p, NULL, mp, msg);
     }
 #endif
 }
@@ -2552,7 +1458,7 @@ monitor_long_gc(Process *p, Uint time) {
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+	erts_queue_message(monitor_p, NULL, mp, msg);
     }
 #endif
 }
@@ -2627,7 +1533,7 @@ monitor_large_heap(Process *p) {
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+	erts_queue_message(monitor_p, NULL, mp, msg);
     }
 #endif
 }
@@ -2659,7 +1565,7 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg, NIL);
+	erts_queue_message(monitor_p, NULL, mp, msg);
     }
 #endif
 
@@ -2756,71 +1662,13 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
 
 }
 
-
-/* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
- * or   {trace, Pid, What, {Mod, Func, Arity}}
- *
- * where 'What' is supposed to be 'in' or 'out'.
- *
- * Virtual scheduling do not fake scheduling for ports.
- */
-
-
-void trace_virtual_sched(Process *p, Eterm what)
-{
-    trace_sched_aux(p, what, 1);
-}
-
 /* Port profiling */
 
 void
 trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
-    Eterm mess;
-    Eterm* hp;
-
-    if (is_internal_port(ERTS_TRACER_PROC(p))) {
-#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-	hp = local_heap;
-
-	mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name);
-	hp += 6;
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
-	/* No fake schedule */
-	send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	size_t sz_data;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-	
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
-
-	sz_data = 6 + PATCH_TS_SIZE(p);
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(p),
-			    ERTS_TRACE_FLAGS(p));
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref);
-
-	mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name);
-	hp += 6;
-
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
-
-	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
 
+    send_to_tracer_nif(NULL, &p->common, calling_pid, NULL, am_open,
+                       p->common.id, drv_name);
 }
 
 /* Sends trace message:
@@ -2832,53 +1680,11 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
  */
 void
 trace_port(Port *t_p, Eterm what, Eterm data) {
-    Eterm mess;
-    Eterm* hp;
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
-
-    if (is_internal_port(ERTS_TRACER_PROC(t_p))) {
-#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-	hp = local_heap;
-	mess = TUPLE4(hp, am_trace, t_p->common.id, what, data);
-	hp += 5;
-	erts_smp_mtx_lock(&smq_mtx);
-	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL);
-	/* No fake schedule */
-	send_to_port(NULL,mess,&ERTS_TRACER_PROC(t_p),&ERTS_TRACE_FLAGS(t_p));
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else {
-	ErlHeapFragment *bp;
-	ErlOffHeap *off_heap;
-	size_t sz_data;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p)));
-
-	sz_data = 5 + PATCH_TS_SIZE(t_p);
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(t_p),
-			    ERTS_TRACE_FLAGS(t_p));
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref);
-
-	mess = TUPLE4(hp, am_trace, t_p->common.id, what, data);
-	hp += 5;
-
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref);
-
-	ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
+    send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, NULL,
+                       am_open, data, THE_NON_VALUE);
 }
 
 /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
@@ -2891,83 +1697,14 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 
 void
 trace_sched_ports(Port *p, Eterm what) {
-    trace_sched_ports_where(p,what, make_small(0));
+    trace_sched_ports_where(p, what, make_small(0));
 }
 
-void 
-trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
-    Eterm mess;
-    Eterm* hp;
-    int ws = 5;	
-    Eterm sched_id = am_undefined;
-    
-    if (is_internal_port(ERTS_TRACER_PROC(p))) {
-#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE)
-	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-	hp = local_heap;
-	
-	if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
-#ifdef ERTS_SMP
-		ErtsSchedulerData *esd = erts_get_scheduler_data();
-		if (esd) sched_id = make_small(esd->no);
-		else sched_id = am_undefined;
-#else
-		sched_id = make_small(1);
-#endif
-		mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, where);
-		ws = 6;
-	} else {
-		mess = TUPLE4(hp, am_trace, p->common.id, what, where);
-		ws = 5;
-	}
-	hp += ws;
-	
-	erts_smp_mtx_lock(&smq_mtx);
-
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL);
-	
-	/* No fake scheduling */
-	send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p));
-	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-	erts_smp_mtx_unlock(&smq_mtx);
-    } else  {
-	ErlHeapFragment *bp;
-    	ErlOffHeap *off_heap;
-	ERTS_TRACER_REF_TYPE tracer_ref;
-
-	ASSERT(is_internal_pid(ERTS_TRACER_PROC(p)));
-	
-	if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) ws = 6; /* Make place for scheduler id */
-
-	ERTS_GET_TRACER_REF(tracer_ref,
-			    ERTS_TRACER_PROC(p),
-			    ERTS_TRACE_FLAGS(p));
-
-	hp = ERTS_ALLOC_SYSMSG_HEAP(ws+PATCH_TS_SIZE(p), &bp, &off_heap, tracer_ref);
-
-	if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
-#ifdef ERTS_SMP
-		ErtsSchedulerData *esd = erts_get_scheduler_data();
-		if (esd) sched_id = make_small(esd->no);
-		else sched_id = am_undefined;
-#else
-		sched_id = make_small(1);
-#endif
-		mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, where);
-	} else {
-		mess = TUPLE4(hp, am_trace, p->common.id, what, where);
-	}
-	hp += ws;
-
-	erts_smp_mtx_lock(&smq_mtx);
+void
+trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) {
 
-	PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref);
-	ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp);
-	erts_smp_mtx_unlock(&smq_mtx);
-    }
+    send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
+                       NULL, am_open, where, THE_NON_VALUE);
 }
 
 /* Port profiling */
@@ -3083,28 +1820,6 @@ profile_runnable_proc(Process *p, Eterm status){
 
 #ifdef ERTS_SMP
 
-void
-erts_check_my_tracer_proc(Process *p)
-{
-    if (is_internal_pid(ERTS_TRACER_PROC(p))) {
-	Process *tracer = erts_pid2proc(p,
-					ERTS_PROC_LOCK_MAIN,
-					ERTS_TRACER_PROC(p),
-					ERTS_PROC_LOCK_STATUS);
-	int invalid_tracer = (!tracer
-			      || !(ERTS_TRACE_FLAGS(tracer) & F_TRACER));
-	if (tracer)
-	    erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS);
-	if (invalid_tracer) {
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-	    ERTS_TRACE_FLAGS(p) &= ~TRACEE_FLAGS;
-	    ERTS_TRACER_PROC(p) = NIL;
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-	}
-    }
-}
-
-
 typedef struct ErtsSysMsgQ_ ErtsSysMsgQ;
 struct  ErtsSysMsgQ_ {
     ErtsSysMsgQ *next;
@@ -3170,24 +1885,11 @@ erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp)
     enqueue_sys_msg(SYS_MSG_TYPE_ERRLGR, from, am_error_logger, msg, bp);
 }
 
-void
-erts_send_sys_msg_proc(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp)
-{
-    ASSERT(is_internal_pid(to));
-    enqueue_sys_msg(SYS_MSG_TYPE_PROC_MSG, from, to, msg, bp);
-}
-
 #ifdef DEBUG_PRINTOUTS
 static void
 print_msg_type(ErtsSysMsgQ *smqp)
 {
     switch (smqp->type) {
-    case SYS_MSG_TYPE_TRACE:
-	erts_fprintf(stderr, "TRACE ");
-	break;
-    case SYS_MSG_TYPE_SEQTRACE:
-	erts_fprintf(stderr, "SEQTRACE ");
-	break;
     case SYS_MSG_TYPE_SYSMON:
 	erts_fprintf(stderr, "SYSMON ");
 	break;
@@ -3197,9 +1899,6 @@ print_msg_type(ErtsSysMsgQ *smqp)
     case SYS_MSG_TYPE_ERRLGR:
 	erts_fprintf(stderr, "ERRLGR ");
 	break;
-    case SYS_MSG_TYPE_PROC_MSG:
-	erts_fprintf(stderr, "PROC_MSG ");
-	break;
     default:
 	erts_fprintf(stderr, "??? ");
 	break;
@@ -3211,17 +1910,6 @@ static void
 sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 {
     switch (smqp->type) {
-    case SYS_MSG_TYPE_TRACE:
-	/* Invalid tracer_proc's are removed when processes
-	   are scheduled in. */
-	break;
-    case SYS_MSG_TYPE_SEQTRACE:
-	/* Reset seq_tracer if it hasn't changed */
-	erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
-	if (system_seq_tracer == receiver)
-	    system_seq_tracer = am_false;
-	erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
-	break;
     case SYS_MSG_TYPE_SYSMON:
 	if (receiver == NIL
 	    && !erts_system_monitor_long_gc
@@ -3281,8 +1969,6 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 		     no_elgger, tag, CAR(list_val(tp[3])));
 	break;
     }
-    case SYS_MSG_TYPE_PROC_MSG:
-	break;
     default:
 	ASSERT(0);
     }
@@ -3400,13 +2086,6 @@ sys_msg_dispatcher_func(void *unused)
 	    print_msg_type(smqp);
 #endif
 	    switch (smqp->type) {
-	    case SYS_MSG_TYPE_TRACE:
-	    case SYS_MSG_TYPE_PROC_MSG:
-		receiver = smqp->to;
-		break;
-	    case SYS_MSG_TYPE_SEQTRACE:
-		receiver = erts_get_system_seq_tracer();
-		break;
 	    case SYS_MSG_TYPE_SYSMON:
 		receiver = erts_get_system_monitor();
 		if (smqp->from == receiver) {
@@ -3441,16 +2120,8 @@ sys_msg_dispatcher_func(void *unused)
 
 	    if (is_internal_pid(receiver)) {
 		proc = erts_pid2proc(NULL, 0, receiver, proc_locks);
-		if (!proc
-		    || (smqp->type == SYS_MSG_TYPE_TRACE
-			&& !(ERTS_TRACE_FLAGS(proc) & F_TRACER))) {
+		if (!proc) {
 		    /* Bad tracer */
-#ifdef DEBUG_PRINTOUTS
-		    if (smqp->type == SYS_MSG_TYPE_TRACE && proc)
-			erts_fprintf(stderr,
-				     "<tracer alive but missing  "
-				     "F_TRACER flag> ");
-#endif
 		    goto failure;
 		}
 		else {
@@ -3458,7 +2129,7 @@ sys_msg_dispatcher_func(void *unused)
 		queue_proc_msg:
 		    mp = erts_alloc_message(0, NULL);
 		    mp->data.heap_frag = smqp->bp;
-		    erts_queue_message(proc,&proc_locks,mp,smqp->msg,NIL);
+		    erts_queue_message(proc,&proc_locks,mp,smqp->msg);
 #ifdef DEBUG_PRINTOUTS
 		    erts_fprintf(stderr, "delivered\n");
 #endif
@@ -3524,12 +2195,6 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm,
     for (sm = sys_message_queue; sm; sm = sm->next) {
 	Eterm to;
 	switch (sm->type) {
-	case SYS_MSG_TYPE_TRACE:
-	    to = sm->to;
-	    break;
-	case SYS_MSG_TYPE_SEQTRACE:
-	    to = erts_get_system_seq_tracer();
-	    break;
 	case SYS_MSG_TYPE_SYSMON:
 	    to = erts_get_system_monitor();
 	    break;
@@ -3567,3 +2232,498 @@ init_sys_msg_dispatcher(void)
 }
 
 #endif
+
+#include "erl_nif.h"
+
+struct ErtsTracerNif_ {
+    HashBucket hb;
+    Eterm module;
+    struct erl_module_nif* nif_mod;
+    ErlNifFunc *enabled;
+    ErlNifFunc *trace;
+};
+
+static Hash *tracer_hash = NULL;
+static erts_smp_rwmtx_t tracer_mtx;
+
+static ErtsTracerNif *
+load_tracer_nif(const ErtsTracer tracer)
+{
+    Module* mod = erts_get_module(ERTS_TRACER_MODULE(tracer),
+                                  erts_active_code_ix());
+    struct erl_module_instance *instance;
+    ErlNifFunc *funcs;
+    int num_of_funcs;
+    ErtsTracerNif tnif_tmpl, *tnif;
+    int i;
+
+    if (mod && mod->curr.nif != NULL) {
+        instance = &mod->curr;
+    } else {
+        return NULL;
+    }
+
+    tnif_tmpl.enabled = NULL;
+    tnif_tmpl.trace = NULL;
+    tnif_tmpl.nif_mod = instance->nif;
+    tnif_tmpl.module = ERTS_TRACER_MODULE(tracer);
+
+    num_of_funcs = erts_nif_get_funcs(instance->nif, &funcs);
+
+    for(i = 0; i < num_of_funcs; i++) {
+        if (strcmp("enabled",funcs[i].name) == 0 && funcs[i].arity == 3) {
+            tnif_tmpl.enabled = funcs + i;
+        } else if (strcmp("trace",funcs[i].name) == 0 && funcs[i].arity == 6) {
+            tnif_tmpl.trace = funcs + i;
+        }
+    }
+
+    if (tnif_tmpl.enabled == NULL ||
+        tnif_tmpl.trace   == NULL ) {
+        return NULL;
+    }
+
+    erts_smp_rwmtx_rwlock(&tracer_mtx);
+    tnif = hash_put(tracer_hash, &tnif_tmpl);
+    erts_smp_rwmtx_rwunlock(&tracer_mtx);
+
+    return tnif;
+}
+
+static ERTS_INLINE ErtsTracerNif *
+lookup_tracer_nif(const ErtsTracer tracer)
+{
+    ErtsTracerNif tnif_tmpl;
+    ErtsTracerNif *tnif;
+    tnif_tmpl.module = ERTS_TRACER_MODULE(tracer);
+    erts_smp_rwmtx_rlock(&tracer_mtx);
+    if ((tnif = hash_get(tracer_hash, &tnif_tmpl)) == NULL) {
+        erts_smp_rwmtx_runlock(&tracer_mtx);
+        tnif = load_tracer_nif(tracer);
+        ASSERT(!tnif || tnif->nif_mod);
+        return tnif;
+    }
+    erts_smp_rwmtx_runlock(&tracer_mtx);
+    ASSERT(tnif->nif_mod);
+    return tnif;
+}
+
+/* This function converts an Erlang tracer term to ErtsTracer.
+   It returns THE_NON_VALUE if an invalid tracer term was given.
+   Accepted input is:
+     pid() || port() || {prefix, pid()} || {prefix, port()} ||
+     {prefix, atom(), term()} || {atom(), term()}
+ */
+ErtsTracer
+erts_term_to_tracer(Eterm prefix, Eterm t)
+{
+    ErtsTracer tracer = erts_tracer_nil;
+    ASSERT(is_atom(prefix) || prefix == THE_NON_VALUE);
+    if (!is_nil(t)) {
+        Eterm module = am_erl_tracer, state = THE_NON_VALUE;
+        Eterm hp[2];
+        if (is_tuple(t)) {
+            Eterm *tp = tuple_val(t);
+            if (prefix != THE_NON_VALUE) {
+                if (arityval(tp[0]) == 2 && tp[1] == prefix)
+                    t = tp[2];
+                else if (arityval(tp[0]) == 3 && tp[1] == prefix && is_atom(tp[2])) {
+                    module = tp[2];
+                    state = tp[3];
+                }
+            } else {
+                if (arityval(tp[0]) == 2 && is_atom(tp[2])) {
+                    module = tp[1];
+                    state = tp[2];
+                }
+            }
+        }
+        if (state == THE_NON_VALUE && (is_internal_pid(t) || is_internal_port(t)))
+            state = t;
+        if (state == THE_NON_VALUE)
+            return THE_NON_VALUE;
+        erts_tracer_update(&tracer, CONS(hp, module, state));
+    }
+    if (!lookup_tracer_nif(tracer)) {
+        ASSERT(ERTS_TRACER_MODULE(tracer) != am_erl_tracer);
+        ERTS_TRACER_CLEAR(&tracer);
+        return THE_NON_VALUE;
+    }
+    return tracer;
+}
+
+Eterm
+erts_tracer_to_term(Process *p, ErtsTracer tracer)
+{
+    if (ERTS_TRACER_IS_NIL(tracer))
+        return am_false;
+    if (ERTS_TRACER_MODULE(tracer) == am_erl_tracer)
+        /* Have to manage these specifically in order to be
+           backwards compatible */
+        return ERTS_TRACER_STATE(tracer);
+    else {
+        Eterm *hp = HAlloc(p, 3);
+        return TUPLE2(hp, ERTS_TRACER_MODULE(tracer),
+                      copy_object(ERTS_TRACER_STATE(tracer), p));
+    }
+}
+
+
+static ERTS_INLINE int
+send_to_tracer_nif_raw(Process *c_p, Process *tracee,
+                       const ErtsTracer tracer, Uint tracee_flags,
+                       Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag, Eterm msg,
+                       Eterm extra, Eterm pam_result)
+{
+    if (tnif || (tnif = lookup_tracer_nif(tracer)) != NULL) {
+#define MAP_SIZE 3
+        Eterm argv[6],
+            local_heap[3+MAP_SIZE /* values */+(MAP_SIZE+1 /* keys */)];
+        flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1));
+        Eterm *map_values = flatmap_get_values(map);
+
+        int argc = 6;
+
+        argv[0] = tag;
+        argv[1] = ERTS_TRACER_STATE(tracer);
+        argv[2] = t_p_id;
+        argv[3] = msg;
+        argv[4] = extra == THE_NON_VALUE ? am_undefined : extra;
+        argv[5] = make_flatmap(map);
+
+        map->thing_word = MAP_HEADER_FLATMAP;
+        map->size = MAP_SIZE;
+        map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id,
+                           am_timestamp);
+
+        *map_values++ = pam_result;
+        if (tracee_flags & F_TRACE_SCHED_NO)
+            *map_values++ = make_small(erts_get_scheduler_id());
+        else
+            *map_values++ = am_undefined;
+        if (tracee_flags & F_NOW_TS)
+#ifdef HAVE_ERTS_NOW_CPU
+            if (erts_cpu_timestamp)
+                *map_values++ = am_cpu_timestamp;
+            else
+#endif
+                *map_values++ = am_timestamp;
+        else if (tracee_flags & F_STRICT_MON_TS)
+            *map_values++ = am_strict_monotonic;
+        else if (tracee_flags & F_MON_TS)
+            *map_values++ = am_monotonic;
+        else
+            *map_values++ = am_undefined;
+
+#undef MAP_SIZE
+        erts_nif_call_function(c_p, tracee ? tracee : c_p,
+                               tnif->nif_mod, tnif->trace, argc, argv);
+    }
+    return 1;
+}
+
+
+static ERTS_INLINE int
+send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
+                   Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag,
+                   Eterm msg, Eterm extra)
+{
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+    if (c_p) {
+        /* We have to hold the main lock of the currently executing process */
+        erts_proc_lc_chk_have_proc_locks(c_p, ERTS_PROC_LOCK_MAIN);
+    }
+    if (is_internal_pid(t_p->id)) {
+        /* We have to have at least one lock */
+        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL);
+    } else {
+        ASSERT(is_internal_port(t_p->id));
+        ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p));
+    }
+#endif
+
+    return send_to_tracer_nif_raw(c_p,
+                                  is_internal_pid(t_p->id) ? (Process*)t_p : NULL,
+                                  t_p->tracer, t_p->trace_flags,
+                                  t_p_id, tnif, tag, msg, extra,
+                                  am_true);
+}
+
+static ERTS_INLINE Eterm
+call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
+                    ErtsTracerNif **tnif_ret, Eterm tag, Eterm t_p_id)
+{
+    ErtsTracerNif *tnif = lookup_tracer_nif(tracer);
+    if (tnif) {
+        Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id};
+        if (tnif_ret) *tnif_ret = tnif;
+        return erts_nif_call_function(
+            c_p, NULL, tnif->nif_mod, tnif->enabled, 3, argv);
+    }
+    return am_remove;
+}
+
+static int
+is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
+                       ErtsPTabElementCommon *t_p,
+                       ErtsTracerNif **tnif_ret, Eterm tag)
+{
+    Eterm nif_result;
+
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+    if (c_p)
+        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == c_p_locks
+                           || erts_thr_progress_is_blocking());
+    if (is_internal_pid(t_p->id)) {
+        /* We have to have at least one lock */
+        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL
+                           || erts_thr_progress_is_blocking());
+    } else {
+        ASSERT(is_internal_port(t_p->id));
+        ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p)
+                           || erts_thr_progress_is_blocking());
+    }
+#endif
+
+    nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, tag, t_p->id);
+    switch (nif_result) {
+    case am_discard: return 0;
+    case am_trace: return 1;
+    case THE_NON_VALUE:
+    case am_remove: break;
+    default:
+        /* only am_remove should be returned, but if
+           something else is returned we fall-through
+           and remove the tracer. */
+        ASSERT(0);
+    }
+
+    /* Only remove tracer on self() and ports */
+    if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) {
+        ErtsProcLocks c_p_xlocks = 0;
+        if (is_internal_pid(t_p->id)) {
+            ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
+            if (c_p_locks != ERTS_PROC_LOCKS_ALL) {
+                c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL;
+                if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) {
+                    erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN);
+                    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                }
+            }
+        }
+        erts_tracer_replace(t_p, erts_tracer_nil);
+        t_p->trace_flags &= ~TRACEE_FLAGS;
+
+        if (c_p_xlocks)
+            erts_smp_proc_unlock(c_p, c_p_xlocks);
+    }
+
+
+    return 0;
+}
+
+int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
+                                ErtsPTabElementCommon *t_p)
+{
+    return is_tracer_proc_enabled(c_p, c_p_locks, t_p, NULL, am_trace_status);
+}
+
+int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer)
+{
+    ErtsTracerNif *tnif = lookup_tracer_nif(tracer);
+    if (tnif) {
+        Eterm nif_result = call_enabled_tracer(c_p, tracer, &tnif,
+                                               am_trace_status,
+                                               c_p->common.id);
+        switch (nif_result) {
+        case am_discard:
+        case am_trace: return 1;
+        default:
+            break;
+        }
+    }
+    return 0;
+}
+
+void erts_tracer_replace(ErtsPTabElementCommon *t_p, const ErtsTracer tracer)
+{
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+    if (is_internal_pid(t_p->id) && !erts_thr_progress_is_blocking()) {
+        erts_proc_lc_chk_have_proc_locks((Process*)t_p, ERTS_PROC_LOCKS_ALL);
+    } else if (is_internal_port(t_p->id)) {
+        ERTS_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p)
+                       || erts_thr_progress_is_blocking());
+    }
+#endif
+    if (ERTS_TRACER_COMPARE(t_p->tracer, tracer))
+        return;
+
+    erts_tracer_update(&t_p->tracer, tracer);
+}
+
+static void free_tracer(void *p)
+{
+    ErtsTracer tracer = (ErtsTracer)p;
+
+    if (is_immed(ERTS_TRACER_STATE(tracer))) {
+        erts_free(ERTS_ALC_T_HEAP_FRAG, ptr_val(tracer));
+    } else {
+        ErlHeapFragment *hf = (void*)((char*)(ptr_val(tracer)) - offsetof(ErlHeapFragment, mem));
+        free_message_buffer(hf);
+    }
+}
+
+/* un-define erts_tracer_update before implementation */
+#ifdef erts_tracer_update
+#undef erts_tracer_update
+#endif
+
+/*
+ * ErtsTracer is either NIL, 'true' or [Mod | State]
+ *
+ * - If State is immediate then the memory for
+ *   the cons cell is just two words + sizeof(ErtsThrPrgrLaterOp) large.
+ * - If State is a complex term then the cons cell
+ *   is allocated in an ErlHeapFragment where the cons
+ *   ptr points to the mem field. So in order to get the
+ *   ptr to the fragment you do this:
+ *   (char*)(ptr_val(tracer)) - offsetof(ErlHeapFragment, mem)
+ *   Normally you shouldn't have to care about this though
+ *   as erts_tracer_update takes care of it for you.
+ *
+ *   When ErtsTracer is stored in the stack as part of a
+ *   return trace, the cons cell is stored on the heap of
+ *   the process.
+ *
+ *   The cons cell is not always stored on the heap as:
+ *     1) for port/meta tracing there is no heap
+ *     2) we would need the main lock in order to
+ *        read the tracer which is undesirable.
+ *
+ * One way to optimize this (memory wise) is to keep an refc and only bump
+ * the refc when *tracer is NIL.
+ */
+void
+erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer)
+{
+    ErlHeapFragment *hf;
+
+    if (is_not_nil(*tracer)) {
+        Uint offs = 2;
+        UWord size = 2 * sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp);
+        ASSERT(is_list(*tracer));
+        if (is_not_immed(ERTS_TRACER_STATE(*tracer))) {
+            hf = (void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem)));
+            offs = hf->used_size;
+            size = hf->alloc_size * sizeof(Eterm) + sizeof(ErlHeapFragment);
+            ASSERT(offs == size_object(*tracer));
+        }
+        /* We schedule the free:ing of the tracer until after a thread progress
+           has been made so that we know that no schedulers have any references
+           to it. Because we do this, it is possible to release all locks of a
+           process/port and still use the ErtsTracer of that port/process
+           without having to worry if it is free'd.
+        */
+        erts_schedule_thr_prgr_later_cleanup_op(
+            free_tracer, (void*)(*tracer),
+            (ErtsThrPrgrLaterOp*)(ptr_val(*tracer) + offs),
+            size);
+    }
+
+    if (is_nil(new_tracer)) {
+        *tracer = new_tracer;
+    } else if (is_immed(ERTS_TRACER_STATE(new_tracer))) {
+        /* If tracer state is an immediate we only allocate a 2 Eterm heap.
+           Not sure if it is worth it, we save 4 words (sizeof(ErlHeapFragment))
+           per tracer. */
+        Eterm *hp = erts_alloc(ERTS_ALC_T_HEAP_FRAG,
+                               2*sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp));
+        *tracer = CONS(hp, ERTS_TRACER_MODULE(new_tracer),
+                       ERTS_TRACER_STATE(new_tracer));
+    } else {
+        Eterm *hp, tracer_state = ERTS_TRACER_STATE(new_tracer),
+            tracer_module = ERTS_TRACER_MODULE(new_tracer);
+        Uint sz = size_object(tracer_state);
+        hf = new_message_buffer(sz + 2  /* cons cell */ + (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm));
+        hp = hf->mem + 2;
+        hf->used_size -= (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm);
+        *tracer = copy_struct(tracer_state, sz, &hp, &hf->off_heap);
+        *tracer = CONS(hf->mem, tracer_module, *tracer);
+        ASSERT((void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem))) == hf);
+    }
+}
+
+static void init_tracer_nif()
+{
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_smp_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx");
+
+    erts_tracer_nif_clear();
+
+}
+
+int erts_tracer_nif_clear()
+{
+
+    erts_smp_rwmtx_rlock(&tracer_mtx);
+    if (!tracer_hash || tracer_hash->nobjs) {
+
+        HashFunctions hf;
+        hf.hash = tracer_hash_fun;
+        hf.cmp = tracer_cmp_fun;
+        hf.alloc = tracer_alloc_fun;
+        hf.free = tracer_free_fun;
+        hf.meta_alloc = (HMALLOC_FUN) erts_alloc;
+        hf.meta_free = (HMFREE_FUN) erts_free;
+        hf.meta_print = (HMPRINT_FUN) erts_print;
+
+        erts_smp_rwmtx_runlock(&tracer_mtx);
+        erts_smp_rwmtx_rwlock(&tracer_mtx);
+
+        if (tracer_hash)
+            hash_delete(tracer_hash);
+
+        tracer_hash = hash_new(ERTS_ALC_T_TRACER_NIF, "tracer_hash", 10, hf);
+
+        erts_smp_rwmtx_rwunlock(&tracer_mtx);
+        return 1;
+    }
+
+    erts_smp_rwmtx_runlock(&tracer_mtx);
+    return 0;
+}
+
+static int tracer_cmp_fun(void* a, void* b)
+{
+    return ((ErtsTracerNif*)a)->module != ((ErtsTracerNif*)b)->module;
+}
+
+static HashValue tracer_hash_fun(void* obj)
+{
+    return make_internal_hash(((ErtsTracerNif*)obj)->module);
+}
+
+static void *tracer_alloc_fun(void* tmpl)
+{
+    ErtsTracerNif *obj = erts_alloc(ERTS_ALC_T_TRACER_NIF,
+                                    sizeof(ErtsTracerNif) +
+                                    sizeof(ErtsThrPrgrLaterOp));
+    memcpy(obj, tmpl, sizeof(*obj));
+    return obj;
+}
+
+static void tracer_free_fun_cb(void* obj)
+{
+    erts_free(ERTS_ALC_T_TRACER_NIF, obj);
+}
+
+static void tracer_free_fun(void* obj)
+{
+    ErtsTracerNif *tnif = obj;
+    erts_schedule_thr_prgr_later_op(
+        tracer_free_fun_cb, obj,
+        (ErtsThrPrgrLaterOp*)(tnif + 1));
+
+}
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 1da27ee128..61f82a6b93 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -46,6 +46,8 @@
 #define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS)				\
     ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)	\
 	    & ERTS_TRACE_TS_TYPE_MASK))
+#define ERTS_SEQTFLGS2TFLGS(SEQTFLGS)                                   \
+    (ERTS_SEQTFLGS2TSTYPE(SEQTFLGS) << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
 
 #endif /* ERL_TRACE_H__FLAGS__ */
 
@@ -62,15 +64,16 @@ void erts_system_profile_clear(Process *c_p);
 /* erl_trace.c */
 void erts_init_trace(void);
 void erts_trace_check_exiting(Eterm exiting);
-Eterm erts_set_system_seq_tracer(Process *c_p,
-				 ErtsProcLocks c_p_locks,
-				 Eterm new);
-Eterm erts_get_system_seq_tracer(void);
-void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp);
-void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp);
+ErtsTracer erts_set_system_seq_tracer(Process *c_p,
+                                      ErtsProcLocks c_p_locks,
+                                      ErtsTracer new);
+ErtsTracer erts_get_system_seq_tracer(void);
+void erts_change_default_tracing(int setflags, Uint *flagsp,
+                                 const ErtsTracer tracerp);
+void erts_get_default_tracing(Uint *flagsp, ErtsTracer *tracerp);
 void erts_set_system_monitor(Eterm monitor);
 Eterm erts_get_system_monitor(void);
-int erts_is_tracer_proc_valid(Process* p);
+int erts_is_tracer_valid(Process* p);
 
 #ifdef ERTS_SMP
 void erts_check_my_tracer_proc(Process *);
@@ -83,22 +86,22 @@ void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
 void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
 #endif
 
-void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
 void trace_send(Process*, Eterm, Eterm);
-void trace_receive(Process*, Eterm);
-Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, Eterm* args,
-		       int local, Eterm *tracer_pid);
-void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid);
+void trace_receive(Process *, Eterm);
+Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec,
+                       Eterm* args, int local, ErtsTracer *tracer);
+void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval,
+                       ErtsTracer *tracer);
 void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
-			  Eterm *tracer);
+                          ErtsTracer *tracer);
 void erts_trace_return_to(Process *p, BeamInstr *pc);
-void trace_sched(Process*, Eterm);
-void trace_proc(Process*, Process*, Eterm, Eterm);
+void trace_sched(Process*, ErtsProcLocks, Eterm);
+void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm);
 void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
 void save_calls(Process *p, Export *);
 void trace_gc(Process *p, Eterm what);
 /* port tracing */
-void trace_virtual_sched(Process*, Eterm);
+void trace_virtual_sched(Process*, ErtsProcLocks, Eterm);
 void trace_sched_ports(Port *pp, Eterm);
 void trace_sched_ports_where(Port *pp, Eterm, Eterm);
 void trace_port(Port *, Eterm what, Eterm data);
@@ -121,7 +124,7 @@ void monitor_large_heap(Process *p);
 void monitor_generic(Process *p, Eterm type, Eterm spec);
 Uint erts_trace_flag2bit(Eterm flag);
 int erts_trace_flags(Eterm List, 
-		 Uint *pMask, Eterm *pTracer, int *pCpuTimestamp);
+		 Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp);
 Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);
 
 #ifdef ERTS_SMP
@@ -161,15 +164,52 @@ int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 			   struct binary* match_prog_set,
 			   struct binary *meta_match_prog_set,
 			   int on, struct trace_pattern_flags,
-			   Eterm meta_tracer_pid, int is_blocking);
+			   ErtsTracer meta_tracer, int is_blocking);
 void
 erts_get_default_trace_pattern(int *trace_pattern_is_on,
 			       struct binary **match_spec,
 			       struct binary **meta_match_spec,
 			       struct trace_pattern_flags *trace_pattern_flags,
-			       Eterm *meta_tracer_pid);
+			       ErtsTracer *meta_tracer);
 int erts_is_default_trace_enabled(void);
 void erts_bif_trace_init(void);
 int erts_finish_breakpointing(void);
 
+/* Nif tracer functions */
+int erts_is_tracer_proc_enabled(Process *c_p, ErtsProcLocks c_p_locks,
+                                ErtsPTabElementCommon *t_p);
+int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer);
+Eterm erts_tracer_to_term(Process *p, ErtsTracer tracer);
+ErtsTracer erts_term_to_tracer(Eterm prefix, Eterm term);
+void erts_tracer_replace(ErtsPTabElementCommon *t_p,
+                         const ErtsTracer new_tracer);
+void erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer);
+int erts_tracer_nif_clear(void);
+
+#define erts_tracer_update(t,n) do { if (*(t) != (n)) erts_tracer_update(t,n); } while(0)
+#define ERTS_TRACER_CLEAR(t) erts_tracer_update(t, erts_tracer_nil)
+
+static const ErtsTracer
+ERTS_DECLARE_DUMMY(erts_tracer_true) = am_true;
+
+static const ErtsTracer
+ERTS_DECLARE_DUMMY(erts_tracer_nil) = NIL;
+
+#define ERTS_TRACER_COMPARE(t1, t2)                     \
+    (EQ((t1), (t2)))
+
+#define ERTS_TRACER_IS_NIL(t1) ERTS_TRACER_COMPARE(t1, erts_tracer_nil)
+
+#define IS_TRACER_VALID(tracer)                                         \
+    (ERTS_TRACER_COMPARE(tracer,erts_tracer_true)                       \
+     || ERTS_TRACER_IS_NIL(tracer)                                      \
+     || (is_list(tracer) && is_atom(CAR(list_val(tracer)))))
+
+#define ERTS_TRACER_FROM_ETERM(termp) \
+    ((ErtsTracer*)(termp))
+
+#define ERTS_TRACER_PROC_IS_ENABLED(PROC)                               \
+    (!ERTS_TRACER_IS_NIL(ERTS_TRACER(PROC))                             \
+     && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN, &(PROC)->common))
+
 #endif /* ERL_TRACE_H__ */
diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d
index 73ef5a108a..237889e0f5 100644
--- a/erts/emulator/beam/erlang_dtrace.d
+++ b/erts/emulator/beam/erlang_dtrace.d
@@ -700,6 +700,35 @@ provider erlang {
      */
     probe efile_drv__return(int, int, char *, int, int, int);
 
+
+/*
+ * The set of probes called by the erlang tracer nif backend. In order
+ * to receive events on these you both have to enable tracing in erlang
+ * using the trace bifs and also from dtrace/systemtap.
+ */
+
+
+    /**
+     * A trace message of type `event` was triggered by process `p`.
+     *
+     *
+     * @param p the PID (string form) of the process
+     * @param event the event that was triggered (i.e. call or spawn)
+     * @param state the state of the tracer nif as a string
+     * @param arg1 first argument to the trace event
+     * @param arg2 second argument to the trace event
+     */
+    probe trace(char *p, char *event, char *state, char *arg1, char *arg2);
+
+    /**
+     * A sequence trace message of type `label` was triggered.
+     *
+     * @param state the state of the tracer nif as a string
+     * @param label the seq trace label
+     * @param seq_info the seq trace info tuple as a string
+     */
+    probe trace_seq(char *state, char *label, char *seq_info);
+
 /*
  * NOTE:
  * For formatting int64_t arguments within a D script, see:
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 1d3704f0af..5c2e8bbd09 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -44,6 +44,8 @@
 #include "erl_port.h"
 #include "erl_gc.h"
 
+struct enif_func_t;
+
 struct enif_environment_t /* ErlNifEnv */
 {
     struct erl_module_nif* mod_nif;
@@ -54,6 +56,7 @@ struct enif_environment_t /* ErlNifEnv */
     int fpe_was_unmasked;
     struct enif_tmp_obj_t* tmp_obj_list;
     int exception_thrown; /* boolean */
+    Process *tracee;
 };
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*);
@@ -62,6 +65,12 @@ extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(int to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
 extern void erl_nif_init(void);
+extern int erts_nif_get_funcs(struct erl_module_nif*,
+                              struct enif_func_t **funcs);
+extern Eterm erts_nif_call_function(Process *p, Process *tracee,
+                                    struct erl_module_nif*,
+                                    struct enif_func_t *,
+                                    int argc, Eterm *argv);
 
 /* Driver handle (wrapper for old plain handle) */
 #define ERL_DE_OK      0
@@ -1532,10 +1541,15 @@ ERTS_GLB_INLINE void dtrace_fun_decode(Process *process,
 ERTS_GLB_INLINE void
 dtrace_pid_str(Eterm pid, char *process_buf)
 {
-    erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "<%lu.%lu.%lu>",
-                  pid_channel_no(pid),
-                  pid_number(pid),
-                  pid_serial(pid));
+    if (is_pid(pid))
+        erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "<%lu.%lu.%lu>",
+                      pid_channel_no(pid),
+                      pid_number(pid),
+                      pid_serial(pid));
+    else if (is_port(pid))
+        erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>",
+                      port_channel_no(pid),
+                      port_number(pid));
 }
 
 ERTS_GLB_INLINE void
@@ -1547,9 +1561,7 @@ dtrace_proc_str(Process *process, char *process_buf)
 ERTS_GLB_INLINE void
 dtrace_port_str(Port *port, char *port_buf)
 {
-    erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>",
-                  port_channel_no(port->common.id),
-                  port_number(port->common.id));
+    dtrace_pid_str(port->common.id, port_buf);
 }
 
 ERTS_GLB_INLINE void
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index de73da8e22..a35c305046 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -212,6 +212,7 @@ static ERTS_INLINE void
 kill_port(Port *pp)
 {
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
+    ERTS_TRACER_CLEAR(&ERTS_TRACER(pp));
     erts_ptab_delete_element(&erts_port, &pp->common); /* Time of death */
     erts_port_task_free_port(pp);
     /* In non-smp case the port structure may have been deallocated now */
@@ -404,7 +405,7 @@ static Port *create_port(char *name,
     prt->os_pid = -1;
 
     /* Set default tracing */
-    erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt));
+    erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER(prt));
 
     ERTS_CT_ASSERT(offsetof(Port,common) == 0);
 
@@ -1300,7 +1301,7 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	reds_left_in = CONTEXT_REDS/10;
     else {
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
-	    trace_virtual_sched(c_p, am_out);
+	    trace_sched(c_p, ERTS_PROC_LOCK_MAIN, am_out);
 	/*
 	 * No status lock held while sending runnable
 	 * proc trace messages. It is however not needed
@@ -1384,7 +1385,7 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	}
 
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
-	    trace_virtual_sched(c_p, am_in);
+	    trace_sched(c_p, ERTS_PROC_LOCK_MAIN, am_in);
 	/*
 	 * No status lock held while sending runnable
 	 * proc trace messages. It is however not needed
@@ -1447,7 +1448,7 @@ queue_port_sched_op_reply(Process *rp,
 
     erts_factory_trim_and_close(factory, &msg, 1);
 
-    erts_queue_message(rp, rp_locksp, factory->message, msg, NIL);
+    erts_queue_message(rp, rp_locksp, factory->message, msg);
 }
 
 static void
@@ -1535,9 +1536,8 @@ erts_schedule_proc2port_signal(Process *c_p,
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	c_p->msg.save = c_p->msg.last;
 
-	erts_smp_proc_unlock(c_p,
-			     (ERTS_PROC_LOCK_MAIN
-			      | ERTS_PROC_LOCKS_MSG_RECEIVE));
+        erts_smp_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE
+                                   | ERTS_PROC_LOCK_MAIN));
     }
 
 
@@ -2948,7 +2948,7 @@ port_link_failure(Eterm port_id, Eterm linker)
 	    if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
 		/* We didn't exit the process and it is traced */
 		if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-		    trace_proc(NULL, rp, am_getting_unlinked, port_id);
+		    trace_proc(NULL, 0, rp, am_getting_unlinked, port_id);
 	    }
 	    if (rp_locks)
 		erts_smp_proc_unlock(rp, rp_locks);
@@ -3417,7 +3417,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
 				     sz_res + 3, &hp, &ohp);
 	res = copy_struct(res, sz_res, &hp, ohp);
 	tuple = TUPLE2(hp, sender, res);
-	erts_queue_message(rp, &rp_locks, mp, tuple, NIL);
+	erts_queue_message(rp, &rp_locks, mp, tuple);
 
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
@@ -3513,7 +3513,8 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined);
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
+    erts_queue_message(rp, &rp_locks, mp, tuple);
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
@@ -3680,7 +3681,8 @@ deliver_vec_message(Port* prt,			/* Port */
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined);
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
+    erts_queue_message(rp, &rp_locks, mp, tuple);
     erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
 	erts_proc_dec_refc(rp);
@@ -3939,7 +3941,7 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
 		if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
 		    /* We didn't exit the process and it is traced */
 		    if (IS_TRACED_FL(rp, F_TRACE_PROCS)) {
-			trace_proc(NULL, rp, am_getting_unlinked,
+			trace_proc(NULL, 0, rp, am_getting_unlinked,
 				   psc->port);
 		    }
 		}
@@ -5036,7 +5038,7 @@ reply_io_bytes(void *vreq)
 	eout = erts_bld_uint64(&hp, NULL, out);
 
 	msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout);
-	erts_queue_message(rp, &rp_locks, mp, msg, NIL);
+	erts_queue_message(rp, &rp_locks, mp, msg);
 
 	if (req->sched_id == sched_id)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -5529,7 +5531,8 @@ void driver_report_exit(ErlDrvPort ix, int status)
    hp += 3;
    tuple = TUPLE2(hp, prt->common.id, tuple);
 
-   erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined);
+   ERL_MESSAGE_TOKEN(mp) = am_undefined;
+   erts_queue_message(rp, &rp_locks, mp, tuple);
 
    erts_smp_proc_unlock(rp, rp_locks);
    if (!scheduler)
@@ -6123,7 +6126,8 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	mess = ESTACK_POP(stack);  /* get resulting value */
 	erts_factory_trim_and_close(&factory, &mess, 1);
 	/* send message */
-	erts_queue_message(rp, &rp_locks, factory.message, mess, am_undefined);
+        ERL_MESSAGE_TOKEN(factory.message) = am_undefined;
+	erts_queue_message(rp, &rp_locks, factory.message, mess);
     }
     else {
 	if (b2t.ix > b2t.used)
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 071f6aee08..f4a889b8f8 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -226,7 +226,8 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
     rp = (RegProc*) hash_put(&process_reg, (void*) &r);
     if (proc && rp->p == proc) {
 	if (IS_TRACED_FL(proc, F_TRACE_PROCS)) {
-	    trace_proc(c_p, proc, am_register, name);
+	    trace_proc(proc, ERTS_PROC_LOCK_MAIN,
+                       proc, am_register, name);
 	}
 	proc->common.u.alive.reg = rp;
     }
@@ -470,8 +471,8 @@ int erts_unregister_name(Process *c_p,
     int res = 0;
     RegProc r, *rp;
     Port *port = c_prt;
+    ErtsProcLocks current_c_p_locks = 0;
 #ifdef ERTS_SMP
-    ErtsProcLocks current_c_p_locks;
 
     /*
      * SMP note: If 'c_prt != NULL' and 'c_prt->reg->name == name',
@@ -555,7 +556,8 @@ int erts_unregister_name(Process *c_p,
 #endif
 	    rp->p->common.u.alive.reg = NULL;
 	    if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) {
-		trace_proc(c_p, rp->p, am_unregister, r.name);
+		trace_proc(c_p, current_c_p_locks,
+                           rp->p, am_unregister, r.name);
 	    }
 #ifdef ERTS_SMP
 	    if (rp->p != c_p) {
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 8b3eb2db1d..748fba15c7 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -644,6 +644,15 @@ typedef struct preload {
     unsigned char* code;	/* Code pointer */
 } Preload;
 
+/*
+ * ErtsTracer is either NIL, 'true' or [Mod | State]
+ *
+ * If set to NIL, it means no tracer.
+ * If set to 'true' it means the current process' tracer.
+ * If set to [Mod | State], there is a tracer.
+ *  See erts_tracer_update for more details
+ */
+typedef Eterm ErtsTracer;
 
 /*
  * This structure contains options to all built in drivers.
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 384b340a1c..b280995488 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2298,7 +2298,7 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(p, NULL /* only used for smp build */, mp, message, NIL);
+	erts_queue_message(p, NULL /* only used for smp build */, mp, message);
     }
 #endif
 }
-- 
cgit v1.2.3


From 6cb6b59cd4cd5bd4383053e12ae8ab192711c827 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Mon, 8 Feb 2016 18:31:43 +0100
Subject: erts: Extend process and port tracing

This commit completes the tracing for processes so that
all messages sent by a process (via nifs or otherwise) will
be traced.

The commit also adds tracing of all types of events from ports.

When enabling tracing using erlang:trace, the 'all' flag now also
enables tracing on all ports.

OTP-13496
---
 erts/emulator/beam/atom.names      |   7 +
 erts/emulator/beam/bif.c           |  35 ++--
 erts/emulator/beam/dist.c          |   4 +-
 erts/emulator/beam/erl_bif_port.c  |  24 +--
 erts/emulator/beam/erl_bif_trace.c |  70 ++++++--
 erts/emulator/beam/erl_message.c   |   6 +-
 erts/emulator/beam/erl_nif.c       |  14 +-
 erts/emulator/beam/erl_port.h      |   2 +-
 erts/emulator/beam/erl_port_task.c |   2 +
 erts/emulator/beam/erl_process.c   |   4 +-
 erts/emulator/beam/erl_process.h   |  19 +-
 erts/emulator/beam/erl_trace.c     | 345 ++++++++++++++++++++++++++++++++-----
 erts/emulator/beam/erl_trace.h     |  17 +-
 erts/emulator/beam/io.c            | 236 +++++++++++++++++++------
 erts/emulator/beam/register.c      |   6 +-
 15 files changed, 628 insertions(+), 163 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index aeea6cc989..bd1005867c 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -161,6 +161,7 @@ atom close
 atom closed
 atom code
 atom command
+atom commandv
 atom compact
 atom compat_rel
 atom compile
@@ -245,6 +246,9 @@ atom exact_reductions
 atom exclusive
 atom exit_status
 atom existing
+atom existing_processes
+atom existing_ports
+atom existing
 atom exiting
 atom exports
 atom external
@@ -404,6 +408,8 @@ atom net_kernel_terminated
 atom never_utf
 atom new
 atom new_index
+atom new_processes
+atom new_ports
 atom new_uniq
 atom newline
 atom next
@@ -543,6 +549,7 @@ atom scheme
 atom scientific
 atom scope
 atom seconds
+atom send_to_non_existing_process
 atom sensitive
 atom sequential_tracer
 atom sequential_trace_token
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d633a982ca..ed5b2983dd 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1911,7 +1911,7 @@ static Sint remote_send(Process *p, DistEntry *dep,
     }
 
     if (res >= 0) {
-	if (IS_TRACED(p))
+	if (IS_TRACED_FL(p, F_TRACE_SEND))
 	    trace_send(p, full_to, msg);
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 	    save_calls(p, &exp_send);
@@ -1930,7 +1930,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
     Eterm* tp;
 
     if (is_internal_pid(to)) {
-	if (IS_TRACED(p))
+	if (IS_TRACED_FL(p, F_TRACE_SEND))
 	    trace_send(p, to, msg);
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 	    save_calls(p, &exp_send);
@@ -1959,7 +1959,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 
 	rp = erts_proc_lookup_raw(id);
 	if (rp) {
-	    if (IS_TRACED(p))
+	    if (IS_TRACED_FL(p, F_TRACE_SEND))
 		trace_send(p, to, msg);
 	    if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 		save_calls(p, &exp_send);
@@ -1975,7 +1975,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 	    goto port_common;
 	}
 
-	if (IS_TRACED(p))
+	if (IS_TRACED_FL(p, F_TRACE_SEND))
 	    trace_send(p, to, msg);
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 	    save_calls(p, &exp_send);
@@ -2006,11 +2006,20 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 
       port_common:
 	ret_val = 0;
-        
+
 	if (pt) {
 	    int ps_flags = ctx->suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND;
 	    *refp = NIL;
 
+            if (IS_TRACED_FL(p, F_TRACE_SEND)) 	/* trace once only !! */
+                trace_send(p, portid, msg);
+
+            if (have_seqtrace(SEQ_TRACE_TOKEN(p))) {
+                seq_trace_update_send(p);
+                seq_trace_output(SEQ_TRACE_TOKEN(p), msg,
+                                 SEQ_TRACE_SEND, portid, p);
+            }
+
 	    switch (erts_port_command(p, ps_flags, pt, msg, refp)) {
 	    case ERTS_PORT_OP_CALLER_EXIT:
 		/* We are exiting... */
@@ -2043,18 +2052,10 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 		break;
 	    }
 	}
-	
-	if (IS_TRACED(p)) 	/* trace once only !! */
-	    trace_send(p, portid, msg);
+
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 	    save_calls(p, &exp_send);
-	
-        if (have_seqtrace(SEQ_TRACE_TOKEN(p))) {
-	    seq_trace_update_send(p);
-	    seq_trace_output(SEQ_TRACE_TOKEN(p), msg, 
-			     SEQ_TRACE_SEND, portid, p);
-	}	    
-	
+
 	if (ERTS_PROC_IS_EXITING(p)) {
 	    KILL_CATCHES(p); /* Must exit */
 	    return SEND_USER_ERROR;
@@ -2077,7 +2078,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 	if (dep == erts_this_dist_entry) {
 	    Eterm id;
 	    erts_deref_dist_entry(dep);
-	    if (IS_TRACED(p))
+	    if (IS_TRACED_FL(p, F_TRACE_SEND))
 		trace_send(p, to, msg);
 	    if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 		save_calls(p, &exp_send);
@@ -2108,7 +2109,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 	}
 	return ret;
     } else {
-	if (IS_TRACED(p)) /* XXX Is this really neccessary ??? */
+	if (IS_TRACED_FL(p, F_TRACE_SEND))
 	    trace_send(p, to, msg);
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 	    save_calls(p, &exp_send);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 8cd3f5fbe6..52fd57e101 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1723,7 +1723,7 @@ decode_error:
     }
 data_error:
     UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
-    erts_deliver_port_exit(prt, dep->cid, am_killed, 0);
+    erts_deliver_port_exit(prt, dep->cid, am_killed, 0, 1);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     return -1;
 }
@@ -2093,7 +2093,7 @@ erts_dist_command(Port *prt, int reds_limit)
     erts_smp_de_runlock(dep);
 
     if (status & ERTS_DE_SFLG_EXITING) {
-	erts_deliver_port_exit(prt, prt->common.id, am_killed, 0);
+	erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1);
 	erts_deref_dist_entry(dep);
 	return reds + ERTS_PORT_REDS_DIST_CMD_EXIT;
     }
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index eb55bd585e..37f4e1de49 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -110,6 +110,10 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
     erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id);
     erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port->common.id);
 
+    if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS))
+        trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, BIF_P,
+                   am_link, port->common.id);
+
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 
     erts_port_release(port);
@@ -340,8 +344,7 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1)
     if (!prt)
 	BIF_RET(am_badarg);
 
-
-    switch (erts_port_exit(BIF_P, 0, prt, prt->common.id, am_normal, &ref)) {
+    switch (erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, am_normal, &ref)) {
     case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
     case ERTS_PORT_OP_DROPPED:
@@ -374,7 +377,7 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2)
     ref = NIL;
 #endif
 
-    switch (erts_port_connect(BIF_P, 0, prt, prt->common.id, BIF_ARG_2, &ref)) {
+    switch (erts_port_connect(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, &ref)) {
     case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
     case ERTS_PORT_OP_DROPPED:
@@ -929,21 +932,22 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
         DTRACE3(port_open, process_str, name_buf, port_str);
     }
 #endif
+
+    if (port && IS_TRACED_FL(port, F_TRACE_PORTS))
+        trace_port(port, am_getting_linked, p->common.id);
+
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 
+    if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
+        trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
+    }
+
     if (!port) {
 	DEBUGF(("open_driver returned (%d:%d)\n",
 		err_typep ? *err_typep : 4711,
 		err_nump ? *err_nump : 4711));
-    	if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
-            trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
-    	}
 	goto do_return;
     }
-    
-    if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
-        trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
-    }
 
     if (linebuf && port->linebuf == NULL){
 	port->linebuf = allocate_linebuf(linebuf);
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 049899211f..7e2fe7fcd4 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -487,7 +487,8 @@ start_trace(Process *c_p, ErtsTracer tracer,
             && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) {
             /* This tracee is already being traced, and not by the
              * tracer to be */
-            if (erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCKS_ALL, common)) {
+            if (erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCKS_ALL,
+                                            common, am_trace_status)) {
                 /* The tracer is still in use */
                 return 1;
             }
@@ -654,18 +655,27 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 	}
 #endif
 	
-	if (pid_spec == am_all || pid_spec == am_existing) {
+	if (pid_spec == am_all || pid_spec == am_existing ||
+            pid_spec == am_ports || pid_spec == am_processes ||
+            pid_spec == am_existing_ports || pid_spec == am_existing_processes
+            ) {
 	    int i;
 	    int procs = 0;
 	    int ports = 0;
 	    int mods = 0;
 
 	    if (mask & (ERTS_PROC_TRACEE_FLAGS & ~ERTS_TRACEE_MODIFIER_FLAGS))
-		procs = 1;
+		procs = pid_spec != am_ports && pid_spec != am_existing_ports;
 	    if (mask & (ERTS_PORT_TRACEE_FLAGS & ~ERTS_TRACEE_MODIFIER_FLAGS))
-		ports = 1;
-	    if (mask & ERTS_TRACEE_MODIFIER_FLAGS)
-		mods = 1;
+		ports = pid_spec != am_processes && pid_spec != am_existing_processes;
+	    if (mask & ERTS_TRACEE_MODIFIER_FLAGS) {
+                if (pid_spec == am_ports || pid_spec == am_existing_ports)
+                    ports = 1;
+                else if (pid_spec == am_processes || pid_spec == am_existing_processes)
+                    procs = 1;
+                else
+                    mods = 1;
+            }
 
 #ifdef ERTS_SMP
 	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
@@ -697,17 +707,25 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 		    if (state & ERTS_PORT_SFLGS_DEAD)
 			continue;
                     start_trace(p, tracer, &tracee_port->common, on, mask);
-		    /* matches are not counted for ports since it would violate compatibility */
-		    /* This could be a reason to modify this function or make a new one. */
+                    matches++;
 		}
 	    }
 	}
 
-	if (pid_spec == am_all || pid_spec == am_new) {
-	    Uint def_flags = mask;
+	if (pid_spec == am_all || pid_spec == am_new
+            || pid_spec == am_ports || pid_spec == am_processes
+            || pid_spec == am_new_ports || pid_spec == am_new_processes
+            ) {
 
 	    ok = 1;
-	    erts_change_default_tracing(on, &def_flags, tracer);
+            if (mask & ERTS_PROC_TRACEE_FLAGS &&
+                pid_spec != am_ports && pid_spec != am_new_ports)
+                erts_change_default_proc_tracing(
+                    on, mask & ERTS_PROC_TRACEE_FLAGS, tracer);
+            if (mask & ERTS_PORT_TRACEE_FLAGS &&
+                pid_spec != am_processes && pid_spec != am_new_processes)
+                erts_change_default_port_tracing(
+                    on, mask & ERTS_PORT_TRACEE_FLAGS, tracer);
 
 #ifdef HAVE_ERTS_NOW_CPU
 	    if (cpu_ts && !on) {
@@ -773,7 +791,7 @@ Eterm trace_info_2(BIF_ALIST_2)
 
     if (What == am_on_load) {
 	res = trace_info_on_load(p, Key);
-    } else if (is_atom(What) || is_pid(What)) {
+    } else if (is_atom(What) || is_pid(What) || is_port(What)) {
 	res = trace_info_pid(p, What, Key);
     } else if (is_tuple(What)) {
 	res = trace_info_func(p, What, Key);
@@ -792,11 +810,32 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
     Uint trace_flags = am_false;
     Eterm* hp;
 
-    if (pid_spec == am_new) {
+    if (pid_spec == am_new || pid_spec == am_new_processes) {
+        ErtsTracer def_tracer;
+	erts_get_default_proc_tracing(&trace_flags, &def_tracer);
+        tracer = erts_tracer_to_term(p, def_tracer);
+        ERTS_TRACER_CLEAR(&def_tracer);
+    } else if (pid_spec == am_new_ports) {
         ErtsTracer def_tracer;
-	erts_get_default_tracing(&trace_flags, &def_tracer);
+	erts_get_default_port_tracing(&trace_flags, &def_tracer);
         tracer = erts_tracer_to_term(p, def_tracer);
         ERTS_TRACER_CLEAR(&def_tracer);
+    } else if (is_internal_port(pid_spec)) {
+        Port *tracee;
+        tracee = erts_id2port_sflgs(pid_spec, p, ERTS_PROC_LOCK_MAIN,
+                                    ERTS_PORT_SFLGS_INVALID_LOOKUP);
+
+        if (!tracee)
+            return am_undefined;
+
+        if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee)))
+            erts_is_tracer_proc_enabled(NULL, 0, &tracee->common, am_trace_status);
+
+        tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee));
+        trace_flags = ERTS_TRACE_FLAGS(tracee);
+
+        erts_port_release(tracee);
+
     } else if (is_internal_pid(pid_spec)) {
 	Process *tracee;
 	tracee = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
@@ -806,7 +845,8 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 	    return am_undefined;
 
         if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee)))
-            erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN, &tracee->common);
+            erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN,
+                                        &tracee->common, am_trace_status);
 
         tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee));
         trace_flags = ERTS_TRACE_FLAGS(tracee);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 1cb02ec274..9beff52835 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -596,7 +596,7 @@ erts_try_alloc_message_on_heap(Process *pp,
 #if defined(ERTS_SMP)
 	*plp & ERTS_PROC_LOCK_MAIN
 #else
-	1
+	pp
 #endif
 	) {
 #ifdef ERTS_SMP
@@ -626,7 +626,7 @@ erts_try_alloc_message_on_heap(Process *pp,
 	*on_heap_p = !0;
     }
 #ifdef ERTS_SMP
-    else if (erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
+    else if (pp && erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
 	locked_main = 1;
 	*psp = erts_smp_atomic32_read_nob(&pp->state);
 	*plp |= ERTS_PROC_LOCK_MAIN;
@@ -1488,7 +1488,7 @@ erts_factory_message_create(ErtsHeapFactory* factory,
     int on_heap;
     erts_aint32_t state;
 
-    state = erts_smp_atomic32_read_nob(&proc->state);
+    state = proc ? erts_smp_atomic32_read_nob(&proc->state) : 0;
 
     if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) {
 	msgp = erts_alloc_message(sz, &hp);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f35c08450c..59a2b20ff2 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -417,6 +417,7 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 	  ? erts_proc_lookup(receiver)
 	  : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
 			      receiver, rp_locks, ERTS_P2P_FLG_INC_REFC));
+
     if (rp == NULL) {
 	ASSERT(env == NULL || receiver != c_p->common.id);
 	return 0;
@@ -450,9 +451,17 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     }
 
     if (!env || !env->tracee) {
-        erts_queue_message(rp, &rp_locks, mp, msg);
+
+        if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND))
+            trace_send(c_p, receiver, msg);
+
+#ifndef ERTS_SMP
     }
+#endif
+
+        erts_queue_message(rp, &rp_locks, mp, msg);
 #ifdef ERTS_SMP
+    }
     else {
         /* This clause is taken when the nif is called in the context
            of a traced process. We do not know which locks we have
@@ -548,6 +557,9 @@ enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port,
     if (!prt)
         return 0;
 
+    if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+        trace_port_receive(prt, env->proc->common.id, am_command, msg);
+
     return erts_port_output_async(prt, env->proc->common.id, msg);
 }
 
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 59aa034f48..c2588e718d 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -944,7 +944,7 @@ erts_schedule_proc2port_signal(Process *,
 			       ErtsPortTaskHandle *,
 			       ErtsProc2PortSigCallback);
 
-int erts_deliver_port_exit(Port *, Eterm, Eterm, int);
+int erts_deliver_port_exit(Port *, Eterm, Eterm, int, int);
 
 /*
  * Port signal flags
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 30ce181ebb..3102e44c11 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1766,6 +1766,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 		if (!(state & ERTS_PORT_SFLGS_DEAD)) {
 		    DTRACE_DRIVER(driver_timeout, pp);
 		    LTTNG_DRIVER(driver_timeout, pp);
+                    if (IS_TRACED_FL(pp, F_TRACE_RECEIVE))
+                        trace_port(pp, am_receive, am_timeout);
 		    (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data);
 		}
 	    }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e7d0d127e2..1398bc9c61 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2312,7 +2312,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 						    ERTS_PORT_SFLG_HALT);
 		erts_smp_atomic32_inc_nob(&erts_halt_progress);
 		if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING)))
-		    erts_deliver_port_exit(prt, prt->common.id, am_killed, 0);
+		    erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1);
 	    }
 
 	    erts_port_release(prt);
@@ -11103,7 +11103,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	    : STORE_NC(&p->htop, &p->off_heap, parent->group_leader);
     }
 
-    erts_get_default_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER(p));
+    erts_get_default_proc_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER(p));
 
     p->msg.first = NULL;
     p->msg.last = &p->msg.first;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8261ae28f8..61acf5924b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1409,12 +1409,14 @@ extern int erts_system_profile_ts_type;
 		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
 		     | F_TRACE_SCHED_EXIT)
 
+
 #define ERTS_TRACEE_MODIFIER_FLAGS \
-  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
-#define ERTS_PORT_TRACEE_FLAGS \
-  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
+    (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO \
+     | F_TRACE_RECEIVE | F_TRACE_SEND)
+#define ERTS_PORT_TRACEE_FLAGS                                     \
+    (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
 #define ERTS_PROC_TRACEE_FLAGS \
-  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)
+    ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)
 
 #define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))
 
@@ -2326,14 +2328,17 @@ erts_alloc_message_heap_state(Process *pp,
 			      ErlOffHeap **ohpp)
 {
     int on_heap;
+    ErtsMessage *mp;
 
     if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) {
-	ErtsMessage *mp = erts_alloc_message(sz, hpp);
+	mp = erts_alloc_message(sz, hpp);
 	*ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
 	return mp;
     }
 
-    return erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap);
+    mp = erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap);
+    ASSERT(pp || !on_heap);
+    return mp;
 }
 
 ERTS_GLB_INLINE ErtsMessage *
@@ -2343,7 +2348,7 @@ erts_alloc_message_heap(Process *pp,
 			Eterm **hpp,
 			ErlOffHeap **ohpp)
 {
-    erts_aint32_t state = erts_smp_atomic32_read_nob(&pp->state);
+    erts_aint32_t state = pp ? erts_smp_atomic32_read_nob(&pp->state) : 0;
     return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp);
 }
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 1e6ae8c82e..5ba58e9350 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -52,8 +52,10 @@
 Export exp_send, exp_receive, exp_timeout;
 
 static ErtsTracer system_seq_tracer;
-static Uint default_trace_flags;
-static ErtsTracer default_tracer;
+static Uint default_proc_trace_flags;
+static ErtsTracer default_proc_tracer;
+static Uint default_port_trace_flags;
+static ErtsTracer default_port_tracer;
 
 static Eterm system_monitor;
 static Eterm system_profile;
@@ -327,8 +329,10 @@ void erts_init_trace(void) {
     erts_bif_trace_init();
     erts_system_monitor_clear(NULL);
     erts_system_profile_clear(NULL);
-    default_trace_flags = F_INITIAL_TRACE_FLAGS;
-    default_tracer = erts_tracer_nil;
+    default_proc_trace_flags = F_INITIAL_TRACE_FLAGS;
+    default_proc_tracer = erts_tracer_nil;
+    default_port_trace_flags = F_INITIAL_TRACE_FLAGS;
+    default_port_tracer = erts_tracer_nil;
     system_seq_tracer = erts_tracer_nil;
 #ifdef ERTS_SMP
     init_sys_msg_dispatcher();
@@ -456,56 +460,111 @@ erts_get_system_seq_tracer(void)
 }
 
 static ERTS_INLINE void
-get_default_tracing(Uint *flagsp, ErtsTracer *tracerp)
+get_default_tracing(Uint *flagsp, ErtsTracer *tracerp,
+                    Uint *default_trace_flags,
+                    ErtsTracer *default_tracer)
 {
-    if (!(default_trace_flags & TRACEE_FLAGS))
-	ERTS_TRACER_CLEAR(&default_tracer);
+    if (!(*default_trace_flags & TRACEE_FLAGS))
+	ERTS_TRACER_CLEAR(default_tracer);
 
-    if (ERTS_TRACER_IS_NIL(default_tracer)) {
-	default_trace_flags &= ~TRACEE_FLAGS;
+    if (ERTS_TRACER_IS_NIL(*default_tracer)) {
+	*default_trace_flags &= ~TRACEE_FLAGS;
     } else {
         Eterm nif_result = call_enabled_tracer(
-            NULL, default_tracer, NULL,
+            NULL, *default_tracer, NULL,
             am_trace_status, am_undefined);
         switch (nif_result) {
         case am_trace: break;
-        default:
-            default_trace_flags &= ~TRACEE_FLAGS;
-            ERTS_TRACER_CLEAR(&default_tracer);
+        default: {
+            ErtsTracer curr_default_tracer = *default_tracer;
+            if (tracerp) {
+                /* we only have a rlock, so we have to unlock and then rwlock */
+                erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+                erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+            }
+            /* check if someone else changed default tracer
+               while we got the write lock, if so we don't do
+               anything. */
+            if (curr_default_tracer == *default_tracer) {
+                *default_trace_flags &= ~TRACEE_FLAGS;
+                ERTS_TRACER_CLEAR(default_tracer);
+            }
+            if (tracerp) {
+                erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+                erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+            }
+        }
         }
     }
 
     if (flagsp)
-	*flagsp = default_trace_flags;
+	*flagsp = *default_trace_flags;
     if (tracerp) {
-	erts_tracer_update(tracerp,default_tracer);
+	erts_tracer_update(tracerp,*default_tracer);
     }
 }
 
+static ERTS_INLINE void
+erts_change_default_tracing(int setflags, Uint flags,
+                            const ErtsTracer tracer,
+                            Uint *default_trace_flags,
+                            ErtsTracer *default_tracer)
+{
+    if (setflags)
+        *default_trace_flags |= flags;
+    else
+        *default_trace_flags &= ~flags;
+
+    erts_tracer_update(default_tracer, tracer);
+
+    get_default_tracing(NULL, NULL, default_trace_flags, default_tracer);
+}
+
 void
-erts_change_default_tracing(int setflags, Uint *flagsp,
-                            const ErtsTracer tracer)
+erts_change_default_proc_tracing(int setflags, Uint flagsp,
+                                 const ErtsTracer tracer)
 {
     erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
-    if (flagsp) {
-	if (setflags)
-	    default_trace_flags |= *flagsp;
-	else
-	    default_trace_flags &= ~(*flagsp);
-    }
-
-    erts_tracer_update(&default_tracer, tracer);
+    erts_change_default_tracing(
+        setflags, flagsp, tracer,
+        &default_proc_trace_flags,
+        &default_proc_tracer);
+    erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+}
 
-    get_default_tracing(flagsp, NULL);
+void
+erts_change_default_port_tracing(int setflags, Uint flagsp,
+                                 const ErtsTracer tracer)
+{
+    erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+    erts_change_default_tracing(
+        setflags, flagsp, tracer,
+        &default_port_trace_flags,
+        &default_port_tracer);
     erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
 
 void
-erts_get_default_tracing(Uint *flagsp, ErtsTracer *tracerp)
+erts_get_default_proc_tracing(Uint *flagsp, ErtsTracer *tracerp)
+{
+    erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    *tracerp = erts_tracer_nil; /* initialize */
+    get_default_tracing(
+        flagsp, tracerp,
+        &default_proc_trace_flags,
+        &default_proc_tracer);
+    erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+}
+
+void
+erts_get_default_port_tracing(Uint *flagsp, ErtsTracer *tracerp)
 {
     erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
     *tracerp = erts_tracer_nil; /* initialize */
-    get_default_tracing(flagsp, tracerp);
+    get_default_tracing(
+        flagsp, tracerp,
+        &default_port_trace_flags,
+        &default_port_tracer);
     erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
 }
 
@@ -710,9 +769,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
     Eterm operation = am_send;
     ErtsTracerNif *tnif = NULL;
 
-    if (!ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)) {
-	return;
-    }
+    ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND));
 
     if (is_internal_pid(to)) {
 	if (!erts_proc_lookup(to))
@@ -720,10 +777,8 @@ trace_send(Process *p, Eterm to, Eterm msg)
     }
     else if(is_external_pid(to)
 	    && external_pid_dist_entry(to) == erts_this_dist_entry) {
-	char *s;
     send_to_non_existing_process:
-	s = "send_to_non_existing_process";
-	operation = am_atom_put(s, sys_strlen(s));
+	operation = am_send_to_non_existing_process;
     }
 
     if (is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, operation))
@@ -1666,9 +1721,11 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
 
 void
 trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
-
-    send_to_tracer_nif(NULL, &p->common, calling_pid, NULL, am_open,
-                       p->common.id, drv_name);
+    ErtsTracerNif *tnif = NULL;
+    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    if (is_tracer_proc_enabled(NULL, 0, &p->common, &tnif, am_open))
+        send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, am_open,
+                           calling_pid, drv_name);
 }
 
 /* Sends trace message:
@@ -1681,10 +1738,208 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
 void
 trace_port(Port *t_p, Eterm what, Eterm data) {
 
+    ErtsTracerNif *tnif = NULL;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
-    send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, NULL,
-                       am_open, data, THE_NON_VALUE);
+    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what))
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+                           what, data, THE_NON_VALUE);
+}
+
+
+static Eterm
+trace_port_tmp_binary(char *bin, Sint sz, Binary **bptrp, Eterm **hp)
+{
+    if (sz <= ERL_ONHEAP_BIN_LIMIT) {
+        ErlHeapBin *hb = (ErlHeapBin *)*hp;
+        hb->thing_word = header_heap_bin(sz);
+        hb->size = sz;
+        sys_memcpy(hb->data, bin, sz);
+        *hp += heap_bin_size(sz);
+        return make_binary(hb);
+    } else {
+        ProcBin* pb = (ProcBin *)*hp;
+        Binary *bptr = erts_bin_nrml_alloc(sz);
+        erts_refc_init(&bptr->refc, 1);
+        sys_memcpy(bptr->orig_bytes, bin, sz);
+        pb->thing_word = HEADER_PROC_BIN;
+        pb->size = sz;
+        pb->next = NULL;
+        pb->val = bptr;
+        pb->bytes = (byte*) bptr->orig_bytes;
+        pb->flags = 0;
+        *bptrp = bptr;
+        *hp += PROC_BIN_SIZE;
+        return make_binary(pb);
+    }
+}
+
+/* Sends trace message:
+ *    {trace, PortPid, 'receive', {pid(), {command, iolist()}}}
+ *    {trace, PortPid, 'receive', {pid(), {control, pid()}}}
+ *    {trace, PortPid, 'receive', {pid(), exit}}
+ *
+ */
+void
+trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
+{
+    ErtsTracerNif *tnif = NULL;
+    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+		       || erts_thr_progress_is_blocking());
+    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, am_receive)) {
+        /* We can use a stack heap here, as the nif is called in the
+           context of a port */
+#define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT) + 3)
+        DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+
+        Eterm *hp, data, *orig_hp = NULL;
+        Binary *bptr = NULL;
+        va_list args;
+        UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+        hp = local_heap;
+
+        if (what == am_close) {
+            data = what;
+        } else {
+            Eterm arg;
+            va_start(args, what);
+            if (what == am_command) {
+                char *bin = va_arg(args, char *);
+                Sint sz = va_arg(args, Sint);
+                va_end(args);
+                arg = trace_port_tmp_binary(bin, sz, &bptr, &hp);
+            } else if (what == am_call || what == am_control) {
+                unsigned int command = va_arg(args, unsigned int);
+                char *bin = va_arg(args, char *);
+                Sint sz = va_arg(args, Sint);
+                Eterm cmd;
+                va_end(args);
+                arg = trace_port_tmp_binary(bin, sz, &bptr, &hp);
+#if defined(ARCH_32)
+                if (!IS_USMALL(0, command)) {
+                    *hp = make_pos_bignum_header(1);
+                    BIG_DIGIT(hp, 0) = (Uint)command;
+                    cmd = make_big(hp);
+                    hp += 2;
+                } else
+#endif
+                {
+                    cmd = make_small((Sint)command);
+                }
+                arg = TUPLE2(hp, cmd, arg);
+                hp += 3;
+            } else if (what == am_commandv) {
+                ErlIOVec *evp = va_arg(args, ErlIOVec*);
+                int i;
+                va_end(args);
+                if ((6 + evp->vsize * (2+PROC_BIN_SIZE+ERL_SUB_BIN_SIZE)) > LOCAL_HEAP_SIZE) {
+                    hp = erts_alloc(ERTS_ALC_T_TMP,
+                                    (6 + evp->vsize * (2+PROC_BIN_SIZE+ERL_SUB_BIN_SIZE)) * sizeof(Eterm));
+                    orig_hp = hp;
+                }
+                arg = NIL;
+                /* Convert each element in the ErlIOVec to a sub bin that points
+                   to a procbin. We don't have to increment the proc bin refc as
+                   the port task keeps the reference alive. */
+                for (i = evp->vsize-1; i >= 0; i--) {
+                    if (evp->iov[i].iov_len) {
+                        ProcBin* pb = (ProcBin*)hp;
+                        ErlSubBin *sb;
+                        ASSERT(evp->binv[i]);
+                        pb->thing_word = HEADER_PROC_BIN;
+                        pb->val = ErlDrvBinary2Binary(evp->binv[i]);
+                        pb->size = pb->val->orig_size;
+                        pb->next = NULL;
+                        pb->bytes = (byte*) pb->val->orig_bytes;
+                        pb->flags = 0;
+                        hp += PROC_BIN_SIZE;
+
+                        sb = (ErlSubBin*) hp;
+                        sb->thing_word = HEADER_SUB_BIN;
+                        sb->size = evp->iov[i].iov_len;
+                        sb->offs = (byte*)(evp->iov[i].iov_base) - pb->bytes;
+                        sb->orig = make_binary(pb);
+                        sb->bitoffs = 0;
+                        sb->bitsize = 0;
+                        sb->is_writable = 0;
+                        hp += ERL_SUB_BIN_SIZE;
+
+                        arg = CONS(hp, make_binary(sb), arg);
+                        hp += 2;
+                    }
+                }
+                what = am_command;
+            } else {
+                arg = va_arg(args, Eterm);
+                va_end(args);
+            }
+            data = TUPLE2(hp, what, arg);
+            hp += 3;
+        }
+
+        data = TUPLE2(hp, caller, data);
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+                           am_receive, data, THE_NON_VALUE);
+
+        if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
+            erts_bin_free(bptr);
+
+        if (orig_hp)
+            erts_free(ERTS_ALC_T_TMP, orig_hp);
+
+        UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+    }
+#undef LOCAL_HEAP_SIZE
+}
+
+void
+trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists)
+{
+    ErtsTracerNif *tnif = NULL;
+    Eterm op = exists ? am_send : am_send_to_non_existing_process;
+    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+		       || erts_thr_progress_is_blocking());
+    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, op))
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+                           op, msg, receiver);
+}
+
+void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
+{
+    ErtsTracerNif *tnif = NULL;
+    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+		       || erts_thr_progress_is_blocking());
+    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, am_send)) {
+        Eterm msg;
+        Binary* bptr = NULL;
+#define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT))
+        DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+
+        Eterm *hp;
+
+        ERTS_CT_ASSERT(heap_bin_size(ERL_ONHEAP_BIN_LIMIT) >= PROC_BIN_SIZE);
+        UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+        hp = local_heap;
+
+        msg = trace_port_tmp_binary(bin, sz, &bptr, &hp);
+
+        msg = TUPLE2(hp, what, msg);
+        hp += 3;
+        msg = TUPLE2(hp, t_p->common.id, msg);
+        hp += 3;
+
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+                           am_send, msg, to);
+        if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
+            erts_bin_free(bptr);
+
+        UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
+    }
 }
 
 /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
@@ -1702,9 +1957,13 @@ trace_sched_ports(Port *p, Eterm what) {
 
 void
 trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) {
-
-    send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
-                       NULL, am_open, where, THE_NON_VALUE);
+    ErtsTracerNif *tnif = NULL;
+    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+		       || erts_thr_progress_is_blocking());
+    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what))
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
+                           tnif, what, where, THE_NON_VALUE);
 }
 
 /* Port profiling */
@@ -2523,7 +2782,7 @@ is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
 }
 
 int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
-                                ErtsPTabElementCommon *t_p)
+                                ErtsPTabElementCommon *t_p, Eterm type)
 {
     return is_tracer_proc_enabled(c_p, c_p_locks, t_p, NULL, am_trace_status);
 }
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 61f82a6b93..0fefd6f70d 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -68,9 +68,12 @@ ErtsTracer erts_set_system_seq_tracer(Process *c_p,
                                       ErtsProcLocks c_p_locks,
                                       ErtsTracer new);
 ErtsTracer erts_get_system_seq_tracer(void);
-void erts_change_default_tracing(int setflags, Uint *flagsp,
-                                 const ErtsTracer tracerp);
-void erts_get_default_tracing(Uint *flagsp, ErtsTracer *tracerp);
+void erts_change_default_proc_tracing(int setflags, Uint flagsp,
+                                      const ErtsTracer tracerp);
+void erts_get_default_proc_tracing(Uint *flagsp, ErtsTracer *tracerp);
+void erts_change_default_port_tracing(int setflags, Uint flagsp,
+                                      const ErtsTracer tracerp);
+void erts_get_default_port_tracing(Uint *flagsp, ErtsTracer *tracerp);
 void erts_set_system_monitor(Eterm monitor);
 Eterm erts_get_system_monitor(void);
 int erts_is_tracer_valid(Process* p);
@@ -106,6 +109,9 @@ void trace_sched_ports(Port *pp, Eterm);
 void trace_sched_ports_where(Port *pp, Eterm, Eterm);
 void trace_port(Port *, Eterm what, Eterm data);
 void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name);
+void trace_port_receive(Port *, Eterm calling_pid, Eterm tag, ...);
+void trace_port_send(Port *, Eterm to, Eterm msg, int exists);
+void trace_port_send_binary(Port *, Eterm to, Eterm what, char *bin, Sint sz);
 
 /* system_profile */
 void erts_set_system_profile(Eterm profile);
@@ -177,7 +183,7 @@ int erts_finish_breakpointing(void);
 
 /* Nif tracer functions */
 int erts_is_tracer_proc_enabled(Process *c_p, ErtsProcLocks c_p_locks,
-                                ErtsPTabElementCommon *t_p);
+                                ErtsPTabElementCommon *t_p, Eterm type);
 int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer);
 Eterm erts_tracer_to_term(Process *p, ErtsTracer tracer);
 ErtsTracer erts_term_to_tracer(Eterm prefix, Eterm term);
@@ -210,6 +216,7 @@ ERTS_DECLARE_DUMMY(erts_tracer_nil) = NIL;
 
 #define ERTS_TRACER_PROC_IS_ENABLED(PROC)                               \
     (!ERTS_TRACER_IS_NIL(ERTS_TRACER(PROC))                             \
-     && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN, &(PROC)->common))
+     && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN,          \
+                                    &(PROC)->common, am_trace_status))
 
 #endif /* ERL_TRACE_H__ */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index a35c305046..18cfb1e641 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -88,7 +88,7 @@ int erts_port_parallelism = 0;
 static erts_atomic64_t bytes_in;
 static erts_atomic64_t bytes_out;
 
-static void deliver_result(Eterm sender, Eterm pid, Eterm res);
+static void deliver_result(Port *p, Eterm sender, Eterm pid, Eterm res);
 static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *);
 static void terminate_port(Port *p);
 static void pdl_init(void);
@@ -405,7 +405,7 @@ static Port *create_port(char *name,
     prt->os_pid = -1;
 
     /* Set default tracing */
-    erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER(prt));
+    erts_get_default_port_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER(prt));
 
     ERTS_CT_ASSERT(offsetof(Port,common) == 0);
 
@@ -710,7 +710,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
     if (driver->start) {
         ERTS_MSACC_PUSH_STATE_M();
 	if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) {
-	    trace_sched_ports_where(port, am_in, am_start);
+	    trace_sched_ports_where(port, am_in, am_open);
 	}
 	port->caller = pid;
 #ifdef USE_VM_PROBES
@@ -754,7 +754,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	ERTS_MSACC_POP_STATE_M();
 	port->caller = NIL;
 	if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) {
-	    trace_sched_ports_where(port, am_out, am_start);
+	    trace_sched_ports_where(port, am_out, am_open);
 	}
 #ifdef ERTS_SMP
 	if (port->xports)
@@ -1742,6 +1742,10 @@ call_driver_outputv(int bang_op,
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)	
 			   || ERTS_IS_CRASH_DUMPING);
 
+
+        if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+            trace_port_receive(prt, caller, am_commandv, evp);
+
 #ifdef USE_VM_PROBES
 	if (DTRACE_ENABLED(driver_outputv)) {
 	    DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt);
@@ -1868,6 +1872,9 @@ call_driver_output(int bang_op,
         }
 #endif
 
+        if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+            trace_port_receive(prt, caller, am_command, bufp, size);
+
 	prt->caller = caller;
 	(*drv->output)((ErlDrvData) prt->drv_data, bufp, size);
 	prt->caller = NIL;
@@ -2589,7 +2596,10 @@ call_deliver_port_exit(int bang_op,
 	    return ERTS_PORT_OP_DROPPED;
     }
 
-    if (!erts_deliver_port_exit(prt, from, reason, bang_op))
+    if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+        trace_port_receive(prt, from, am_close);
+
+    if (!erts_deliver_port_exit(prt, from, reason, bang_op, broken_link))
 	return ERTS_PORT_OP_DROPPED;
 
 #ifdef USE_VM_PROBES
@@ -2660,13 +2670,13 @@ erts_port_exit(Process *c_p,
 					       ERTS_PORT_SFLGS_INVALID_LOOKUP,
 					       0,
 					       !refp,
-					       am_exit);
+					       am_close);
 
 
 	switch (try_imm_drv_call(&try_call_state)) {
 	case ERTS_TRY_IMM_DRV_CALL_OK: {
 	    res = call_deliver_port_exit(flags & ERTS_PORT_SIG_FLG_BANG_OP,
-					 from,
+					 c_p ? c_p->common.id : from,
 					 prt,
 					 try_call_state.state,
 					 reason,
@@ -2745,8 +2755,11 @@ set_port_connected(int bang_op,
 	    return ERTS_PORT_OP_DROPPED;
 	}
 
+        if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+            trace_port_receive(prt, from, am_connect, connect);
+
 	ERTS_PORT_SET_CONNECTED(prt, connect);
-	deliver_result(prt->common.id, from, am_connected);
+	deliver_result(prt, prt->common.id, from, am_connected);
 
 #ifdef USE_VM_PROBES
 	if(DTRACE_ENABLED(port_command)) {
@@ -2768,10 +2781,22 @@ set_port_connected(int bang_op,
 	erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, prt->common.id);
 	erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, connect);
 
+        if (IS_TRACED_FL(rp, F_TRACE_PROCS))
+            trace_proc(NULL, 0, rp, am_getting_linked, prt->common.id);
+
 	ERTS_PORT_SET_CONNECTED(prt, connect);
 
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
+        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+            trace_port(prt, am_getting_linked, connect);
+        if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+            trace_port_receive(prt, from, am_connect, connect);
+        if (IS_TRACED_FL(prt, F_TRACE_SEND)) {
+            Eterm hp[3];
+            trace_port_send(prt, from, TUPLE2(hp, prt->common.id, am_connected), 1);
+        }
+
 #ifdef USE_VM_PROBES
 	if (DTRACE_ENABLED(port_connect)) {
 	    DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
@@ -2874,8 +2899,11 @@ static void
 port_unlink(Port *prt, Eterm from)
 {
     ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from);
-    if (lnk)
+    if (lnk) {
+        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+            trace_port(prt, am_getting_unlinked, from);
 	erts_destroy_link(lnk);
+    }
 }
 
 static int
@@ -2945,7 +2973,7 @@ port_link_failure(Eterm port_id, Eterm linker)
 					     NIL,
 					     NULL,
 					     0);
-	    if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
+	    if (xres >= 0) {
 		/* We didn't exit the process and it is traced */
 		if (IS_TRACED_FL(rp, F_TRACE_PROCS))
 		    trace_proc(NULL, 0, rp, am_getting_unlinked, port_id);
@@ -2959,10 +2987,15 @@ port_link_failure(Eterm port_id, Eterm linker)
 static void
 port_link(Port *prt, erts_aint32_t state, Eterm to)
 {
-    if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP))
+    if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+        trace_port(prt, am_getting_linked, to);
+    if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
 	erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, to);
-    else
+    } else {
 	port_link_failure(prt->common.id, to);
+        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+            trace_port(prt, am_unlink, to);
+    }
 }
 
 static int
@@ -2970,8 +3003,13 @@ port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigd
 {
     if (op == ERTS_PROC2PORT_SIG_EXEC)
 	port_link(prt, state, sigdp->u.link.to);
-    else
+    else {
+        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+            trace_port(prt, am_getting_linked, sigdp->u.link.to);
 	port_link_failure(sigdp->u.link.port, sigdp->u.link.to);
+        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+            trace_port(prt, am_unlink, sigdp->u.link.to);
+    }
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
 	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true);
     return ERTS_PORT_REDS_LINK;
@@ -3391,7 +3429,7 @@ static int read_linebuf(LineBufContext *bp)
 }
 
 static void
-deliver_result(Eterm sender, Eterm pid, Eterm res)
+deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res)
 {
     Process *rp;
     ErtsProcLocks rp_locks = 0;
@@ -3399,12 +3437,22 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
+    ASSERT(!prt || prt->common.id == sender);
+#ifdef ERTS_SMP
+    ASSERT(!prt || erts_lc_is_port_locked(prt));
+#endif
+
     ASSERT(is_internal_port(sender) && is_internal_pid(pid));
 
     rp = (scheduler
 	  ? erts_proc_lookup(pid)
 	  : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC));
 
+    if (prt && IS_TRACED_FL(prt, F_TRACE_SEND)) {
+        Eterm hp[3];
+        trace_port_send(prt, pid, TUPLE2(hp, sender, res), !!rp);
+    }
+
     if (rp) {
 	Eterm tuple;
 	ErtsMessage *mp;
@@ -3449,6 +3497,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     ErlOffHeap *ohp;
     ErtsProcLocks rp_locks = 0;
     int scheduler = erts_get_scheduler_id() != 0;
+    int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND);
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));    
     ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -3471,7 +3520,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     if (!rp)
 	return;
 
-    mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp);
+    mp = erts_alloc_message_heap(trace_send ? NULL : rp, &rp_locks, need, &hp, &ohp);
 
     listp = NIL;
     if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) {
@@ -3513,6 +3562,9 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
+    if (trace_send)
+        trace_port_send(prt, to, tuple, 1);
+
     ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_message(rp, &rp_locks, mp, tuple);
     if (rp_locks)
@@ -3593,6 +3645,7 @@ deliver_vec_message(Port* prt,			/* Port */
     ErtsProcLocks rp_locks = 0;
     int scheduler = erts_get_scheduler_id() != 0;
     erts_aint32_t state;
+    int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND);
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
     ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -3620,7 +3673,7 @@ deliver_vec_message(Port* prt,			/* Port */
 	need += (hlen+csize)*2;
     }
 
-    mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp);
+    mp = erts_alloc_message_heap(trace_send ? NULL : rp, &rp_locks, need, &hp, &ohp);
 
     listp = NIL;
     iov += vsize;
@@ -3681,6 +3734,9 @@ deliver_vec_message(Port* prt,			/* Port */
     tuple = TUPLE2(hp, prt->common.id, tuple);
     hp += 3;
 
+    if (IS_TRACED_FL(prt, F_TRACE_SEND))
+        trace_port_send(prt, to, tuple, 1);
+
     ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_message(rp, &rp_locks, mp, tuple);
     erts_smp_proc_unlock(rp, rp_locks);
@@ -3822,6 +3878,11 @@ terminate_port(Port *prt)
 	ASSERT(!prt->xports);
 #endif
     }
+
+    if (is_internal_port(send_closed_port_id)
+        && IS_TRACED_FL(prt, F_TRACE_SEND))
+        trace_port_send(prt, connected_id, am_closed, 1);
+
     if(drv->handle != NULL) {
 	erts_smp_rwmtx_rlock(&erts_driver_list_lock);
 	erts_ddll_decrement_port_count(drv->handle); 
@@ -3853,7 +3914,7 @@ terminate_port(Port *prt)
 	erts_flush_async_exit(erts_halt_code, "");
     }
     if (is_internal_port(send_closed_port_id))
-	deliver_result(send_closed_port_id, connected_id, am_closed);
+	deliver_result(NULL, send_closed_port_id, connected_id, am_closed);
 }
 
 void
@@ -3886,7 +3947,7 @@ static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc)
 
 
 typedef struct {
-    Eterm port;
+    Port *port;
     Eterm reason;
 } SweepContext;
 
@@ -3895,10 +3956,13 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
     SweepContext *psc = vpsc;
     DistEntry *dep;
     Process *rp;
-    
+    Eterm port_id = psc->port->common.id;
 
     ASSERT(lnk->type == LINK_PID);
-    
+
+    if (IS_TRACED_FL(psc->port, F_TRACE_PORTS))
+        trace_port(psc->port, am_unlink, lnk->pid);
+
     if (is_external_pid(lnk->pid)) {
 	dep = external_pid_dist_entry(lnk->pid);
 	if(dep != erts_this_dist_entry) {
@@ -3911,9 +3975,9 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
 	    case ERTS_DSIG_PREP_NOT_CONNECTED:
 		break;
 	    case ERTS_DSIG_PREP_CONNECTED:
-		erts_remove_dist_link(&dld, psc->port, lnk->pid, dep);
+		erts_remove_dist_link(&dld, port_id, lnk->pid, dep);
 		erts_destroy_dist_link(&dld);
-		code = erts_dsig_send_exit(&dsd, psc->port, lnk->pid,
+		code = erts_dsig_send_exit(&dsd, port_id, lnk->pid,
 					   psc->reason);
 		ASSERT(code == ERTS_DSIG_SEND_OK);
 		break;
@@ -3927,23 +3991,25 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
 	ASSERT(is_internal_pid(lnk->pid));
 	rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks);
 	if (rp) {
-	    ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), psc->port);
+	    ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id);
 
 	    if (rlnk) {
 		int xres = erts_send_exit_signal(NULL,
-						 psc->port,
+						 port_id,
 						 rp,
 						 &rp_locks, 
 						 psc->reason,
 						 NIL,
 						 NULL,
 						 0);
-		if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
+		if (xres >= 0) {
+                    if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
+                        erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
+                        rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
+                    }
 		    /* We didn't exit the process and it is traced */
-		    if (IS_TRACED_FL(rp, F_TRACE_PROCS)) {
-			trace_proc(NULL, 0, rp, am_getting_unlinked,
-				   psc->port);
-		    }
+		    if (IS_TRACED_FL(rp, F_TRACE_PROCS))
+			trace_proc(NULL, 0, rp, am_getting_unlinked, port_id);
 		}
 		erts_destroy_link(rlnk);
 	    }
@@ -3965,7 +4031,8 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
  */
 
 int
-erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
+erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed,
+                       int drop_normal)
 {
    ErtsLink *lnk;
    Eterm rreason;
@@ -3995,8 +4062,10 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
 		| ERTS_PORT_SFLG_CLOSING))
        return 0;
 
-   if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(p) && from != p->common.id)
+   if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(p)
+       && from != p->common.id && drop_normal) {
        return 0;
+   }
 
    set_state_flags = ERTS_PORT_SFLG_EXITING;
    if (send_closed)
@@ -4007,9 +4076,8 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
    state = erts_atomic32_read_bor_mb(&p->state, set_state_flags);
    state |= set_state_flags;
 
-   if (IS_TRACED_FL(p, F_TRACE_PORTS)) {
+   if (IS_TRACED_FL(p, F_TRACE_PORTS))
    	trace_port(p, am_closed, reason);
-   }
 
    erts_trace_check_exiting(p->common.id);
 
@@ -4019,7 +4087,7 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
        (void) erts_unregister_name(NULL, 0, p, p->common.u.alive.reg->name);
 
    {
-       SweepContext sc = {p->common.id, rreason};
+       SweepContext sc = {p, rreason};
        lnk = ERTS_P_LINKS(p);
        ERTS_P_LINKS(p) = NULL;
        erts_sweep_links(lnk, &sweep_one_link, &sc);
@@ -4141,7 +4209,10 @@ call_driver_control(Eterm caller,
                 command, size);
     }
 #endif
-    
+
+    if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+        trace_port_receive(prt, caller, am_control, command, bufp, size);
+
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
 
 #ifdef USE_LTTNG_VM_TRACEPOINTS
@@ -4168,6 +4239,9 @@ call_driver_control(Eterm caller,
     if (cres < 0)
 	return ERTS_PORT_OP_BADARG;
 
+    if (IS_TRACED_FL(prt, F_TRACE_SEND))
+        trace_port_send_binary(prt, caller, am_control, *resp_bufp, cres);
+
     *from_size = (ErlDrvSizeT) cres;
 
     return ERTS_PORT_OP_DONE;
@@ -4577,6 +4651,9 @@ call_driver_call(Eterm caller,
     }
 #endif
 
+    if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+        trace_port_receive(prt, caller, am_call, command, bufp, size);
+
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
 
     prt->caller = caller;
@@ -4595,6 +4672,9 @@ call_driver_call(Eterm caller,
 	|| ((byte) (*resp_bufp)[0]) != VERSION_MAGIC)
 	return ERTS_PORT_OP_BADARG;
 
+    if (IS_TRACED_FL(prt, F_TRACE_SEND))
+        trace_port_send_binary(prt, caller, am_call, *resp_bufp, cres);
+
     *from_size = (ErlDrvSizeT) cres;
 
     return ERTS_PORT_OP_DONE;
@@ -5038,6 +5118,7 @@ reply_io_bytes(void *vreq)
 	eout = erts_bld_uint64(&hp, NULL, out);
 
 	msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout);
+
 	erts_queue_message(rp, &rp_locks, mp, msg);
 
 	if (req->sched_id == sched_id)
@@ -5509,6 +5590,7 @@ void driver_report_exit(ErlDrvPort ix, int status)
    ErtsProcLocks rp_locks = 0;
    int scheduler = erts_get_scheduler_id() != 0;
    Port* prt = erts_drvport2port(ix);
+   int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND);
 
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
        return;
@@ -5525,12 +5607,15 @@ void driver_report_exit(ErlDrvPort ix, int status)
    if (!rp)
        return;
 
-   mp = erts_alloc_message_heap(rp, &rp_locks, 3+3, &hp, &ohp);
+   mp = erts_alloc_message_heap(trace_send ? NULL : rp, &rp_locks, 3+3, &hp, &ohp);
 
    tuple = TUPLE2(hp, am_exit_status, make_small(status));
    hp += 3;
    tuple = TUPLE2(hp, prt->common.id, tuple);
 
+    if (IS_TRACED_FL(prt, F_TRACE_SEND))
+        trace_port_send(prt, pid, tuple, 1);
+
    ERL_MESSAGE_TOKEN(mp) = am_undefined;
    erts_queue_message(rp, &rp_locks, mp, tuple);
 
@@ -5626,13 +5711,13 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp)
  */
 
 static int
-driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
+driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len)
 {
 #define HEAP_EXTRA 200
 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0)
     Uint need = 0;
     int depth = 0;
-    int res;
+    int res = 0;
     ErlDrvTermData* ptr;
     ErlDrvTermData* ptr_end;
     DECLARE_ESTACK(stack); 
@@ -5851,11 +5936,26 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	  ? erts_proc_lookup(to)
 	  : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC));
     if (!rp) {
-	res = 0;
-	goto done;
-    }
+        if (!prt || !IS_TRACED_FL(prt, F_TRACE_SEND))
+            goto done;
+        if (!erts_is_tracer_proc_enabled(NULL, 0, &prt->common, am_send))
+            goto done;
 
-    (void) erts_factory_message_create(&factory, rp, &rp_locks, need);
+	res = -2;
+
+        /* We allocate a temporary heap to be used to create
+           the message that may be sent using tracing */
+        erts_factory_tmp_init(&factory, erts_alloc(ERTS_ALC_T_DRIVER, need*sizeof(Eterm)),
+                              need, ERTS_ALC_T_DRIVER);
+
+    } else {
+        /* We force the creation of a heap fragment (rp == NULL) when send
+           tracing so that we don't have the main lock of the process while
+           tracing */
+        Process *trace_rp = prt && IS_TRACED_FL(prt, F_TRACE_SEND) ? NULL : rp;
+        (void) erts_factory_message_create(&factory, trace_rp, &rp_locks, need);
+        res = 1;
+    }
 
     /*
      * Interpret the instructions and build the term.
@@ -6118,23 +6218,40 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 	ESTACK_PUSH(stack, mess);
     }
 
-    res = 1;
-
  done:
 
     if (res > 0) {
 	mess = ESTACK_POP(stack);  /* get resulting value */
 	erts_factory_trim_and_close(&factory, &mess, 1);
+
+        if (prt && IS_TRACED_FL(prt, F_TRACE_SEND))
+            trace_port_send(prt, to, mess, 1);
+
 	/* send message */
         ERL_MESSAGE_TOKEN(factory.message) = am_undefined;
 	erts_queue_message(rp, &rp_locks, factory.message, mess);
     }
+    else if (res == -2) {
+        /* this clause only happens when we were requested to
+           generate a send trace, but the process to send to
+           did not exist any more */
+        mess = ESTACK_POP(stack);  /* get resulting value */
+
+        trace_port_send(prt, to, mess, 0);
+
+	erts_factory_trim_and_close(&factory, &mess, 1);
+        erts_free(ERTS_ALC_T_DRIVER, factory.hp_start);
+        res = 0;
+    }
     else {
 	if (b2t.ix > b2t.used)
 	    b2t.used = b2t.ix;
 	for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++)
 	    erts_binary2term_abort(&b2t.state[b2t.ix]);
-	erts_factory_undo(&factory);
+        if (factory.mode != FACTORY_CLOSED) {
+            ERL_MESSAGE_TERM(factory.message) = am_undefined;
+            erts_factory_undo(&factory);
+        }
     }
     if (rp) {
 	if (rp_locks)
@@ -6150,7 +6267,8 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 }
 
 static ERTS_INLINE int
-deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p)
+deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p,
+                        Port **trace_prt)
 {
 #ifdef ERTS_SMP
     ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay();
@@ -6178,8 +6296,11 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p)
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
 	erts_thr_progress_unmanaged_continue(dhndl);
 	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
-    }
+    } else
 #endif
+    {
+        *trace_prt = prt;
+    }
     ERTS_SMP_LC_ASSERT(dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED
 		       ? erts_lc_is_port_locked(prt)
 		       : !erts_lc_is_port_locked(prt));
@@ -6190,10 +6311,11 @@ int erl_drv_output_term(ErlDrvTermData port_id, ErlDrvTermData* data, int len)
 {
     /* May be called from arbitrary thread */
     Eterm connected;
-    int res = deliver_term_check_port(port_id, &connected);
+    Port *prt = NULL;
+    int res = deliver_term_check_port(port_id, &connected, &prt);
     if (res <= 0)
 	return res;
-    return driver_deliver_term(connected, data, len);
+    return driver_deliver_term(prt, connected, data, len);
 }
 
 /*
@@ -6217,7 +6339,7 @@ driver_output_term(ErlDrvPort drvport, ErlDrvTermData* data, int len)
     if (state & ERTS_PORT_SFLG_CLOSING)
 	return 0;
 
-    return driver_deliver_term(ERTS_PORT_GET_CONNECTED(prt), data, len);
+    return driver_deliver_term(prt, ERTS_PORT_GET_CONNECTED(prt), data, len);
 }
 
 int erl_drv_send_term(ErlDrvTermData port_id,
@@ -6226,10 +6348,11 @@ int erl_drv_send_term(ErlDrvTermData port_id,
 		      int len)
 {
     /* May be called from arbitrary thread */
-    int res = deliver_term_check_port(port_id, NULL);
+    Port *prt = NULL;
+    int res = deliver_term_check_port(port_id, NULL, &prt);
     if (res <= 0)
 	return res;
-    return driver_deliver_term(to, data, len);
+    return driver_deliver_term(prt, to, data, len);
 }
 
 /*
@@ -6248,20 +6371,21 @@ driver_send_term(ErlDrvPort drvport,
      * to make this access safe without using a less efficient
      * internal data representation for ErlDrvPort.
      */
+    Port* prt = NULL;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 #ifdef ERTS_SMP
     if (erts_thr_progress_is_managed_thread())
 #endif
     {
 	erts_aint32_t state;
-	Port* prt = erts_drvport2port_state(drvport, &state);
+	prt = erts_drvport2port_state(drvport, &state);
 	if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	    return -1; /* invalid (dead) */
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 	if (state & ERTS_PORT_SFLG_CLOSING)
 	    return 0;
     }
-    return driver_deliver_term(to, data, len);
+    return driver_deliver_term(prt, to, data, len);
 }
 
 
@@ -7437,7 +7561,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
     if (state & ERTS_PORT_SFLG_CLOSING) {
 	terminate_port(prt);
     } else if (eof && (state & ERTS_PORT_SFLG_SOFT_EOF)) {
-	deliver_result(prt->common.id, ERTS_PORT_GET_CONNECTED(prt), am_eof);
+	deliver_result(prt, prt->common.id, ERTS_PORT_GET_CONNECTED(prt), am_eof);
     } else {
 	/* XXX UGLY WORK AROUND, Let erts_deliver_port_exit() terminate the port */
 	if (prt->port_data_lock)
@@ -7445,7 +7569,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
 	prt->ioq.size = 0;
 	if (prt->port_data_lock)
 	    driver_pdl_unlock(prt->port_data_lock);
-	erts_deliver_port_exit(prt, prt->common.id, eof ? am_normal : term, 0);
+	erts_deliver_port_exit(prt, prt->common.id, eof ? am_normal : term, 0, 0);
     }
     return 0;
 }
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index f4a889b8f8..e995dcbd8a 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -538,8 +538,12 @@ int erts_unregister_name(Process *c_p,
 	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port));
 
 	    rp->pt->common.u.alive.reg = NULL;
-	    
+
 	    if (IS_TRACED_FL(port, F_TRACE_PORTS)) {
+                if (current_c_p_locks) {
+                    erts_smp_proc_unlock(c_p, current_c_p_locks);
+                    current_c_p_locks = 0;
+                }
 		trace_port(port, am_unregister, r.name);
 	    }
 
-- 
cgit v1.2.3


From d15ddcb9d407462133ec5d84c45353bd5a928167 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 9 Feb 2016 22:04:57 +0100
Subject: erts: Fix FPE bug in erl_nif

erts_block/unblock_fpe should only be called at entry to/exit from
native user code.
---
 erts/emulator/beam/beam_emu.c |  2 +-
 erts/emulator/beam/erl_nif.c  | 51 +++++++++++++++++++++++++++----------------
 erts/emulator/beam/global.h   |  2 +-
 3 files changed, 34 insertions(+), 21 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c356863b60..7cb259cb8a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3460,7 +3460,7 @@ do {						\
 		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
-		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
+		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
 		live_hf_end = c_p->mbuf;
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 59a2b20ff2..7ab5421c91 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -120,7 +120,8 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need)
 }
 #endif
 
-void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
+void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
+                  Process* tracee)
 {
     env->mod_nif = mod_nif;
     env->proc = p;
@@ -130,7 +131,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
     env->fpe_was_unmasked = erts_block_fpe();
     env->tmp_obj_list = NULL;
     env->exception_thrown = 0;
-    env->tracee = NULL;
+    env->tracee = tracee;
 }
 
 /* Temporary object header, auto-deallocated when NIF returns
@@ -232,7 +233,8 @@ struct enif_msg_environment_t
 
 static ERTS_INLINE void
 setup_nif_env(struct enif_msg_environment_t* msg_env,
-              struct erl_module_nif* mod)
+              struct erl_module_nif* mod,
+              Process* tracee)
 {
     Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */
 
@@ -241,7 +243,6 @@ setup_nif_env(struct enif_msg_environment_t* msg_env,
     msg_env->env.heap_frag = NULL;
     msg_env->env.mod_nif = mod;
     msg_env->env.tmp_obj_list = NULL;
-    msg_env->env.fpe_was_unmasked = erts_block_fpe();
     msg_env->env.proc = &msg_env->phony_proc;
     msg_env->env.exception_thrown = 0;
     memset(&msg_env->phony_proc, 0, sizeof(Process));
@@ -255,13 +256,14 @@ setup_nif_env(struct enif_msg_environment_t* msg_env,
     msg_env->phony_proc.space_verified = 0;
     msg_env->phony_proc.space_verified_from = NULL;
 #endif
+    msg_env->env.tracee = tracee;
 }
 
 ErlNifEnv* enif_alloc_env(void)
 {
     struct enif_msg_environment_t* msg_env =
 	erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t));
-    setup_nif_env(msg_env, NULL);
+    setup_nif_env(msg_env, NULL, NULL);
     return &msg_env->env;
 }
 void enif_free_env(ErlNifEnv* env)
@@ -270,6 +272,20 @@ void enif_free_env(ErlNifEnv* env)
     erts_free(ERTS_ALC_T_NIF, env);
 }
 
+static ERTS_INLINE void pre_nif_noproc(struct enif_msg_environment_t* msg_env,
+                                       struct erl_module_nif* mod,
+                                       Process* tracee)
+{
+    setup_nif_env(msg_env, mod, tracee);
+    msg_env->env.fpe_was_unmasked = erts_block_fpe();
+}
+
+static ERTS_INLINE void post_nif_noproc(struct enif_msg_environment_t* msg_env)
+{
+    erts_unblock_fpe(msg_env->env.fpe_was_unmasked);
+    enif_clear_env(&msg_env->env);
+}
+
 static ERTS_INLINE void clear_offheap(ErlOffHeap* oh)
 {
     oh->first = NULL;
@@ -294,7 +310,6 @@ void enif_clear_env(ErlNifEnv* env)
     menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
     
     ASSERT(!is_offheap(&MSO(p)));
-    erts_unblock_fpe(env->fpe_was_unmasked);
     free_tmp_objs(env);
 }
 
@@ -465,7 +480,7 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     else {
         /* This clause is taken when the nif is called in the context
            of a traced process. We do not know which locks we have
-           so we have to do a try lock and if that fails we en queue
+           so we have to do a try lock and if that fails we enqueue
            the message in a special trace message output queue of the
            tracee */
         ErlTraceMessageQueue *msgq;
@@ -1648,9 +1663,9 @@ static void close_lib(struct erl_module_nif* lib)
 
     if (lib->entry != NULL && lib->entry->unload != NULL) {
 	struct enif_msg_environment_t msg_env;
-        setup_nif_env(&msg_env, lib);
+        pre_nif_noproc(&msg_env, lib, NULL);
 	lib->entry->unload(&msg_env.env, lib->priv_data);
-        enif_clear_env(&msg_env.env);
+        post_nif_noproc(&msg_env);
     }
     if (!erts_is_static_nif(lib->handle))
       erts_sys_ddll_close(lib->handle);
@@ -1797,9 +1812,9 @@ static void nif_resource_dtor(Binary* bin)
 
     if (type->dtor != NULL) {
         struct enif_msg_environment_t msg_env;
-        setup_nif_env(&msg_env, type->owner);
+        pre_nif_noproc(&msg_env, type->owner, NULL);
 	type->dtor(&msg_env.env, resource->data);
-        enif_clear_env(&msg_env.env);
+        post_nif_noproc(&msg_env);
     }
     if (erts_refc_dectest(&type->refc, 0) == 0) {
 	ASSERT(type->next == NULL);
@@ -2959,7 +2974,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    }
 	    old_func = next_func(mod->curr.nif->entry, &old_incr, old_func);
 	}
-	erts_pre_nif(&env, BIF_P, lib);
+	erts_pre_nif(&env, BIF_P, lib, NULL);
 	veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2);
 	erts_post_nif(&env);
 	if (veto) {
@@ -2980,7 +2995,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library.");
 		goto error;
 	    }
-	    erts_pre_nif(&env, BIF_P, lib);
+	    erts_pre_nif(&env, BIF_P, lib, NULL);
 	    veto = entry->upgrade(&env, &lib->priv_data, &mod->old.nif->priv_data, BIF_ARG_2);
 	    erts_post_nif(&env);
 	    if (veto) {
@@ -2991,7 +3006,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		commit_opened_resource_types(lib);
 	}
 	else if (entry->load != NULL) { /********* Initial load ***********/
-	    erts_pre_nif(&env, BIF_P, lib);
+	    erts_pre_nif(&env, BIF_P, lib, NULL);
 	    veto = entry->load(&env, &lib->priv_data, BIF_ARG_2);
 	    erts_post_nif(&env);
 	    if (veto) {
@@ -3152,8 +3167,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
     if (p) {
         struct enif_environment_t env;
         ASSERT(is_internal_pid(p->common.id));
-        erts_pre_nif(&env, p, mod);
-        env.tracee = tracee;
+        erts_pre_nif(&env, p, mod, tracee);
         nif_result = (*fun->fptr)(&env, argc, argv);
         if (env.exception_thrown)
             nif_result = THE_NON_VALUE;
@@ -3162,12 +3176,11 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
         /* Nif call was done without a process context,
            so we create a phony one. */
         struct enif_msg_environment_t msg_env;
-        setup_nif_env(&msg_env, mod);
-        msg_env.env.tracee = tracee;
+        pre_nif_noproc(&msg_env, mod, tracee);
         nif_result = (*fun->fptr)(&msg_env.env, argc, argv);
         if (msg_env.env.exception_thrown)
             nif_result = THE_NON_VALUE;
-        enif_clear_env(&msg_env.env);
+        post_nif_noproc(&msg_env);
     }
 
     return nif_result;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 5c2e8bbd09..a7bc990deb 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -59,7 +59,7 @@ struct enif_environment_t /* ErlNifEnv */
     Process *tracee;
 };
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
-			 struct erl_module_nif*);
+			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(int to, void* to_arg);
-- 
cgit v1.2.3


From d698386e5b101a02786c85af70f1513c6beb191b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 3 Mar 2016 16:33:22 +0100
Subject: erts: Silence harmless valgrind warning in dec_term

provoked by nif_SUITE:nif_binary_to_term.

If we fail to decode an immediate (unsafe atom for example) with
a dummy factory then hp and factory->hp will both be uninitialized
and valgrind will complain about comparing them.
---
 erts/emulator/beam/external.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9f43240b7e..723c25ff77 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3958,9 +3958,13 @@ error:
      * Must unlink all off-heap objects that may have been
      * linked into the process. 
      */
-    if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */
-	factory->hp = hp;   /* the largest must be the freshest */
+    if (factory->mode != FACTORY_CLOSED) {
+	if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */
+	    factory->hp = hp;   /* the largest must be the freshest */
+	}
     }
+    else ASSERT(factory->hp == hp);
+
 error_hamt:
     erts_factory_undo(factory);
     PSTACK_DESTROY(hamt_array);
-- 
cgit v1.2.3


From 71a67e69ce792f5464479fb15257ad07dc30b856 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 1 Mar 2016 18:19:15 +0100
Subject: erts: Do 'unregister' as "self-tracing"

We have the main lock on rp->p, so why not?
---
 erts/emulator/beam/register.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index e995dcbd8a..77f79fcea4 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -560,7 +560,7 @@ int erts_unregister_name(Process *c_p,
 #endif
 	    rp->p->common.u.alive.reg = NULL;
 	    if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) {
-		trace_proc(c_p, current_c_p_locks,
+                trace_proc(rp->p, (c_p == rp->p) ? c_p_locks : ERTS_PROC_LOCK_MAIN,
                            rp->p, am_unregister, r.name);
 	    }
 #ifdef ERTS_SMP
-- 
cgit v1.2.3


From b8ca148ba2c98a2bec923dbac6ed52a80aa167ca Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 3 Mar 2016 16:52:07 +0100
Subject: erts: send and receive no longer need status lock

Rickards said that this was ok
---
 erts/emulator/beam/erl_process_lock.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index d81ef63af2..2cccf0697a 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -209,10 +209,8 @@ typedef struct erts_proc_lock_t_ {
 
 /* ERTS_PROC_LOCKS_* are combinations of process locks */
 
-#define ERTS_PROC_LOCKS_MSG_RECEIVE	(ERTS_PROC_LOCK_MSGQ		\
-					 | ERTS_PROC_LOCK_STATUS)
-#define ERTS_PROC_LOCKS_MSG_SEND	(ERTS_PROC_LOCK_MSGQ		\
-					 | ERTS_PROC_LOCK_STATUS)
+#define ERTS_PROC_LOCKS_MSG_RECEIVE	ERTS_PROC_LOCK_MSGQ
+#define ERTS_PROC_LOCKS_MSG_SEND	ERTS_PROC_LOCK_MSGQ
 #define ERTS_PROC_LOCKS_XSIG_SEND	ERTS_PROC_LOCK_STATUS
 
 #define ERTS_PROC_LOCKS_ALL \
-- 
cgit v1.2.3


From 3738103d0b1195e570a7525c4370cd490a5368aa Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Mon, 29 Feb 2016 15:09:26 +0100
Subject: erts: Add 'spawned' trace event to 'procs' trace flag

OTP-13497

This trace event is triggered when a process is created from the
process that is created.
---
 erts/emulator/beam/atom.names    |  1 +
 erts/emulator/beam/erl_process.c | 13 ++++++++++++-
 erts/emulator/beam/erl_trace.c   |  6 +++---
 erts/emulator/beam/erl_trace.h   |  2 +-
 4 files changed, 17 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index bd1005867c..8c51f788c0 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -571,6 +571,7 @@ atom size
 atom sl_alloc
 atom spawn_executable
 atom spawn_driver
+atom spawned
 atom ssl_tls
 atom stack_size
 atom start
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1398bc9c61..a520005c35 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11176,12 +11176,23 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
             locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
             erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
             erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
-            trace_proc_spawn(parent, p->common.id, mod, func, args);
+            trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args);
             if (so->flags & SPO_LINK)
                 trace_proc(parent, locks, parent, am_link, p->common.id);
         }
     }
 
+    if (IS_TRACED_FL(p, F_TRACE_PROCS)) {
+        if (locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) {
+            locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_smp_proc_unlock(p, locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE));
+            erts_smp_proc_unlock(parent, locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE));
+        }
+        trace_proc_spawn(p, am_spawned, parent->common.id, mod, func, args);
+        if (so->flags & SPO_LINK)
+            trace_proc(p, locks, p, am_getting_linked, parent->common.id);
+    }
+
     /*
      * Check if this process should be initially linked to its parent.
      */
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 5ba58e9350..3027e34968 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1249,20 +1249,20 @@ trace_proc(Process *c_p, ErtsProcLocks c_p_locks,
  * and 'args' may be a deep term.
  */
 void
-trace_proc_spawn(Process *p, Eterm pid,
+trace_proc_spawn(Process *p, Eterm what, Eterm pid,
 		 Eterm mod, Eterm func, Eterm args)
 {
     ErtsTracerNif *tnif = NULL;
     if (is_tracer_proc_enabled(p, ERTS_PROC_LOCKS_ALL &
                                ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE),
-                               &p->common, &tnif, am_spawn)) {
+                               &p->common, &tnif, what)) {
         Eterm mfa;
         Eterm* hp;
 
         hp = HAlloc(p, 4);
         mfa = TUPLE3(hp, mod, func, args);
         hp += 4;
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, am_spawn, pid, mfa);
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, what, pid, mfa);
     }
 }
 
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 0fefd6f70d..14d0c36016 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -100,7 +100,7 @@ void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
 void erts_trace_return_to(Process *p, BeamInstr *pc);
 void trace_sched(Process*, ErtsProcLocks, Eterm);
 void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm);
-void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
+void trace_proc_spawn(Process*, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args);
 void save_calls(Process *p, Export *);
 void trace_gc(Process *p, Eterm what);
 /* port tracing */
-- 
cgit v1.2.3


From a1cde0f881ee4865d38e2e6e88cc71dcd362fdd2 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 17 Mar 2016 11:15:54 +0100
Subject: erts: Deallocate heap fragments from trace nif calls

Any heap fragment created during a nif call to a tracer nif
should be free'd immediately in order for the GC not to treat
it as live data.
---
 erts/emulator/beam/beam_emu.c |  2 +-
 erts/emulator/beam/erl_nif.c  | 31 ++++++++++++++++++++++++++++---
 2 files changed, 29 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 7cb259cb8a..254bfe4ba0 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3346,7 +3346,7 @@ do {						\
  OpCase(normal_exit): {
      SWAPOUT;
      c_p->freason = EXC_NORMAL;
-     c_p->arity = 0;		/* In case this process will ever be garbed again. */
+     c_p->arity = 0;		/* In case this process will never be garbed again. */
      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
      erts_do_exit_process(c_p, am_normal);
      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 7ab5421c91..73c0eb8eba 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -92,11 +92,12 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need)
 }
 
 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
-{    
+{
     env->hp = hp;
-    if (env->heap_frag == NULL) {       
+    if (env->heap_frag == NULL) {
 	ASSERT(HEAP_LIMIT(env->proc) == env->hp_end);
-	HEAP_TOP(env->proc) = env->hp;	
+        ASSERT(env->hp + need > env->hp_end);
+	HEAP_TOP(env->proc) = env->hp;
     }
     else {
 	env->heap_frag->used_size = hp - env->heap_frag->mem;
@@ -3165,13 +3166,37 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
                            || erts_smp_thr_progress_is_blocking());
 #endif
     if (p) {
+        /* This is almost a normal nif call like in beam_emu,
+           except that any heap fragment created in the nif will be
+           discarded without checking if anything in it is live.
+           This is because we cannot do a GC here as we don't know
+           the number of live registers that have to be preserved.
+           This means that any heap part of the returned term may
+           not be used outside this function. */
         struct enif_environment_t env;
+        ErlHeapFragment *orig_hf = MBUF(p);
+        ErlOffHeap orig_oh = MSO(p);
         ASSERT(is_internal_pid(p->common.id));
+        MBUF(p) = NULL;
+        clear_offheap(&MSO(p));
+
         erts_pre_nif(&env, p, mod, tracee);
         nif_result = (*fun->fptr)(&env, argc, argv);
         if (env.exception_thrown)
             nif_result = THE_NON_VALUE;
         erts_post_nif(&env);
+
+        /* Free any offheap and heap fragments created in nif */
+        if (MSO(p).first) {
+            erts_cleanup_offheap(&MSO(p));
+            clear_offheap(&MSO(p));
+        }
+        if (MBUF(p))
+            free_message_buffer(MBUF(p));
+
+        /* restore original heap fragment list */
+        MBUF(p) = orig_hf;
+        MSO(p) = orig_oh;
     } else {
         /* Nif call was done without a process context,
            so we create a phony one. */
-- 
cgit v1.2.3


From a1b352dfc68d501a385240cdb7f45a96cf9e3358 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Wed, 30 Mar 2016 10:22:28 +0200
Subject: erts: Add comment about future trace optimizations

---
 erts/emulator/beam/erl_trace.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 3027e34968..cd3c14e213 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -20,6 +20,19 @@
 
 /*
  * Support functions for tracing.
+ *
+ * Ideas for future speed improvements in tracing framework:
+ *  * Move ErtsTracerNif into ErtsTracer
+ *     + Removes need for locking
+ *     + Removes hash lookup overhead
+ *     + Use a refc on the ErtsTracerNif to know when it can
+ *       be freed. We don't want to allocate a separate
+ *       ErtsTracerNif for each module used.
+ *  * Optimize GenericBp for cache locality by reusing equivalent
+ *    GenericBp and GenericBpData in multiple tracer points.
+ *     + Possibly we want to use specialized instructions for different
+ *       types of trace so that the knowledge of which struct is used
+ *       can be in the instruction.
  */
 
 #ifdef HAVE_CONFIG_H
-- 
cgit v1.2.3


From c3e7acb4fe304d117f7361292d36f5d73df3e1c7 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 5 Apr 2016 11:26:52 +0200
Subject: erts: Make trace_delivered go via sys msg dispatcher again

This is needed as otherwise messages from system_profile
will not be guaranteed to arrive before trace delivered.
---
 erts/emulator/beam/erl_bif_trace.c | 23 +++++++++++++++--------
 erts/emulator/beam/erl_trace.c     | 16 ++++++++++++++++
 erts/emulator/beam/erl_trace.h     |  1 +
 3 files changed, 32 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 7e2fe7fcd4..ff2018aa27 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -2253,25 +2253,32 @@ static void
 reply_trace_delivered_all(void *vtdarp)
 {
     ErtsTraceDeliveredAll *tdarp = (ErtsTraceDeliveredAll *) vtdarp;
-    ErtsProcLocks rp_locks = 0;
 
     if (erts_smp_atomic32_dec_read_nob(&tdarp->refc) == 0) {
+        Eterm ref_copy, msg;
         Process *rp = tdarp->proc;
         Eterm *hp = NULL;
-        ErlOffHeap *ohp = NULL;
-        ErtsMessage *mp = NULL;
-        Eterm ref_copy, msg;
-
+        ErlOffHeap *ohp;
+#ifdef ERTS_SMP
+        ErlHeapFragment *bp;
+        bp = new_message_buffer(4 + NC_HEAP_SIZE(tdarp->ref));
+        hp = &bp->mem[0];
+        ohp = &bp->off_heap;
+#else
+        ErtsProcLocks rp_locks = 0;
+        ErtsMessage *mp;
         mp = erts_alloc_message_heap(
             rp, &rp_locks, 4 + NC_HEAP_SIZE(tdarp->ref), &hp, &ohp);
+#endif
 
         ref_copy = STORE_NC(&hp, ohp, tdarp->ref);
         msg = TUPLE3(hp, am_trace_delivered, tdarp->target, ref_copy);
 
+#ifdef ERTS_SMP
+        erts_send_sys_msg_proc(rp->common.id, rp->common.id, msg, bp);
+#else
         erts_queue_message(rp, &rp_locks, mp, msg);
-
-        if (rp_locks)
-            erts_smp_proc_unlock(rp, rp_locks);
+#endif
 
 	erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp);
         erts_proc_dec_refc(rp);
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index cd3c14e213..bd88769dfc 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -84,6 +84,7 @@ enum ErtsSysMsgType {
     SYS_MSG_TYPE_UNDEFINED,
     SYS_MSG_TYPE_SYSMON,
     SYS_MSG_TYPE_ERRLGR,
+    SYS_MSG_TYPE_PROC_MSG,
     SYS_MSG_TYPE_SYSPROF
 };
 
@@ -2157,6 +2158,13 @@ erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp)
     enqueue_sys_msg(SYS_MSG_TYPE_ERRLGR, from, am_error_logger, msg, bp);
 }
 
+void
+erts_send_sys_msg_proc(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp)
+{
+    ASSERT(is_internal_pid(to));
+    enqueue_sys_msg(SYS_MSG_TYPE_PROC_MSG, from, to, msg, bp);
+}
+
 #ifdef DEBUG_PRINTOUTS
 static void
 print_msg_type(ErtsSysMsgQ *smqp)
@@ -2171,6 +2179,9 @@ print_msg_type(ErtsSysMsgQ *smqp)
     case SYS_MSG_TYPE_ERRLGR:
 	erts_fprintf(stderr, "ERRLGR ");
 	break;
+    case SYS_MSG_TYPE_PROC_MSG:
+       erts_fprintf(stderr, "PROC_MSG ");
+       break;
     default:
 	erts_fprintf(stderr, "??? ");
 	break;
@@ -2241,6 +2252,8 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 		     no_elgger, tag, CAR(list_val(tp[3])));
 	break;
     }
+    case SYS_MSG_TYPE_PROC_MSG:
+        break;
     default:
 	ASSERT(0);
     }
@@ -2358,6 +2371,9 @@ sys_msg_dispatcher_func(void *unused)
 	    print_msg_type(smqp);
 #endif
 	    switch (smqp->type) {
+            case SYS_MSG_TYPE_PROC_MSG:
+                receiver = smqp->to;
+                break;
 	    case SYS_MSG_TYPE_SYSMON:
 		receiver = erts_get_system_monitor();
 		if (smqp->from == receiver) {
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 14d0c36016..177fd373a6 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -87,6 +87,7 @@ void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
 					    Eterm,
 					    ErlHeapFragment *));
 void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
+void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
 #endif
 
 void trace_send(Process*, Eterm, Eterm);
-- 
cgit v1.2.3


From cff38617986001e0a5f3f48de20acbeccceea978 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 7 Apr 2016 16:33:26 +0200
Subject: erts: Don't trace on link events when port is dead

---
 erts/emulator/beam/io.c | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 18cfb1e641..b14ca77a04 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3004,11 +3004,7 @@ port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigd
     if (op == ERTS_PROC2PORT_SIG_EXEC)
 	port_link(prt, state, sigdp->u.link.to);
     else {
-        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
-            trace_port(prt, am_getting_linked, sigdp->u.link.to);
 	port_link_failure(sigdp->u.link.port, sigdp->u.link.to);
-        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
-            trace_port(prt, am_unlink, sigdp->u.link.to);
     }
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
 	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true);
-- 
cgit v1.2.3


From 6cbd63b281a9690b2db5fead3f756d34140f8fb5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 12 Apr 2016 20:51:49 +0200
Subject: erts: Tweak defaults for literal allocator

Reduce main carrier size
and number of free descriptors.
---
 erts/emulator/beam/erl_alloc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 9cbe00d719..6b5f5bd63e 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -307,9 +307,9 @@ set_default_literal_alloc_opts(struct au_init *ip)
     ip->init.util.name_prefix	= "literal_";
     ip->init.util.alloc_no	= ERTS_ALC_A_LITERAL;
 #ifndef SMALL_MEMORY
-    ip->init.util.mmbcs 	= 2*1024*1024; /* Main carrier size */
+    ip->init.util.mmbcs 	= 1024*1024; /* Main carrier size */
 #else
-    ip->init.util.mmbcs 	= 1*1024*1024; /* Main carrier size */
+    ip->init.util.mmbcs 	= 256*1024; /* Main carrier size */
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_LITERAL;
     ip->init.util.asbcst	= 0;
-- 
cgit v1.2.3


From 499a872d5f6ea09d23eb7b04ea5de2f6d3fabd98 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 11 Apr 2016 18:13:31 +0200
Subject: erts: Refactor callbacks for literal mseg alloc

Make the callbacks more general to be usable for any allocator
that that uses its own ErtsMemMapper.
---
 erts/emulator/beam/erl_alloc.c      |  7 ++++---
 erts/emulator/beam/erl_alloc_util.c | 14 ++++++++------
 erts/emulator/beam/erl_alloc_util.h |  8 +++++---
 3 files changed, 17 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 6b5f5bd63e..efe31193b9 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -329,9 +329,10 @@ set_default_literal_alloc_opts(struct au_init *ip)
     ip->init.util.sys_dealloc  = &erts_alcu_literal_32_sys_dealloc;
 #elif defined(ARCH_64)
 # ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
-    ip->init.util.mseg_alloc    = &erts_alcu_literal_64_mseg_alloc;
-    ip->init.util.mseg_realloc  = &erts_alcu_literal_64_mseg_realloc;
-    ip->init.util.mseg_dealloc  = &erts_alcu_literal_64_mseg_dealloc;
+    ip->init.util.mseg_alloc    = &erts_alcu_mmapper_mseg_alloc;
+    ip->init.util.mseg_realloc  = &erts_alcu_mmapper_mseg_realloc;
+    ip->init.util.mseg_dealloc  = &erts_alcu_mmapper_mseg_dealloc;
+    ip->init.util.mseg_mmapper  = &erts_literal_mmapper;
 # endif
 #else
 # error Unknown architecture
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 6e682019ba..e362a03646 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -898,8 +898,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
 
 #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
 
+/* Used by literal allocator that has its own mmapper (super carrier) */
 void*
-erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
 {
     void* res;
     UWord size = (UWord) *size_p;
@@ -907,33 +908,33 @@ erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
     if (flags & ERTS_MSEG_FLG_2POW)
         mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
 
-    res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size);
+    res = erts_mmap(allctr->mseg_mmapper, mmap_flags, &size);
     *size_p = (Uint)size;
     INC_CC(allctr->calls.mseg_alloc);
     return res;
 }
 
 void*
-erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg,
+erts_alcu_mmapper_mseg_realloc(Allctr_t *allctr, void *seg,
                                Uint old_size, Uint *new_size_p)
 {
     void *res;
     UWord new_size = (UWord) *new_size_p;
-    res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
+    res = erts_mremap(allctr->mseg_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
     *new_size_p = (Uint) new_size;
     INC_CC(allctr->calls.mseg_realloc);
     return res;
 }
 
 void
-erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
+erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
                                Uint flags)
 {
     Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY;
     if (flags & ERTS_MSEG_FLG_2POW)
         mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
 
-    erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size);
+    erts_munmap(allctr->mseg_mmapper, mmap_flags, seg, (UWord)size);
     INC_CC(allctr->calls.mseg_dealloc);
 }
 #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
@@ -6127,6 +6128,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
         allctr->mseg_alloc   = init->mseg_alloc;
         allctr->mseg_realloc = init->mseg_realloc;
         allctr->mseg_dealloc = init->mseg_dealloc;
+        allctr->mseg_mmapper = init->mseg_mmapper;
     }
     else {
         ASSERT(!init->mseg_realloc && !init->mseg_dealloc);
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index afdff1a71e..51af59e40c 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -72,6 +72,7 @@ typedef struct {
     void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
     void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
     void  (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+    ErtsMemMapper *mseg_mmapper;
 #endif
     void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
     void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
@@ -205,9 +206,9 @@ void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uin
 void  erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
 
 # elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-void* erts_alcu_literal_64_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
-void* erts_alcu_literal_64_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
-void  erts_alcu_literal_64_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void  erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
 # endif
 #endif /* HAVE_ERTS_MSEG */
 
@@ -601,6 +602,7 @@ struct Allctr_t_ {
     void*               (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
     void*               (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
     void                (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+    ErtsMemMapper       *mseg_mmapper;
 #endif
     void*               (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
     void*               (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
-- 
cgit v1.2.3


From 85a6623152988c267cea008d20616b61ea9c223c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 12 Apr 2016 20:41:58 +0200
Subject: erts: Add 'exec_alloc' for hipe code

that uses its own super carrier (erts_exec_mmapper)
to guarantee low addressed and executable memory (PROT_EXEC).

Currently only used on x86_64 that needs low memory
for HiPE/AMD64's small code model.

By initializing erts_exec_mapper early we secure
its low memory area before erts_literal_mmapper might
steal it.
---
 erts/emulator/beam/erl_alloc.c     | 77 +++++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_alloc.types | 12 ++++++
 erts/emulator/beam/erl_bif_info.c  |  1 +
 3 files changed, 89 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index efe31193b9..00911ff027 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -30,6 +30,7 @@
 #endif
 #define ERTS_ALLOC_C__
 #define ERTS_ALC_INTERNAL__
+#define ERTS_WANT_MEM_MAPPERS
 #include "sys.h"
 #define ERL_THREADS_EMU_INTERNAL__
 #include "erl_threads.h"
@@ -131,6 +132,9 @@ static ErtsAllocatorState_t ets_alloc_state;
 static ErtsAllocatorState_t driver_alloc_state;
 static ErtsAllocatorState_t fix_alloc_state;
 static ErtsAllocatorState_t literal_alloc_state;
+#ifdef ERTS_ALC_A_EXEC
+static ErtsAllocatorState_t exec_alloc_state;
+#endif
 static ErtsAllocatorState_t test_alloc_state;
 
 typedef struct {
@@ -216,6 +220,7 @@ typedef struct {
     struct au_init driver_alloc;
     struct au_init fix_alloc;
     struct au_init literal_alloc;
+    struct au_init exec_alloc;
     struct au_init test_alloc;
 } erts_alc_hndl_args_init_t;
 
@@ -339,6 +344,38 @@ set_default_literal_alloc_opts(struct au_init *ip)
 #endif
 }
 
+#ifdef ERTS_ALC_A_EXEC
+static void
+set_default_exec_alloc_opts(struct au_init *ip)
+{
+    SET_DEFAULT_ALLOC_OPTS(ip);
+    ip->enable			= 1;
+    ip->thr_spec		= 0;
+    ip->disable_allowed         = 0;
+    ip->thr_spec_allowed        = 0;
+    ip->carrier_migration_allowed = 0;
+    ip->atype			= BESTFIT;
+    ip->init.bf.ao		= 1;
+    ip->init.util.ramv		= 0;
+    ip->init.util.mmsbc		= 0;
+    ip->init.util.sbct		= ~((UWord) 0);
+    ip->init.util.name_prefix	= "exec_";
+    ip->init.util.alloc_no	= ERTS_ALC_A_EXEC;
+    ip->init.util.mmbcs 	= 0; /* No main carrier */
+    ip->init.util.ts 		= ERTS_ALC_MTA_EXEC;
+    ip->init.util.asbcst	= 0;
+    ip->init.util.rsbcst	= 0;
+    ip->init.util.rsbcmt	= 0;
+    ip->init.util.rmbcmt	= 0;
+    ip->init.util.acul		= 0;
+
+    ip->init.util.mseg_alloc    = &erts_alcu_mmapper_mseg_alloc;
+    ip->init.util.mseg_realloc  = &erts_alcu_mmapper_mseg_realloc;
+    ip->init.util.mseg_dealloc  = &erts_alcu_mmapper_mseg_dealloc;
+    ip->init.util.mseg_mmapper  = &erts_exec_mmapper;
+}
+#endif /* ERTS_ALC_A_EXEC */
+
 static void
 set_default_temp_alloc_opts(struct au_init *ip)
 {
@@ -660,6 +697,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_default_fix_alloc_opts(&init.fix_alloc,
 			       fix_type_sizes);
     set_default_literal_alloc_opts(&init.literal_alloc);
+#ifdef ERTS_ALC_A_EXEC
+    set_default_exec_alloc_opts(&init.exec_alloc);
+#endif
     set_default_test_alloc_opts(&init.test_alloc);
 
     if (argc && argv)
@@ -689,6 +729,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     init.driver_alloc.thr_spec = 0;
     init.fix_alloc.thr_spec = 0;
     init.literal_alloc.thr_spec = 0;
+#ifdef ERTS_ALC_A_EXEC
+    init.exec_alloc.thr_spec = 0;
+#endif
 #endif
 
     /* Make adjustments for carrier migration support */
@@ -702,6 +745,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_carrier_migration_support(&init.driver_alloc);
     adjust_carrier_migration_support(&init.fix_alloc);
     adjust_carrier_migration_support(&init.literal_alloc);
+#ifdef ERTS_ALC_A_EXEC
+    adjust_carrier_migration_support(&init.exec_alloc);
+#endif
 
     if (init.erts_alloc_config) {
 	/* Adjust flags that erts_alloc_config won't like */
@@ -717,6 +763,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.driver_alloc.thr_spec = 0;
 	init.fix_alloc.thr_spec = 0;
         init.literal_alloc.thr_spec = 0;
+#ifdef ERTS_ALC_A_EXEC
+        init.exec_alloc.thr_spec = 0;
+#endif
 
 	/* No carrier migration */
 	init.temp_alloc.init.util.acul = 0;
@@ -729,6 +778,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.driver_alloc.init.util.acul = 0;
 	init.fix_alloc.init.util.acul = 0;
         init.literal_alloc.init.util.acul = 0;
+#ifdef ERTS_ALC_A_EXEC
+        init.exec_alloc.init.util.acul = 0;
+#endif
     }
 
 #ifdef ERTS_SMP
@@ -746,6 +798,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_tpref(&init.driver_alloc, erts_no_schedulers);
     adjust_tpref(&init.fix_alloc, erts_no_schedulers);
     adjust_tpref(&init.literal_alloc, erts_no_schedulers);
+#ifdef ERTS_ALC_A_EXEC
+    adjust_tpref(&init.exec_alloc, erts_no_schedulers);
+#endif
 
 #else
     /* No thread specific if not smp */
@@ -765,6 +820,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     refuse_af_strategy(&init.driver_alloc);
     refuse_af_strategy(&init.fix_alloc);
     refuse_af_strategy(&init.literal_alloc);
+#ifdef ERTS_ALC_A_EXEC
+    refuse_af_strategy(&init.exec_alloc);
+#endif
 
 #ifdef ERTS_SMP 
     if (!init.temp_alloc.thr_spec)
@@ -809,6 +867,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
     set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu);
+#ifdef ERTS_ALC_A_EXEC
+    set_au_allocator(ERTS_ALC_A_EXEC, &init.exec_alloc, ncpu);
+#endif
     set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu);
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
@@ -865,7 +926,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     start_au_allocator(ERTS_ALC_A_LITERAL,
                        &init.literal_alloc,
                        &literal_alloc_state);
-
+#ifdef ERTS_ALC_A_EXEC
+    start_au_allocator(ERTS_ALC_A_EXEC,
+                       &init.exec_alloc,
+                       &exec_alloc_state);
+#endif
     start_au_allocator(ERTS_ALC_A_TEST,
 		       &init.test_alloc,
 		       &test_alloc_state);
@@ -1500,6 +1565,16 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
                     else
                         handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
 		    break;
+                case 'X':
+                    if (has_prefix("scs", argv[i]+3)) {
+#ifdef ERTS_ALC_A_EXEC
+                        init->mseg.exec_mmap.scs =
+#endif
+                            get_mb_value(argv[i]+6, argv, &i);
+                    }
+                    else
+                        handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0);
+                    break;
 		case 'D':
 		    handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
 		    break;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 14067283bd..0649ed903a 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -87,6 +87,9 @@ allocator	EHEAP		true		eheap_alloc
 allocator	ETS		true		ets_alloc
 allocator	FIXED_SIZE	true		fix_alloc
 allocator	LITERAL 	true		literal_alloc
++if exec_alloc
+allocator	EXEC     	true	        exec_alloc
++endif
 
 +else # Non smp build
 
@@ -98,6 +101,9 @@ allocator	EHEAP		false		eheap_alloc
 allocator	ETS		false		ets_alloc
 allocator	FIXED_SIZE	false		fix_alloc
 allocator	LITERAL 	false		literal_alloc
++if exec_alloc
+allocator	EXEC     	false	        exec_alloc
++endif
 
 +endif
 
@@ -335,8 +341,14 @@ type	SL_MPATHS	SHORT_LIVED	SYSTEM		sl_migration_paths
 # Currently most hipe code use this type.
 type	HIPE		SYSTEM		SYSTEM		hipe_data
 
++if exec_alloc
+type	HIPE_EXEC	EXEC            CODE		hipe_code
++endif
+
 +endif
 
+
+
 +if heap_frag_elim_test
 
 type	SSB		SHORT_LIVED	PROCESSES	ssb
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 0b2c26a548..163070e85a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -22,6 +22,7 @@
 #  include "config.h"
 #endif
 
+#define ERTS_WANT_MEM_MAPPERS
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
-- 
cgit v1.2.3


From 1afc15d988ee9fc57eaf9728dc52d8df75b5b12d Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 18 Apr 2016 17:38:45 +0200
Subject: erts: Fix lock order bug when only child is procs traced

---
 erts/emulator/beam/erl_process.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a520005c35..1a66044627 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11183,10 +11183,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     }
 
     if (IS_TRACED_FL(p, F_TRACE_PROCS)) {
-        if (locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) {
+        if ((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE))
+              == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) {
+            /* This happens when parent was not traced, but child is */
             locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
-            erts_smp_proc_unlock(p, locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE));
-            erts_smp_proc_unlock(parent, locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE));
+            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
         }
         trace_proc_spawn(p, am_spawned, parent->common.id, mod, func, args);
         if (so->flags & SPO_LINK)
-- 
cgit v1.2.3


From 11e8e1e38bba355c2996d766b7d2893e441c1211 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 18 Feb 2016 16:21:30 +0100
Subject: runtime_tools: Initial lttng tracing framework

---
 erts/emulator/beam/erl_nif.c   |   2 +-
 erts/emulator/beam/erl_trace.c | 175 ++++++++++++++++++++++++++++-------------
 2 files changed, 123 insertions(+), 54 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 73c0eb8eba..9a0eea42f0 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3158,7 +3158,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
     /* Verify that function is part of this module */
     int i;
     for (i = 0; i < mod->entry->num_of_funcs; i++)
-        if (fun == mod->entry->funcs+i)
+        if (fun == &(mod->entry->funcs[i]))
             break;
     ASSERT(i < mod->entry->num_of_funcs);
     if (p)
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index bd88769dfc..96fc46c817 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -359,14 +359,30 @@ void erts_init_trace(void) {
    *(OHPP) = &(*(BPP))->off_heap, \
    (*(BPP))->mem)
 
+enum ErtsTracerOpt {
+    TRACE_FUN_DEFAULT = 0,
+    TRACE_FUN_ENABLED = 1,
+    TRACE_FUN_SEND    = 2,
+    TRACE_FUN_RECEIVE = 3,
+    TRACE_FUN_CALL    = 4,
+    TRACE_FUN_RUNNING = 5,
+    TRACE_FUN_GC      = 6,
+    TRACE_FUN_PROCS   = 7,
+    TRACE_FUN_PORTS   = 8
+};
+
+#define NIF_TRACER_TYPES (9)
+
+
 static ERTS_INLINE int
 send_to_tracer_nif_raw(Process *c_p, Process *tracee, const ErtsTracer tracer,
                        Uint trace_flags, Eterm t_p_id, ErtsTracerNif *tnif,
+                       enum ErtsTracerOpt topt,
                        Eterm tag, Eterm msg, Eterm extra, Eterm pam_result);
 static ERTS_INLINE int
 send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
-                   Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag,
-                   Eterm msg, Eterm extra);
+                   Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt,
+                   Eterm tag, Eterm msg, Eterm extra);
 static ERTS_INLINE Eterm
 call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
                     ErtsTracerNif **tnif_ref, Eterm tag, Eterm t_p_id);
@@ -375,9 +391,9 @@ is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
                        ErtsPTabElementCommon *t_p,
                        ErtsTracerNif **tnif_ret, Eterm tag);
 
-#define SEND_TO_TRACER(c_p, tag, msg)                                   \
-    send_to_tracer_nif(c_p, &(c_p)->common, (c_p)->common.id, NULL, tag, \
-                       msg, THE_NON_VALUE)
+#define SEND_TO_TRACER(c_p, tag, msg)                                 \
+    send_to_tracer_nif(c_p, &(c_p)->common, (c_p)->common.id, NULL,   \
+            TRACE_FUN_DEFAULT, tag, msg, THE_NON_VALUE)
 
 static Uint active_sched;
 
@@ -756,7 +772,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	hp += 4;
     }
 
-    send_to_tracer_nif(p, &p->common, p->common.id, tnif,
+    send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_RUNNING,
                        what, tmp, THE_NON_VALUE);
 }
 
@@ -796,7 +812,8 @@ trace_send(Process *p, Eterm to, Eterm msg)
     }
 
     if (is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, operation))
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, operation, msg, to);
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_SEND,
+                operation, msg, to);
 }
 
 /* Send {trace_ts, Pid, receive, Msg, Timestamp}
@@ -809,7 +826,8 @@ trace_receive(Process *c_p, Eterm msg)
     if (is_tracer_proc_enabled(NULL, 0, &c_p->common,
                                &tnif, am_receive))
         send_to_tracer_nif(NULL, &c_p->common, c_p->common.id,
-                           tnif, am_receive, msg, THE_NON_VALUE);
+                           tnif, TRACE_FUN_RECEIVE,
+                           am_receive, msg, THE_NON_VALUE);
 }
 
 int
@@ -897,14 +915,13 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
         msg = TUPLE3(hp, am_EXIT, exitfrom, msg);
         hp += 4;
     }
-    mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token),
-                  receiver, msg);
+    mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), receiver, msg);
     hp += 6;
 
     seq_tracer_flags |=  ERTS_SEQTFLGS2TFLGS(unsigned_val(SEQ_TRACE_T_FLAGS(token)));
 
     send_to_tracer_nif_raw(NULL, process, seq_tracer, seq_tracer_flags,
-                           label, NULL, am_seq_trace, mess,
+                           label, NULL, TRACE_FUN_DEFAULT, am_seq_trace, mess,
                            THE_NON_VALUE, am_true);
 
     UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
@@ -983,7 +1000,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer)
     mfa = TUPLE3(hp, mod, name, make_small(arity));
     hp += 4;
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           NULL, am_return_from, mfa, retval, am_true);
+                           NULL, TRACE_FUN_DEFAULT, am_return_from, mfa, retval, am_true);
 }
 
 /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, 
@@ -1038,7 +1055,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
     cv = TUPLE2(hp, class, value);
     hp += 3;
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           NULL, am_exception_from, mfa_tuple, cv, am_true);
+                           NULL, TRACE_FUN_DEFAULT, am_exception_from, mfa_tuple, cv, am_true);
 }
 
 /*
@@ -1227,7 +1244,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
      * Build the trace tuple and send it to the port.
      */
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           tnif, am_call, mfa_tuple, THE_NON_VALUE, pam_result);
+                           tnif, TRACE_FUN_CALL, am_call, mfa_tuple, THE_NON_VALUE, pam_result);
     erts_match_set_release_result(p);
 
     if (match_spec && tracer == &pre_ms_tracer)
@@ -1250,7 +1267,7 @@ trace_proc(Process *c_p, ErtsProcLocks c_p_locks,
 {
     ErtsTracerNif *tnif = NULL;
     if (is_tracer_proc_enabled(c_p, c_p_locks, &t_p->common, &tnif, what))
-        send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif,
+        send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_PROCS,
                            what, data, THE_NON_VALUE);
 }
 
@@ -1276,7 +1293,8 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
         hp = HAlloc(p, 4);
         mfa = TUPLE3(hp, mod, func, args);
         hp += 4;
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, what, pid, mfa);
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_PROCS,
+                am_spawn, pid, mfa);
     }
 }
 
@@ -1323,8 +1341,8 @@ trace_gc(Process *p, Eterm what)
 
         msg = erts_process_gc_info(p, NULL, &hp);
 
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, what,
-                           msg, THE_NON_VALUE);
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_GC,
+                what, msg, THE_NON_VALUE);
     }
 }
 
@@ -1738,8 +1756,8 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
     ErtsTracerNif *tnif = NULL;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     if (is_tracer_proc_enabled(NULL, 0, &p->common, &tnif, am_open))
-        send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, am_open,
-                           calling_pid, drv_name);
+        send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_PORTS,
+                am_open, calling_pid, drv_name);
 }
 
 /* Sends trace message:
@@ -1757,7 +1775,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what))
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_PORTS,
                            what, data, THE_NON_VALUE);
 }
 
@@ -1894,7 +1912,7 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
         }
 
         data = TUPLE2(hp, caller, data);
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_RECEIVE,
                            am_receive, data, THE_NON_VALUE);
 
         if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
@@ -1917,7 +1935,7 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists)
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, op))
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_SEND,
                            op, msg, receiver);
 }
 
@@ -1946,7 +1964,7 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
         msg = TUPLE2(hp, t_p->common.id, msg);
         hp += 3;
 
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_SEND,
                            am_send, msg, to);
         if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
             erts_bin_free(bptr);
@@ -1977,7 +1995,8 @@ trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) {
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
-                           tnif, what, where, THE_NON_VALUE);
+                           tnif, TRACE_FUN_RUNNING,
+                           what, where, THE_NON_VALUE);
 }
 
 /* Port profiling */
@@ -2523,14 +2542,60 @@ init_sys_msg_dispatcher(void)
 
 #include "erl_nif.h"
 
+typedef struct {
+    char *name;
+    Uint arity;
+    ErlNifFunc *cb;
+} ErtsTracerType;
+
 struct ErtsTracerNif_ {
     HashBucket hb;
     Eterm module;
     struct erl_module_nif* nif_mod;
-    ErlNifFunc *enabled;
-    ErlNifFunc *trace;
+    ErtsTracerType tracers[NIF_TRACER_TYPES];
 };
 
+static void init_tracer_template(ErtsTracerNif *tnif) {
+
+    /* default tracer functions */
+    tnif->tracers[TRACE_FUN_DEFAULT].name  = "trace";
+    tnif->tracers[TRACE_FUN_DEFAULT].arity = 6;
+    tnif->tracers[TRACE_FUN_DEFAULT].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_ENABLED].name  = "enabled";
+    tnif->tracers[TRACE_FUN_ENABLED].arity = 3;
+    tnif->tracers[TRACE_FUN_ENABLED].cb    = NULL;
+
+    /* specific tracer functions */
+    tnif->tracers[TRACE_FUN_SEND].name  = "trace_send";
+    tnif->tracers[TRACE_FUN_SEND].arity = 6;
+    tnif->tracers[TRACE_FUN_SEND].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_RECEIVE].name  = "trace_receive";
+    tnif->tracers[TRACE_FUN_RECEIVE].arity = 6;
+    tnif->tracers[TRACE_FUN_RECEIVE].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_CALL].name  = "trace_call";
+    tnif->tracers[TRACE_FUN_CALL].arity = 6;
+    tnif->tracers[TRACE_FUN_CALL].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_RUNNING].name  = "trace_running";
+    tnif->tracers[TRACE_FUN_RUNNING].arity = 6;
+    tnif->tracers[TRACE_FUN_RUNNING].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_GC].name  = "trace_garbage_collection";
+    tnif->tracers[TRACE_FUN_GC].arity = 6;
+    tnif->tracers[TRACE_FUN_GC].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_PROCS].name  = "trace_procs";
+    tnif->tracers[TRACE_FUN_PROCS].arity = 6;
+    tnif->tracers[TRACE_FUN_PROCS].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_PORTS].name  = "trace_ports";
+    tnif->tracers[TRACE_FUN_PORTS].arity = 6;
+    tnif->tracers[TRACE_FUN_PORTS].cb    = NULL;
+}
+
 static Hash *tracer_hash = NULL;
 static erts_smp_rwmtx_t tracer_mtx;
 
@@ -2543,31 +2608,32 @@ load_tracer_nif(const ErtsTracer tracer)
     ErlNifFunc *funcs;
     int num_of_funcs;
     ErtsTracerNif tnif_tmpl, *tnif;
-    int i;
+    ErtsTracerType *tracers;
+    int i,j;
 
-    if (mod && mod->curr.nif != NULL) {
-        instance = &mod->curr;
-    } else {
+    if (!mod || !mod->curr.nif) {
         return NULL;
     }
 
-    tnif_tmpl.enabled = NULL;
-    tnif_tmpl.trace = NULL;
+    instance = &mod->curr;
+
+    init_tracer_template(&tnif_tmpl);
     tnif_tmpl.nif_mod = instance->nif;
     tnif_tmpl.module = ERTS_TRACER_MODULE(tracer);
+    tracers = tnif_tmpl.tracers;
 
     num_of_funcs = erts_nif_get_funcs(instance->nif, &funcs);
 
     for(i = 0; i < num_of_funcs; i++) {
-        if (strcmp("enabled",funcs[i].name) == 0 && funcs[i].arity == 3) {
-            tnif_tmpl.enabled = funcs + i;
-        } else if (strcmp("trace",funcs[i].name) == 0 && funcs[i].arity == 6) {
-            tnif_tmpl.trace = funcs + i;
+        for (j = 0; j < NIF_TRACER_TYPES; j++) {
+            if (strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) {
+                tracers[j].cb = &(funcs[i]);
+                break;
+            }
         }
     }
 
-    if (tnif_tmpl.enabled == NULL ||
-        tnif_tmpl.trace   == NULL ) {
+    if (tracers[TRACE_FUN_DEFAULT].cb == NULL || tracers[TRACE_FUN_ENABLED].cb == NULL ) {
         return NULL;
     }
 
@@ -2660,17 +2726,16 @@ erts_tracer_to_term(Process *p, ErtsTracer tracer)
 static ERTS_INLINE int
 send_to_tracer_nif_raw(Process *c_p, Process *tracee,
                        const ErtsTracer tracer, Uint tracee_flags,
-                       Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag, Eterm msg,
-                       Eterm extra, Eterm pam_result)
+                       Eterm t_p_id, ErtsTracerNif *tnif,
+                       enum ErtsTracerOpt topt_arg,
+                       Eterm tag, Eterm msg, Eterm extra, Eterm pam_result)
 {
     if (tnif || (tnif = lookup_tracer_nif(tracer)) != NULL) {
 #define MAP_SIZE 3
-        Eterm argv[6],
-            local_heap[3+MAP_SIZE /* values */+(MAP_SIZE+1 /* keys */)];
+        Eterm argv[6], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)];
         flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1));
         Eterm *map_values = flatmap_get_values(map);
-
-        int argc = 6;
+        enum ErtsTracerOpt topt = (tnif->tracers[topt_arg].cb) ? topt_arg : TRACE_FUN_DEFAULT;
 
         argv[0] = tag;
         argv[1] = ERTS_TRACER_STATE(tracer);
@@ -2681,8 +2746,7 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee,
 
         map->thing_word = MAP_HEADER_FLATMAP;
         map->size = MAP_SIZE;
-        map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id,
-                           am_timestamp);
+        map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id, am_timestamp);
 
         *map_values++ = pam_result;
         if (tracee_flags & F_TRACE_SCHED_NO)
@@ -2705,16 +2769,18 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee,
 
 #undef MAP_SIZE
         erts_nif_call_function(c_p, tracee ? tracee : c_p,
-                               tnif->nif_mod, tnif->trace, argc, argv);
+                               tnif->nif_mod,
+                               tnif->tracers[topt].cb,
+                               tnif->tracers[topt].arity,
+                               argv);
     }
     return 1;
 }
 
-
 static ERTS_INLINE int
 send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
-                   Eterm t_p_id, ErtsTracerNif *tnif, Eterm tag,
-                   Eterm msg, Eterm extra)
+                   Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt topt,
+                   Eterm tag, Eterm msg, Eterm extra)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     if (c_p) {
@@ -2733,7 +2799,7 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
     return send_to_tracer_nif_raw(c_p,
                                   is_internal_pid(t_p->id) ? (Process*)t_p : NULL,
                                   t_p->tracer, t_p->trace_flags,
-                                  t_p_id, tnif, tag, msg, extra,
+                                  t_p_id, tnif, topt, tag, msg, extra,
                                   am_true);
 }
 
@@ -2746,7 +2812,10 @@ call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
         Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id};
         if (tnif_ret) *tnif_ret = tnif;
         return erts_nif_call_function(
-            c_p, NULL, tnif->nif_mod, tnif->enabled, 3, argv);
+            c_p, NULL, tnif->nif_mod,
+            tnif->tracers[TRACE_FUN_ENABLED].cb,
+            tnif->tracers[TRACE_FUN_ENABLED].arity,
+            argv);
     }
     return am_remove;
 }
-- 
cgit v1.2.3


From b3c7a581070180750a368ddf3bd55db1bafc11ea Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 15 Apr 2016 18:39:59 +0200
Subject: erts: Make sure literal MBCs have super aligned sizes

on 32-bit, as the granularity of the literal bit vector
is super-alignment.
---
 erts/emulator/beam/erl_alloc_util.c | 53 ++++++++++++++++++++++++-------------
 erts/emulator/beam/erl_alloc_util.h | 16 +++++------
 2 files changed, 42 insertions(+), 27 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index e362a03646..680899c919 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -859,28 +859,34 @@ void*
 erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
 {
     void* res;
+    Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p);
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
                    && !allctr->t && allctr->thread_safe);
 
-    res = erts_alcu_mseg_alloc(allctr, size_p, flags);
-    if (res)
-        set_literal_range(res, *size_p);
+    res = erts_alcu_mseg_alloc(allctr, &sz, flags);
+    if (res) {
+        set_literal_range(res, sz);
+        *size_p = sz;
+    }
     return res;
 }
 
 void*
 erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg,
-                               Uint old_size, Uint *new_size_p)
+                                  Uint old_size, Uint *new_size_p)
 {
     void* res;
+    Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p);
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
                    && !allctr->t && allctr->thread_safe);
 
     if (seg && old_size)
         clear_literal_range(seg, old_size);
-    res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p);
-    if (res)
-        set_literal_range(res, *new_size_p);
+    res = erts_alcu_mseg_realloc(allctr, seg, old_size, &new_sz);
+    if (res) {
+        set_literal_range(res, new_sz);
+        *new_size_p = new_sz;
+    }
     return res;
 }
 
@@ -942,9 +948,10 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
 #endif /* HAVE_ERTS_MSEG */
 
 void*
-erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
 {
     void *res;
+    const Uint size = *size_p;
 #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
     if (superalign)
 	res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size);
@@ -958,9 +965,10 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
 }
 
 void*
-erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign)
 {
     void *res;
+    const Uint size = *size_p;
 
 #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
     if (superalign)
@@ -995,30 +1003,37 @@ erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
 #ifdef ARCH_32
 
 void*
-erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
 {
     void* res;
+    Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
                    && !allctr->t && allctr->thread_safe);
 
-    res = erts_alcu_sys_alloc(allctr, size, 1);
-    if (res)
+    res = erts_alcu_sys_alloc(allctr, &size, 1);
+    if (res) {
         set_literal_range(res, size);
+        *size_p = size;
+    }
     return res;
 }
 
 void*
-erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint old_size, int superalign)
 {
     void* res;
+    Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
+
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
                    && !allctr->t && allctr->thread_safe);
 
     if (ptr && old_size)
         clear_literal_range(ptr, old_size);
-    res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1);
-    if (res)
+    res = erts_alcu_sys_realloc(allctr, ptr, &size, old_size, 1);
+    if (res) {
         set_literal_range(res, size);
+        *size_p = size;
+    }
     return res;
 }
 
@@ -3865,12 +3880,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	      ? UNIT_CEILING(bcrr_sz)
 	      : SYS_ALLOC_CARRIER_CEILING(bcrr_sz));
 
-    crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+    crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC);
 	
     if (!crr) {
 	if (crr_sz > UNIT_CEILING(bcrr_sz)) {
 	    crr_sz = UNIT_CEILING(bcrr_sz);
-	    crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+	    crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC);
 	}
 	if (!crr) {
 #if HAVE_ERTS_MSEG
@@ -4028,7 +4043,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 
 	    new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
 						     (void *) old_crr,
-						     new_crr_sz,
+						     &new_crr_sz,
 						     old_crr_sz,
 						     0);
 	    if (new_crr) {
@@ -4049,7 +4064,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		new_crr_sz = UNIT_CEILING(new_crr_sz);
 		new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
 							 (void *) old_crr,
-							 new_crr_sz,
+							 &new_crr_sz,
 							 old_crr_sz,
 							 0);
 		if (new_crr)
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 51af59e40c..b3ae4c3281 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -74,8 +74,8 @@ typedef struct {
     void  (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
     ErtsMemMapper *mseg_mmapper;
 #endif
-    void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
-    void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
+    void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign);
+    void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign);
     void  (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
 } AllctrInit_t;
 
@@ -212,12 +212,12 @@ void  erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags
 # endif
 #endif /* HAVE_ERTS_MSEG */
 
-void* erts_alcu_sys_alloc(Allctr_t*, Uint size, int superalign);
-void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign);
+void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
+void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
 void  erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
 #ifdef ARCH_32
-void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint size, int superalign);
-void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign);
+void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
+void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
 void  erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
 #endif
 
@@ -604,8 +604,8 @@ struct Allctr_t_ {
     void                (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
     ErtsMemMapper       *mseg_mmapper;
 #endif
-    void*               (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign);
-    void*               (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign);
+    void*               (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign);
+    void*               (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign);
     void                (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
 
     void		(*init_atoms)		(void);
-- 
cgit v1.2.3


From 4319cd608063e3612c74b534180b19ab29173667 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 15 Apr 2016 14:16:14 +0200
Subject: erts: Produce statistics for literal and hipe super carriers

called 'literal_mmap' and 'exec_mmap'.

Also moved existing erts_mmap info from 'mseg_alloc'
to its own system_info({allocator, erts_mmap})

with "allocators" default_mmap, literal_mmap and exec_mmap.
---
 erts/emulator/beam/erl_alloc.c | 79 +++++++++++++++++++++++++++++++++---------
 1 file changed, 62 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d04977b9ae..5b76f34165 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -137,6 +137,11 @@ static ErtsAllocatorState_t exec_alloc_state;
 #endif
 static ErtsAllocatorState_t test_alloc_state;
 
+#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1)
+#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2)
+#define ERTS_ALC_INFO_A_ERTS_MMAP  (ERTS_ALC_A_MAX + 3)
+#define ERTS_ALC_INFO_A_MAX        ERTS_ALC_INFO_A_ERTS_MMAP
+
 typedef struct {
     erts_smp_atomic32_t refc;
     int only_sz;
@@ -145,13 +150,9 @@ typedef struct {
     Process *proc;
     Eterm ref;
     Eterm ref_heap[REF_THING_SIZE];
-    int allocs[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+1+2];
+    int allocs[ERTS_ALC_INFO_A_MAX - ERTS_ALC_A_MIN + 1 + 1];
 } ErtsAllocInfoReq;
 
-#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1)
-#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2)
-#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC
-
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
                                  ErtsAllocInfoReq,
                                  5,
@@ -2842,8 +2843,16 @@ erts_allocator_info(int to, void *arg)
 	    erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
 	    erts_mseg_info(i, &to, arg, 0, NULL, NULL);
 	}
-	erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n");
+	erts_print(to, arg, "=allocator:erts_mmap.default_mmap\n");
 	erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis);
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+        erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n");
+        erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis);
+#endif
+#ifdef ERTS_ALC_A_EXEC
+        erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n");
+        erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis);
+#endif
     }
 #endif
 
@@ -3075,7 +3084,13 @@ reply_alloc_info(void *vair)
     Uint sz, *szp;
     ErlOffHeap *ohp = NULL;
     ErtsMessage *mp = NULL;
-    struct erts_mmap_info_struct emis;
+    struct erts_mmap_info_struct mmap_info_dflt;
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+    struct erts_mmap_info_struct mmap_info_literal;
+#endif
+#ifdef ERTS_ALC_A_EXEC
+    struct erts_mmap_info_struct mmap_info_exec;
+#endif
     int i;
     Eterm (*info_func)(Allctr_t *,
 		       int,
@@ -3181,6 +3196,39 @@ reply_alloc_info(void *vair)
 					   make_small(0),
 					   ainfo);
 		    break;
+                case ERTS_ALC_INFO_A_ERTS_MMAP:
+                    alloc_atom = erts_bld_atom(hpp, szp, "erts_mmap");
+
+                    ainfo = (air->only_sz ? NIL :
+                             erts_mmap_info(&erts_dflt_mmapper, NULL, NULL,
+                                            hpp, szp, &mmap_info_dflt));
+                    ainfo = erts_bld_tuple3(hpp, szp,
+                                            alloc_atom,
+                                            erts_bld_atom(hpp,szp,"default_mmap"),
+                                            ainfo);
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+                    ai_list = erts_bld_cons(hpp, szp,
+                                            ainfo, ai_list);
+                    ainfo = (air->only_sz ? NIL :
+                             erts_mmap_info(&erts_literal_mmapper, NULL, NULL,
+                                            hpp, szp, &mmap_info_literal));
+                    ainfo = erts_bld_tuple3(hpp, szp,
+                                            alloc_atom,
+                                            erts_bld_atom(hpp,szp,"literal_mmap"),
+                                            ainfo);
+#endif
+#ifdef ERTS_ALC_A_EXEC
+                    ai_list = erts_bld_cons(hpp, szp,
+                                            ainfo, ai_list);
+                    ainfo = (air->only_sz ? NIL :
+                             erts_mmap_info(&erts_exec_mmapper, NULL, NULL,
+                                            hpp, szp, &mmap_info_exec));
+                    ainfo = erts_bld_tuple3(hpp, szp,
+                                            alloc_atom,
+                                            erts_bld_atom(hpp,szp,"exec_mmap"),
+                                            ainfo);
+#endif
+                    break;
 		case ERTS_ALC_INFO_A_MSEG_ALLOC:
 		    alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
 #if HAVE_ERTS_MSEG
@@ -3193,14 +3241,6 @@ reply_alloc_info(void *vair)
                                             make_small(0),
                                             ainfo);
 
-		    ai_list = erts_bld_cons(hpp, szp,
-					    ainfo, ai_list);
-		    ainfo = (air->only_sz ? NIL :
-                             erts_mmap_info(&erts_dflt_mmapper, NULL, NULL, hpp, szp, &emis));
-		    ainfo = erts_bld_tuple3(hpp, szp,
-                                            alloc_atom,
-                                            erts_bld_atom(hpp,szp,"erts_mmap"),
-                                            ainfo);
 #else
 		    ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
                                             am_false);
@@ -3232,7 +3272,8 @@ reply_alloc_info(void *vair)
 	    }
 	    switch (ai) {
 	    case ERTS_ALC_A_SYSTEM:
-	    case ERTS_ALC_INFO_A_ALLOC_UTIL:
+            case ERTS_ALC_INFO_A_ALLOC_UTIL:
+            case ERTS_ALC_INFO_A_ERTS_MMAP:
 		break;
 	    case ERTS_ALC_INFO_A_MSEG_ALLOC:
 #if HAVE_ERTS_MSEG && defined(ERTS_SMP)
@@ -3306,7 +3347,7 @@ erts_request_alloc_info(struct process *c_p,
 			int internal)
 {
     ErtsAllocInfoReq *air = aireq_alloc();
-    Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0};
+    Eterm req_ai[ERTS_ALC_INFO_A_MAX+1] = {0};
     Eterm alist;
     Eterm *hp;
     int airix = 0, ai;
@@ -3342,6 +3383,10 @@ erts_request_alloc_info(struct process *c_p,
 	    ai = ERTS_ALC_INFO_A_MSEG_ALLOC;
 	    goto save_alloc;
 	}
+        if (erts_is_atom_str("erts_mmap", alloc, 0)) {
+            ai = ERTS_ALC_INFO_A_ERTS_MMAP;
+            goto save_alloc;
+        }
 	if (erts_is_atom_str("alloc_util", alloc, 0)) {
 	    ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
 	save_alloc:
-- 
cgit v1.2.3


From 5b90345fe0e31c7d97ae41de1c53871c5815f126 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 21 Apr 2016 15:18:16 +0200
Subject: erts: Fix lock-order trace status

---
 erts/emulator/beam/erl_process.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1a66044627..d485affa3b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10932,9 +10932,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     INITIALIZE_SHCOPY(info);
 #endif
 
-#ifdef ERTS_SMP
     erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
 
     /*
      * Check for errors.
@@ -11235,6 +11233,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
      * Schedule process for execution.
      */
 
+    erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+
     schedule_process(p, state, 0);
 
     VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id));
@@ -11248,6 +11248,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
         DTRACE2(process_spawn, process_name, mfa);
     }
 #endif
+    return res;
 
  error:
 
-- 
cgit v1.2.3


From 10cff76885554ab11ebfef2ef83cee9ec7fe410a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 21 Apr 2016 19:55:34 +0200
Subject: erts: Fix copy share with onheap messages

---
 erts/emulator/beam/erl_gc.c | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index f33ade27f3..4698458521 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2053,8 +2053,26 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
 	    *hp++ = val;
 	    break;
 	case TAG_PRIMARY_LIST:
+#ifdef SHCOPY_SEND
+            if (erts_is_literal(val,list_val(val))) {
+                *hp++ = val;
+            } else {
+                *hp++ = offset_ptr(val, offs);
+            }
+#else
+            *hp++ = offset_ptr(val, offs);
+#endif
+            break;
 	case TAG_PRIMARY_BOXED:
-	    *hp++ = offset_ptr(val, offs);
+#ifdef SHCOPY_SEND
+            if (erts_is_literal(val,boxed_val(val))) {
+                *hp++ = val;
+            } else {
+                *hp++ = offset_ptr(val, offs);
+            }
+#else
+            *hp++ = offset_ptr(val, offs);
+#endif
 	    break;
 	case TAG_PRIMARY_HEADER:
 	    *hp++ = val;
-- 
cgit v1.2.3


From ee1cd28701589cd20f240941957f9a2fb7aadd3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 22 Apr 2016 11:52:41 +0200
Subject: erts: Fix comments

---
 erts/emulator/beam/beam_bif_load.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index b10250dc49..87508dcf5f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -832,7 +832,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
 
     /*
      * Message queue can contains funs, but (at least currently) no
-     * constants. If we got references to this module from the message
+     * literals. If we got references to this module from the message
      * queue, a GC cannot remove these...
      */
 
@@ -853,7 +853,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
 	for (; hfrag; hfrag = hfrag->next) {
 	    if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
 		return am_true;
-	    /* Should not contain any constants... */
+	    /* Should not contain any literals... */
 	    ASSERT(!any_heap_refs(&hfrag->mem[0],
 				  &hfrag->mem[hfrag->used_size],
                                   literals,
@@ -908,7 +908,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
 #ifdef DEBUG
 	/*
 	 * Message buffer fragments should not have any references
-	 * to constants, and off heap lists should already have
+	 * to literals, and off heap lists should already have
 	 * been moved into process off heap structure.
 	 */
 	for (msgp = rp->msg_frag; msgp; msgp = msgp->next) {
@@ -945,7 +945,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
 	need_gc &= ~done_gc;
 
 	/*
-	 * Try to get rid of constants by by garbage collecting.
+	 * Try to get rid of literals by by garbage collecting.
 	 * Clear both fvalue and ftrace.
 	 */
 
-- 
cgit v1.2.3


From 65bd8ade865eebe0d8a3c3210a4e2e9f334e229f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Sun, 10 Apr 2016 22:48:55 +0200
Subject: erts: Add BIF maps:take/2

---
 erts/emulator/beam/bif.tab   |  2 ++
 erts/emulator/beam/erl_map.c | 75 +++++++++++++++++++++++++++++++++-----------
 erts/emulator/beam/erl_map.h |  1 +
 erts/emulator/beam/erl_nif.c |  5 ++-
 4 files changed, 62 insertions(+), 21 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 58cd31cee9..872f0f9b2a 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -652,6 +652,8 @@ bif erts_debug:size_shared/1
 bif erts_debug:copy_shared/1
 bif erlang:has_prepared_code_on_load/1
 
+bif maps:take/2
+
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 03a96cb00a..8efc983f04 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -54,6 +54,7 @@
  * - maps:new/0
  * - maps:put/3
  * - maps:remove/2
+ * - maps:take/2
  * - maps:to_list/1
  * - maps:update/3
  * - maps:values/1
@@ -93,7 +94,7 @@ static Uint hashmap_subtree_size(Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node);
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value);
 static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size);
 static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys);
@@ -1521,10 +1522,45 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
-/* maps:remove/3 */
+/* maps:take/2 */
 
-int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
+BIF_RETTYPE maps_take_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+        Eterm res, map, val;
+        if (erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &map, &val)) {
+            Eterm *hp = HAlloc(BIF_P, 3);
+            res   = make_tuple(hp);
+            *hp++ = make_arityval(2);
+            *hp++ = val;
+            *hp++ = map;
+            BIF_RET(res);
+        }
+        BIF_RET(am_error);
+    }
+    BIF_P->fvalue = BIF_ARG_2;
+    BIF_ERROR(BIF_P, BADMAP);
+}
+
+/* maps:remove/2 */
+
+BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
+    if (is_map(BIF_ARG_2)) {
+        Eterm res;
+        (void) erts_maps_take(BIF_P, BIF_ARG_1, BIF_ARG_2, &res, NULL);
+        BIF_RET(res);
+    }
+    BIF_P->fvalue = BIF_ARG_2;
+    BIF_ERROR(BIF_P, BADMAP);
+}
+
+/* erts_maps_take
+ * return 1 if key is found, otherwise 0
+ * If the key is not found res (output map) will be map (input map)
+ */
+int erts_maps_take(Process *p, Eterm key, Eterm map,
+                   Eterm *res, Eterm *value) {
     Uint32 hx;
+    Eterm ret;
     if (is_flatmap(map)) {
 	Sint n;
 	Uint need;
@@ -1537,7 +1573,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 
 	if (n == 0) {
 	    *res = map;
-	    return 1;
+	    return 0;
 	}
 
 	ks = flatmap_get_keys(mp);
@@ -1564,6 +1600,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 	if (is_immed(key)) {
 	    while (1) {
 		if (*ks == key) {
+                    if (value) *value = *vs;
 		    goto found_key;
 		} else if (--n) {
 		    *mhp++ = *vs++;
@@ -1574,6 +1611,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 	} else {
 	    while(1) {
 		if (EQ(*ks, key)) {
+                    if (value) *value = *vs;
 		    goto found_key;
 		} else if (--n) {
 		    *mhp++ = *vs++;
@@ -1589,7 +1627,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) {
 	HRelease(p, hp_start + need, hp_start);
 
 	*res = map;
-	return 1;
+	return 0;
 
 found_key:
 	/* Copy rest of keys and values */
@@ -1601,19 +1639,13 @@ found_key:
     }
     ASSERT(is_hashmap(map));
     hx = hashmap_make_hash(key);
-    *res = hashmap_delete(p, hx, key, map);
-    return 1;
-}
-
-BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
-    if (is_map(BIF_ARG_2)) {
-	Eterm res;
-	if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) {
-	    BIF_RET(res);
-	}
+    ret = hashmap_delete(p, hx, key, map, value);
+    if (is_value(ret)) {
+        *res = ret;
+        return 1;
     }
-    BIF_P->fvalue = BIF_ARG_2;
-    BIF_ERROR(BIF_P, BADMAP);
+    *res = map;
+    return 0;
 }
 
 int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
@@ -2322,7 +2354,8 @@ static Eterm hashmap_values(Process* p, Eterm node) {
     return res;
 }
 
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
+static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key,
+                            Eterm map, Eterm *value) {
     Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
     Eterm *ptr;
     Eterm hdr, res = map, node = map;
@@ -2337,8 +2370,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
 	switch(primary_tag(node)) {
 	    case TAG_PRIMARY_LIST:
 		if (EQ(CAR(list_val(node)), key)) {
+                    if (value) {
+                        *value = CDR(list_val(node));
+                    }
 		    goto unroll;
 		}
+                res = THE_NON_VALUE;
 		goto not_found;
 	    case TAG_PRIMARY_BOXED:
 		ptr = boxed_val(node);
@@ -2365,6 +2402,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
                             n    = hashmap_bitcount(hval);
                         } else {
                             /* not occupied */
+                            res = THE_NON_VALUE;
                             goto not_found;
                         }
 
@@ -2394,6 +2432,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) {
 			    break;
 			}
 			/* not occupied */
+                        res = THE_NON_VALUE;
 			goto not_found;
 		    default:
 			erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK);
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 7af9100906..8b5c9582ba 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -82,6 +82,7 @@ struct ErtsEStack_;
 Eterm  erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
 int    erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
 int    erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
+int    erts_maps_take(Process *p, Eterm key, Eterm map, Eterm *res, Eterm *value);
 
 Eterm  erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
 			   Eterm node, int is_update);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 73c0eb8eba..24e72cc949 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2443,14 +2443,13 @@ int enif_make_map_remove(ErlNifEnv* env,
 			 Eterm key,
 			 Eterm *map_out)
 {
-    int res;
     if (!is_map(map_in)) {
 	return 0;
     }
     flush_env(env);
-    res = erts_maps_remove(env->proc, key, map_in, map_out);
+    (void) erts_maps_take(env->proc, key, map_in, map_out, NULL);
     cache_env(env);
-    return res;
+    return 1;
 }
 
 int enif_map_iterator_create(ErlNifEnv *env,
-- 
cgit v1.2.3


From 43416d57174d93660aec1620eeef0b05271b43e6 Mon Sep 17 00:00:00 2001
From: Simon Cornish <7t9jna402@sneakemail.com>
Date: Mon, 25 Apr 2016 22:25:56 -0700
Subject: Print heap pointers for garbing processes during crashdump

In an SMP emulator, print_garb_info doesn't do anything because the
heap pointers could be invalid.

However, when crashdumping there are no schedulers running so it's ok
to print this additional information which could be useful when
examining a core file.
---
 erts/emulator/beam/break.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 0ddf7f4e6d..e0164c3c73 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -347,8 +347,11 @@ print_process_info(int to, void *to_arg, Process *p)
 static void
 print_garb_info(int to, void *to_arg, Process* p)
 {
+#ifdef ERTS_SMP
     /* ERTS_SMP: A scheduler is probably concurrently doing gc... */
-#ifndef ERTS_SMP
+    if (!ERTS_IS_CRASH_DUMPING)
+      return;
+#endif
     erts_print(to, to_arg, "New heap start: %bpX\n", p->heap);
     erts_print(to, to_arg, "New heap top: %bpX\n", p->htop);
     erts_print(to, to_arg, "Stack top: %bpX\n", p->stop);
@@ -356,7 +359,6 @@ print_garb_info(int to, void *to_arg, Process* p)
     erts_print(to, to_arg, "Old heap start: %bpX\n", OLD_HEAP(p));
     erts_print(to, to_arg, "Old heap top: %bpX\n", OLD_HTOP(p));
     erts_print(to, to_arg, "Old heap end: %bpX\n", OLD_HEND(p));
-#endif
 }
 
 void
-- 
cgit v1.2.3


From 6aae129123c4ce8988cc67b3545db9d1ec51324b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 22 Apr 2016 18:45:30 +0200
Subject: erts: Rename erl flag +xmqd to +hmqd

Flags that control the heap should all fall under the +h flag
---
 erts/emulator/beam/erl_init.c | 48 ++++++++++++++++++-------------------------
 1 file changed, 20 insertions(+), 28 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index dec9bdfedc..2fd1eeb785 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -578,6 +578,8 @@ void erts_usage(void)
 	       VH_DEFAULT_SIZE);
     erts_fprintf(stderr, "-hpds size     initial process dictionary size (default %d)\n",
 	       erts_pd_initial_size);
+    erts_fprintf(stderr, "-hmqd  val     set default message queue data flag for processes,\n");
+    erts_fprintf(stderr, "               valid values are: off_heap | on_heap | mixed\n");
 
     /*    erts_fprintf(stderr, "-i module  set the boot module (default init)\n"); */
 
@@ -655,8 +657,6 @@ void erts_usage(void)
 
     erts_fprintf(stderr, "-W<i|w|e>      set error logger warnings mapping,\n");
     erts_fprintf(stderr, "               see error_logger documentation for details\n");
-    erts_fprintf(stderr, "-xmqd  val     set default message queue data flag for processes,\n");
-    erts_fprintf(stderr, "               valid values are: off_heap | on_heap | mixed\n");
     erts_fprintf(stderr, "-zdbbl size    set the distribution buffer busy limit in kilobytes\n");
     erts_fprintf(stderr, "               valid range is [1-%d]\n", INT_MAX/1024);
     erts_fprintf(stderr, "-zdntgc time   set delayed node table gc in seconds\n");
@@ -1487,6 +1487,7 @@ erl_start(int argc, char **argv)
 	     * h|ms  - min_heap_size
 	     * h|mbs - min_bin_vheap_size
 	     * h|pds - erts_pd_initial_size
+	     * h|mqd - message_queue_data
 	     *
 	     */
 	    if (has_prefix("mbs", sub_param)) {
@@ -1512,6 +1513,23 @@ erl_start(int argc, char **argv)
 		}
 		VERBOSE(DEBUG_SYSTEM, ("using initial process dictionary size %d\n",
 			    erts_pd_initial_size));
+            } else if (has_prefix("mqd", sub_param)) {
+		arg = get_arg(sub_param+3, argv[i+1], &i);
+		if (sys_strcmp(arg, "mixed") == 0)
+		    erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ);
+		else if (sys_strcmp(arg, "on_heap") == 0) {
+		    erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ;
+		    erts_default_spo_flags |= SPO_ON_HEAP_MSGQ;
+		}
+		else if (sys_strcmp(arg, "off_heap") == 0) {
+		    erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ;
+		    erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ;
+		}
+		else {
+		    erts_fprintf(stderr,
+				 "Invalid message_queue_data flag: %s\n", arg);
+		    erts_usage();
+		}
 	    } else {
 	        /* backward compatibility */
 		arg = get_arg(argv[i]+2, argv[i+1], &i);
@@ -2047,32 +2065,6 @@ erl_start(int argc, char **argv)
 	    }
 	    break;
 
-	case 'x': {
-	    char *sub_param = argv[i]+2;
-	    if (has_prefix("mqd", sub_param)) {
-		arg = get_arg(sub_param+3, argv[i+1], &i);
-		if (sys_strcmp(arg, "mixed") == 0)
-		    erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ);
-		else if (sys_strcmp(arg, "on_heap") == 0) {
-		    erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ;
-		    erts_default_spo_flags |= SPO_ON_HEAP_MSGQ;
-		}
-		else if (sys_strcmp(arg, "off_heap") == 0) {
-		    erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ;
-		    erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ;
-		}
-		else {
-		    erts_fprintf(stderr,
-				 "Invalid message_queue_data flag: %s\n", arg);
-		    erts_usage();
-		}
-	    } else {
-		erts_fprintf(stderr, "bad -x option %s\n", argv[i]);
-		erts_usage();
-	    }
-	    break;
-        }
-
 	case 'z': {
 	    char *sub_param = argv[i]+2;
 
-- 
cgit v1.2.3


From d0bca6c9aceb6b5be35342387d6bbbf0f3c5244c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 26 Apr 2016 15:19:56 +0200
Subject: erts: Fix total_heap_size calculation for on_heap

---
 erts/emulator/beam/erl_bif_info.c |  7 ++++---
 erts/emulator/beam/erl_gc.c       | 20 ++++++++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index c9741b361f..99fe847ba2 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1359,9 +1359,10 @@ process_info_aux(Process *BIF_P,
 
 	total_heap_size += rp->mbuf_sz;
 
-	for (mp = rp->msg.first; mp; mp = mp->next)
-	    if (mp->data.attached)
-		total_heap_size += erts_msg_attached_data_size(mp);
+        if (rp->flags & F_ON_HEAP_MSGQ)
+            for (mp = rp->msg.first; mp; mp = mp->next)
+                if (mp->data.attached)
+                    total_heap_size += erts_msg_attached_data_size(mp);
 
 	(void) erts_bld_uint(NULL, &hsz, total_heap_size);
 	hp = HAlloc(BIF_P, hsz);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 4698458521..881876ba59 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -3006,10 +3006,30 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp)
     };
 
     Eterm res = THE_NON_VALUE;
+    ErtsMessage *mp;
 
     ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags));
     ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS);
 
+    if (p->abandoned_heap) {
+        Eterm *htop, *heap;
+        ERTS_GET_ORIG_HEAP(p, heap, htop);
+        values[3] = HIGH_WATER(p) - heap;
+        values[6] = htop - heap;
+    }
+
+    if (p->flags & F_ON_HEAP_MSGQ) {
+        /* If on heap messages in the internal queue are counted
+           as being part of the heap, so we have to add them to the
+           am_mbuf_size value. process_info(total_heap_size) should
+           be the same as adding old_heap_block_size + heap_block_size
+           + mbuf_size.
+        */
+        for (mp = p->msg.first; mp; mp = mp->next)
+            if (mp->data.attached)
+                values[2] += erts_msg_attached_data_size(mp);
+    }
+
     res = erts_bld_atom_uword_2tup_list(hpp,
                                         sizep,
                                         sizeof(values)/sizeof(*values),
-- 
cgit v1.2.3


From c922f2729dcf2bc5ef571f9d77efea75029469de Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 26 Apr 2016 15:56:21 +0200
Subject: Yield after setting sensitive for save_calls

---
 erts/emulator/beam/bif.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ed5b2983dd..c38d031ab8 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1736,7 +1736,9 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 	   ERTS_TRACE_FLAGS(BIF_P) &= ~F_SENSITIVE;
        }
        erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
-       BIF_RET(old_value);
+       /* make sure to bump all reds so that we get
+          rescheduled immediately so setting takes effect */
+       BIF_RET2(old_value, CONTEXT_REDS);
    }
    else if (BIF_ARG_1 == am_monitor_nodes) {
        /*
-- 
cgit v1.2.3


From 071f9bba246684d401260a2729b8d72bbbfa1874 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 19 Apr 2016 11:24:14 +0200
Subject: erts: Remove some dead code

---
 erts/emulator/beam/beam_bp.h   | 12 ------------
 erts/emulator/beam/erl_trace.c | 36 +-----------------------------------
 erts/emulator/beam/utils.c     | 35 -----------------------------------
 3 files changed, 1 insertion(+), 82 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index bb171be8a6..08641b86d6 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -173,19 +173,7 @@ void erts_clear_time_trace_bif(BeamInstr *pc);
 
 BeamInstr *erts_find_local_func(Eterm mfa[3]);
 
-ERTS_GLB_INLINE Uint erts_bp_sched2ix(void);
-
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE Uint erts_bp_sched2ix(void)
-{
-#ifdef ERTS_SMP
-    ErtsSchedulerData *esdp;
-    esdp = erts_get_scheduler_data();
-    return esdp->no - 1;
-#else
-    return 0;
-#endif
-}
 
 extern erts_smp_atomic32_t erts_active_bp_index;
 extern erts_smp_atomic32_t erts_staging_bp_index;
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index bd88769dfc..ba557ab20e 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -629,6 +629,7 @@ do { \
 #  define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0)
 #endif
 
+
 static void
 write_sys_msg_to_port(Eterm unused_to,
 		      Port* trace_port,
@@ -1696,41 +1697,6 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
 
 }
 
-void
-profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us) {
-    Eterm *hp, msg, timestamp;
-    
-#ifndef ERTS_SMP	
-#define LOCAL_HEAP_SIZE (4 + 7)
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-    hp = local_heap;
-#else    
-    ErlHeapFragment *bp;
-    Uint hsz;
-
-    hsz = 4 + 7;
-	
-    bp = new_message_buffer(hsz);
-    hp = bp->mem;
-#endif
-
-    erts_smp_mtx_lock(&smq_mtx);
-
-    timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
-    msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7;
-#ifndef ERTS_SMP
-    profile_send(NIL, msg);
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-#else
-    enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
-#endif
-    erts_smp_mtx_unlock(&smq_mtx);
-
-}
-
 /* Port profiling */
 
 void
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index b280995488..68006e7ef3 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -71,41 +71,6 @@
 #define HAVE_MALLOPT 0
 #endif
 
-/* profile_scheduler mini message queue */
-
-typedef struct {
-    Uint scheduler_id;
-    Uint no_schedulers;
-    Uint Ms;
-    Uint s;
-    Uint us;
-    Eterm state;
-} profile_sched_msg;
-
-typedef struct {
-    profile_sched_msg msg[2];
-    Uint n;
-} profile_sched_msg_q;
-
-#ifdef ERTS_SMP
-
-#if 0 /* Unused */
-static void 
-dispatch_profile_msg_q(profile_sched_msg_q *psmq)
-{
-    int i = 0;
-    profile_sched_msg *msg = NULL;
-    ASSERT(psmq != NULL);
-    for (i = 0; i < psmq->n; i++) {
-        msg = &(psmq->msg[i]);
-	profile_scheduler_q(make_small(msg->scheduler_id), msg->state, am_undefined, msg->Ms, msg->s, msg->us);
-    }
-}
-#endif
-
-#endif
-
-
 Eterm*
 erts_heap_alloc(Process* p, Uint need, Uint xtra)
 {
-- 
cgit v1.2.3


From d8371a0598ee7c831e8f096cfd5b0af0a0503474 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 19 Apr 2016 11:26:03 +0200
Subject: erts: Expand trace tests for refc binaries

Make sure to cover all of the refc binary cases in tracing
---
 erts/emulator/beam/erl_trace.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index ba557ab20e..32d0ba9f4c 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1860,6 +1860,8 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
         }
 
         data = TUPLE2(hp, caller, data);
+        hp += 3;
+        ASSERT(hp <= (local_heap + LOCAL_HEAP_SIZE) || orig_hp);
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif,
                            am_receive, data, THE_NON_VALUE);
 
-- 
cgit v1.2.3


From e48518e4520146d4b8461689c2aab5882478339f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Fri, 29 Apr 2016 16:18:23 +0200
Subject: erl_process: Restore R18 HiPE PCB offsets

It was recently observed that the LLVM backend for HiPE was broken,
generating code that crashes the VM.

It turns out that this was caused by LLVM hard-coding an offset into the
Erlang PCB, which became incorrect as of commit 3ac08f9b, that
introduced a new member into the PCB before the HiPE-specific state,
changing its offsets.

For now, until a proper fix is implemented, we move this new member to
after the HiPE-specific state, allowing the LLVM backend to work once
more.
---
 erts/emulator/beam/erl_process.h | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 61acf5924b..1c01d705a7 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -925,7 +925,6 @@ struct process {
     Eterm* stop;		/* Stack top */
     Eterm* heap;		/* Heap start */
     Eterm* hend;		/* Heap end */
-    Eterm* abandoned_heap;
     Uint heap_sz;		/* Size of heap in words */
     Uint min_heap_size;         /* Minimum size of heap (in words). */
     Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */
@@ -940,6 +939,16 @@ struct process {
     struct hipe_process_state hipe;
 #endif
 
+    /*
+     * Moved to after "struct hipe_process_state hipe", as a temporary fix for
+     * LLVM hard-coding offsetof(struct process, hipe.nstack) (sic!)
+     * (see void X86FrameLowering::adjustForHiPEPrologue(...) in
+     * lib/Target/X86/X86FrameLowering.cpp).
+     *
+     * Used to be below "Eterm* hend".
+     */
+    Eterm* abandoned_heap;
+
     /*
      * Saved x registers.
      */
-- 
cgit v1.2.3


From 0a5d8f3ae45bd8a10498ef35c1e704983afbbf25 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 20 Apr 2016 15:57:47 +0200
Subject: erts: Add literal_mmap and exec_mmap to system_info

erlang:system_info(allocator) ->
  {Allocator, Version, Features, Settings}

Features includes 'literal_mmap' and/or 'exec_mmap'
if they exist.
---
 erts/emulator/beam/erl_alloc.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 5b76f34165..64a323c434 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2999,7 +2999,12 @@ erts_allocator_options(void *proc)
 #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
     terms[length++] = am_atom_put("sys_aligned_alloc", 17);
 #endif
-
+#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+    terms[length++] = ERTS_MAKE_AM("literal_mmap");
+#endif
+#ifdef ERTS_ALC_A_EXEC
+    terms[length++] = ERTS_MAKE_AM("exec_mmap");
+#endif
     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
 
 #if defined(__GLIBC__)
-- 
cgit v1.2.3


From 48d24da6586a80ab67fe5c242d17ea4d044c917c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 29 Apr 2016 16:44:39 +0200
Subject: erts: Add erts_mmap to system_info(allocator)

to the Settings list

{Allocator, Version, Features, Settings}
---
 erts/emulator/beam/erl_alloc.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 64a323c434..0f68c6bb10 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2963,6 +2963,10 @@ erts_allocator_options(void *proc)
     atoms[length] = am_atom_put("alloc_util", 10); 
     terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp);
 
+
+    atoms[length] = ERTS_MAKE_AM("erts_mmap");
+    terms[length++] = erts_mmap_info_options(&erts_dflt_mmapper, NULL, NULL,
+                                             NULL, hpp, szp);
     {
 	Eterm o[3], v[3];
 	o[0] = am_atom_put("m", 1);
-- 
cgit v1.2.3


From 9f5fd83ffda711d80000bfe1cb7f99c40d9fffde Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 29 Apr 2016 16:55:49 +0200
Subject: erts: Fix system_info({allocator_sizes, mseg_alloc})

will now return

[{instance,0,[{segments_size,9961472,9961472,11010048}]},
 {instance,1,[{segments_size,6291456,6291456,6815744}]},
 {instance,2,[{segments_size,524288,524288,786432}]},
 {instance,3,[{segments_size,1048576,1048576,1835008}]},
 {instance,4,[{segments_size,0,0,262144}]}]

and not just empty lists.
---
 erts/emulator/beam/erl_alloc.c | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 0f68c6bb10..01db65d539 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2841,7 +2841,7 @@ erts_allocator_info(int to, void *arg)
 	int i;
 	for (i = 0; i <= max; i++) {
 	    erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
-	    erts_mseg_info(i, &to, arg, 0, NULL, NULL);
+	    erts_mseg_info(i, &to, arg, 0, 0, NULL, NULL);
 	}
 	erts_print(to, arg, "=allocator:erts_mmap.default_mmap\n");
 	erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis);
@@ -3241,10 +3241,8 @@ reply_alloc_info(void *vair)
 		case ERTS_ALC_INFO_A_MSEG_ALLOC:
 		    alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
 #if HAVE_ERTS_MSEG
-		    ainfo = (air->only_sz
-			     ? NIL
-			     : erts_mseg_info(0, NULL, NULL, hpp != NULL,
-					      hpp, szp));
+		    ainfo = erts_mseg_info(0, NULL, NULL, hpp != NULL,
+                                           air->only_sz, hpp, szp);
 		    ainfo = erts_bld_tuple3(hpp, szp,
                                             alloc_atom,
                                             make_small(0),
@@ -3287,10 +3285,8 @@ reply_alloc_info(void *vair)
 	    case ERTS_ALC_INFO_A_MSEG_ALLOC:
 #if HAVE_ERTS_MSEG && defined(ERTS_SMP)
 		alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
-		ainfo = (air->only_sz
-			 ? NIL
-			 : erts_mseg_info(sched_id, NULL, NULL,
-					  hpp != NULL, hpp, szp));
+		ainfo = erts_mseg_info(sched_id, NULL, NULL,
+                                       hpp != NULL, air->only_sz, hpp, szp);
 		ainfo = erts_bld_tuple(hpp, szp, 3,
 				       alloc_atom,
 				       make_small(sched_id),
-- 
cgit v1.2.3


From 8b9b4e8bffe087752463287457330803e69a1c81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 2 Mar 2016 16:22:11 +0100
Subject: runtime_tools: Add lttng 'procs' tracing

---
 erts/emulator/beam/erl_trace.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 96fc46c817..fa1d03c588 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1294,7 +1294,7 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
         mfa = TUPLE3(hp, mod, func, args);
         hp += 4;
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_PROCS,
-                am_spawn, pid, mfa);
+                           what, pid, mfa);
     }
 }
 
-- 
cgit v1.2.3


From f5fa3ac80d2f7fcd11ac3e702c29df5ef6c204db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 3 Mar 2016 17:13:32 +0100
Subject: runtime_tools: Add lttng 'call' tracing

---
 erts/emulator/beam/erl_trace.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index fa1d03c588..0842c0c0a4 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1000,7 +1000,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer)
     mfa = TUPLE3(hp, mod, name, make_small(arity));
     hp += 4;
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           NULL, TRACE_FUN_DEFAULT, am_return_from, mfa, retval, am_true);
+                           NULL, TRACE_FUN_CALL, am_return_from, mfa, retval, am_true);
 }
 
 /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, 
@@ -1055,7 +1055,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
     cv = TUPLE2(hp, class, value);
     hp += 3;
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           NULL, TRACE_FUN_DEFAULT, am_exception_from, mfa_tuple, cv, am_true);
+                           NULL, TRACE_FUN_CALL, am_exception_from, mfa_tuple, cv, am_true);
 }
 
 /*
-- 
cgit v1.2.3


From 6f89f183317c930fe1199ad3320825b02bb861da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 7 Mar 2016 16:04:04 +0100
Subject: erts: Extend garbage collection trace

Replace 'gc_start' and 'gc_end' with
* 'gc_minor_start'
* 'gc_minor_end'
* 'gc_major_start'
* 'gc_major_end'
---
 erts/emulator/beam/atom.names  |  4 ++++
 erts/emulator/beam/erl_gc.c    | 41 ++++++++++++++++++++++-------------------
 erts/emulator/beam/erl_trace.c | 13 ++++++++-----
 erts/emulator/beam/erl_trace.h |  2 +-
 4 files changed, 35 insertions(+), 25 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 8c51f788c0..3022c0a99a 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -273,6 +273,10 @@ atom garbage_collecting
 atom garbage_collection
 atom garbage_collection_info
 atom gc_end
+atom gc_major_end
+atom gc_major_start
+atom gc_minor_end
+atom gc_minor_start
 atom gc_start
 atom Ge='>='
 atom generational
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index f33ade27f3..afb2ec1cc2 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -593,10 +593,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     esdp = erts_get_scheduler_data();
 
-    if (IS_TRACED_FL(p, F_TRACE_GC)) {
-        trace_gc(p, am_gc_start);
-    }
-
     erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     if (erts_system_monitor_long_gc != 0)
 	start_time = erts_get_monotonic_time(esdp);
@@ -619,18 +615,29 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
      */
 
     if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) {
-	DTRACE2(gc_minor_start, pidbuf, need);
-	reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
-	DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
-	if (reds < 0)
-	    goto do_major_collection;
-    }
-    else {
-    do_major_collection:
+        if (IS_TRACED_FL(p, F_TRACE_GC)) {
+            trace_gc(p, am_gc_minor_start, need);
+        }
+        DTRACE2(gc_minor_start, pidbuf, need);
+        reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+        DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
+        if (IS_TRACED_FL(p, F_TRACE_GC)) {
+            trace_gc(p, am_gc_minor_end, reclaimed_now);
+        }
+        if (reds < 0)
+            goto do_major_collection;
+    } else {
+do_major_collection:
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL);
-	DTRACE2(gc_major_start, pidbuf, need);
-	reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
-	DTRACE2(gc_major_end, pidbuf, reclaimed_now);
+        if (IS_TRACED_FL(p, F_TRACE_GC)) {
+            trace_gc(p, am_gc_major_start, need);
+        }
+        DTRACE2(gc_major_start, pidbuf, need);
+        reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+        DTRACE2(gc_major_end, pidbuf, reclaimed_now);
+        if (IS_TRACED_FL(p, F_TRACE_GC)) {
+            trace_gc(p, am_gc_major_end, reclaimed_now);
+        }
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC);
     }
 
@@ -646,10 +653,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 
-    if (IS_TRACED_FL(p, F_TRACE_GC)) {
-        trace_gc(p, am_gc_end);
-    }
-
     if (erts_system_monitor_long_gc != 0) {
 	ErtsMonotonicTime end_time;
 	Uint gc_time;
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 0842c0c0a4..9146cc7c7f 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1326,23 +1326,26 @@ void save_calls(Process *p, Export *e)
  * are all small (atomic) integers.
  */
 void
-trace_gc(Process *p, Eterm what)
+trace_gc(Process *p, Eterm what, Uint size)
 {
     ErtsTracerNif *tnif = NULL;
     Eterm* hp;
     Eterm msg = NIL;
-    Uint size = 0;
+    Uint sz = 0;
+    Eterm tup;
 
     if (is_tracer_proc_enabled(
             p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, what)) {
 
-        (void) erts_process_gc_info(p, &size, NULL);
-        hp = HAlloc(p, size);
+        (void) erts_process_gc_info(p, &sz, NULL);
+        hp = HAlloc(p, sz + 3 + 2);
 
         msg = erts_process_gc_info(p, NULL, &hp);
+	tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3;
+        msg = CONS(hp, tup, msg); hp += 2;
 
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_GC,
-                what, msg, THE_NON_VALUE);
+                what, msg, am_undefined);
     }
 }
 
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 177fd373a6..9a007e62ec 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -103,7 +103,7 @@ void trace_sched(Process*, ErtsProcLocks, Eterm);
 void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm);
 void trace_proc_spawn(Process*, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args);
 void save_calls(Process *p, Export *);
-void trace_gc(Process *p, Eterm what);
+void trace_gc(Process *p, Eterm what, Uint size);
 /* port tracing */
 void trace_virtual_sched(Process*, ErtsProcLocks, Eterm);
 void trace_sched_ports(Port *pp, Eterm);
-- 
cgit v1.2.3


From a4811133b70370e54162d256d61ef4c084e1f4fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 17 Mar 2016 17:53:55 +0100
Subject: erts: Extend 'enabled' tracer callbacks

Adds the following capabilities to a tracer backend

* enabled_procs/3
* enabled_ports/3
* enabled_running/3
* enabled_call/3
* enabled_send/3
* enabled_receive/3
* enabled_garbage_collection/3

These functions will fall back to

    enabled/3

if not provided and the tracepoint is active.
---
 erts/emulator/beam/erl_trace.c | 241 +++++++++++++++++++++++++----------------
 1 file changed, 147 insertions(+), 94 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 9146cc7c7f..a2f94677bb 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -237,6 +237,7 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp)
     }
 }
 
+#ifdef ERTS_SMP
 #define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p))
 
 static ERTS_INLINE Uint
@@ -257,6 +258,7 @@ patch_ts_size(int ts_type)
 	return 0;
     }
 }
+#endif
 
 /*
  * Write a timestamp. The timestamp MUST be the last
@@ -360,18 +362,25 @@ void erts_init_trace(void) {
    (*(BPP))->mem)
 
 enum ErtsTracerOpt {
-    TRACE_FUN_DEFAULT = 0,
-    TRACE_FUN_ENABLED = 1,
-    TRACE_FUN_SEND    = 2,
-    TRACE_FUN_RECEIVE = 3,
-    TRACE_FUN_CALL    = 4,
-    TRACE_FUN_RUNNING = 5,
-    TRACE_FUN_GC      = 6,
-    TRACE_FUN_PROCS   = 7,
-    TRACE_FUN_PORTS   = 8
+    TRACE_FUN_DEFAULT   = 0,
+    TRACE_FUN_ENABLED   = 1,
+    TRACE_FUN_T_SEND    = 2,
+    TRACE_FUN_T_RECEIVE = 3,
+    TRACE_FUN_T_CALL    = 4,
+    TRACE_FUN_T_RUNNING = 5,
+    TRACE_FUN_T_GC      = 6,
+    TRACE_FUN_T_PROCS   = 7,
+    TRACE_FUN_T_PORTS   = 8,
+    TRACE_FUN_E_SEND    = 9,
+    TRACE_FUN_E_RECEIVE = 10,
+    TRACE_FUN_E_CALL    = 11,
+    TRACE_FUN_E_RUNNING = 12,
+    TRACE_FUN_E_GC      = 13,
+    TRACE_FUN_E_PROCS   = 14,
+    TRACE_FUN_E_PORTS   = 15
 };
 
-#define NIF_TRACER_TYPES (9)
+#define NIF_TRACER_TYPES (16)
 
 
 static ERTS_INLINE int
@@ -381,15 +390,19 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee, const ErtsTracer tracer,
                        Eterm tag, Eterm msg, Eterm extra, Eterm pam_result);
 static ERTS_INLINE int
 send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
-                   Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt,
+                   Eterm t_p_id, ErtsTracerNif *tnif,
+                   enum ErtsTracerOpt topt,
                    Eterm tag, Eterm msg, Eterm extra);
 static ERTS_INLINE Eterm
 call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
-                    ErtsTracerNif **tnif_ref, Eterm tag, Eterm t_p_id);
+                    ErtsTracerNif **tnif_ref,
+                    enum ErtsTracerOpt topt,
+                    Eterm tag, Eterm t_p_id);
 static int
-is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
-                       ErtsPTabElementCommon *t_p,
-                       ErtsTracerNif **tnif_ret, Eterm tag);
+is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
+                  ErtsPTabElementCommon *t_p,
+                  ErtsTracerNif **tnif_ret,
+                  enum ErtsTracerOpt topt, Eterm tag);
 
 #define SEND_TO_TRACER(c_p, tag, msg)                                 \
     send_to_tracer_nif(c_p, &(c_p)->common, (c_p)->common.id, NULL,   \
@@ -449,7 +462,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new
     if (!ERTS_TRACER_IS_NIL(new)) {
         Eterm nif_result = call_enabled_tracer(
             NULL, new, NULL,
-            am_trace_status, am_undefined);
+            TRACE_FUN_ENABLED, am_trace_status, am_undefined);
         switch (nif_result) {
         case am_trace: break;
         default:
@@ -481,7 +494,8 @@ erts_get_system_seq_tracer(void)
     erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
 
     if (st != erts_tracer_nil &&
-        call_enabled_tracer(NULL, st, NULL, am_trace_status, am_undefined) == am_remove) {
+        call_enabled_tracer(NULL, st, NULL, TRACE_FUN_ENABLED,
+                            am_trace_status, am_undefined) == am_remove) {
         erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil);
         st = erts_tracer_nil;
     }
@@ -500,10 +514,11 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp,
     if (ERTS_TRACER_IS_NIL(*default_tracer)) {
 	*default_trace_flags &= ~TRACEE_FLAGS;
     } else {
-        Eterm nif_result = call_enabled_tracer(
-            NULL, *default_tracer, NULL,
-            am_trace_status, am_undefined);
-        switch (nif_result) {
+        Eterm nif_res;
+        nif_res = call_enabled_tracer(NULL, *default_tracer,
+                                      NULL, TRACE_FUN_ENABLED,
+                                      am_trace_status, am_undefined);
+        switch (nif_res) {
         case am_trace: break;
         default: {
             ErtsTracer curr_default_tracer = *default_tracer;
@@ -753,7 +768,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	break;
     }
 
-    if (!is_tracer_proc_enabled(p, locks, &p->common, &tnif, what))
+    if (!is_tracer_enabled(p, locks, &p->common, &tnif, TRACE_FUN_E_RUNNING, what))
         return;
 
     if (ERTS_PROC_IS_EXITING(p))
@@ -772,7 +787,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	hp += 4;
     }
 
-    send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_RUNNING,
+    send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_RUNNING,
                        what, tmp, THE_NON_VALUE);
 }
 
@@ -811,8 +826,9 @@ trace_send(Process *p, Eterm to, Eterm msg)
 	operation = am_send_to_non_existing_process;
     }
 
-    if (is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, operation))
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_SEND,
+    if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif,
+                TRACE_FUN_E_SEND, operation))
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND,
                 operation, msg, to);
 }
 
@@ -823,10 +839,10 @@ void
 trace_receive(Process *c_p, Eterm msg)
 {
     ErtsTracerNif *tnif = NULL;
-    if (is_tracer_proc_enabled(NULL, 0, &c_p->common,
-                               &tnif, am_receive))
+    if (is_tracer_enabled(NULL, 0, &c_p->common, &tnif,
+                TRACE_FUN_E_RECEIVE, am_receive))
         send_to_tracer_nif(NULL, &c_p->common, c_p->common.id,
-                           tnif, TRACE_FUN_RECEIVE,
+                           tnif, TRACE_FUN_T_RECEIVE,
                            am_receive, msg, THE_NON_VALUE);
 }
 
@@ -837,8 +853,9 @@ seq_trace_update_send(Process *p)
     ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p))));
     if (have_no_seqtrace(SEQ_TRACE_TOKEN(p)) ||
         (seq_tracer != NIL &&
-         call_enabled_tracer( NULL, seq_tracer, NULL, am_trace_status,
-                              p ? p->common.id : am_undefined) != am_trace)
+         call_enabled_tracer(NULL, seq_tracer, NULL,
+                             TRACE_FUN_ENABLED, am_trace_status,
+                             p ? p->common.id : am_undefined) != am_trace)
 #ifdef USE_VM_PROBES
 	 || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag)
 #endif
@@ -884,9 +901,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
     ASSERT(is_tuple(token) || is_nil(token));
     if (token == NIL || (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE) ||
         ERTS_TRACER_IS_NIL(seq_tracer) ||
-        call_enabled_tracer(
-            NULL, seq_tracer, NULL, am_trace_status,
-            process ? process->common.id : am_undefined) != am_trace) {
+        call_enabled_tracer(NULL, seq_tracer,
+                            NULL, TRACE_FUN_ENABLED,
+                            am_trace_status,
+                            process ? process->common.id : am_undefined) != am_trace) {
 	return;
     }
 
@@ -1000,7 +1018,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer)
     mfa = TUPLE3(hp, mod, name, make_small(arity));
     hp += 4;
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           NULL, TRACE_FUN_CALL, am_return_from, mfa, retval, am_true);
+                           NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true);
 }
 
 /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, 
@@ -1055,7 +1073,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
     cv = TUPLE2(hp, class, value);
     hp += 3;
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           NULL, TRACE_FUN_CALL, am_exception_from, mfa_tuple, cv, am_true);
+                           NULL, TRACE_FUN_T_CALL, am_exception_from, mfa_tuple, cv, am_true);
 }
 
 /*
@@ -1105,7 +1123,8 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	 *     use process flags
 	 */
 	tracee_flags = &ERTS_TRACE_FLAGS(p);
-        if (!is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, am_call)) {
+        if (!is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif,
+                    TRACE_FUN_E_CALL, am_call)) {
             return 0;
         }
     } else {
@@ -1120,7 +1139,9 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
         }
 	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
-        switch (call_enabled_tracer(p, *tracer, &tnif, am_call, p->common.id)) {
+        switch (call_enabled_tracer(p, *tracer,
+                                    &tnif, TRACE_FUN_T_CALL,
+                                    am_call, p->common.id)) {
         default:
         case am_remove: *tracer = erts_tracer_nil;
         case am_discard: return 0;
@@ -1244,7 +1265,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
      * Build the trace tuple and send it to the port.
      */
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           tnif, TRACE_FUN_CALL, am_call, mfa_tuple, THE_NON_VALUE, pam_result);
+                           tnif, TRACE_FUN_T_CALL, am_call, mfa_tuple, THE_NON_VALUE, pam_result);
     erts_match_set_release_result(p);
 
     if (match_spec && tracer == &pre_ms_tracer)
@@ -1266,8 +1287,9 @@ trace_proc(Process *c_p, ErtsProcLocks c_p_locks,
            Process *t_p, Eterm what, Eterm data)
 {
     ErtsTracerNif *tnif = NULL;
-    if (is_tracer_proc_enabled(c_p, c_p_locks, &t_p->common, &tnif, what))
-        send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_PROCS,
+    if (is_tracer_enabled(c_p, c_p_locks, &t_p->common, &tnif,
+                TRACE_FUN_E_PROCS, what))
+        send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS,
                            what, data, THE_NON_VALUE);
 }
 
@@ -1284,16 +1306,16 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
 		 Eterm mod, Eterm func, Eterm args)
 {
     ErtsTracerNif *tnif = NULL;
-    if (is_tracer_proc_enabled(p, ERTS_PROC_LOCKS_ALL &
+    if (is_tracer_enabled(p, ERTS_PROC_LOCKS_ALL &
                                ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE),
-                               &p->common, &tnif, what)) {
+                               &p->common, &tnif, TRACE_FUN_E_PROCS, what)) {
         Eterm mfa;
         Eterm* hp;
 
         hp = HAlloc(p, 4);
         mfa = TUPLE3(hp, mod, func, args);
         hp += 4;
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_PROCS,
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS,
                            what, pid, mfa);
     }
 }
@@ -1334,8 +1356,7 @@ trace_gc(Process *p, Eterm what, Uint size)
     Uint sz = 0;
     Eterm tup;
 
-    if (is_tracer_proc_enabled(
-            p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, what)) {
+    if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, TRACE_FUN_E_GC, what)) {
 
         (void) erts_process_gc_info(p, &sz, NULL);
         hp = HAlloc(p, sz + 3 + 2);
@@ -1344,7 +1365,7 @@ trace_gc(Process *p, Eterm what, Uint size)
 	tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3;
         msg = CONS(hp, tup, msg); hp += 2;
 
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_GC,
+        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC,
                 what, msg, am_undefined);
     }
 }
@@ -1758,8 +1779,8 @@ void
 trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
     ErtsTracerNif *tnif = NULL;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-    if (is_tracer_proc_enabled(NULL, 0, &p->common, &tnif, am_open))
-        send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_PORTS,
+    if (is_tracer_enabled(NULL, 0, &p->common, &tnif, TRACE_FUN_E_PORTS, am_open))
+        send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PORTS,
                 am_open, calling_pid, drv_name);
 }
 
@@ -1777,8 +1798,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what))
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_PORTS,
+    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_PORTS, what))
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PORTS,
                            what, data, THE_NON_VALUE);
 }
 
@@ -1823,7 +1844,7 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, am_receive)) {
+    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_RECEIVE, am_receive)) {
         /* We can use a stack heap here, as the nif is called in the
            context of a port */
 #define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT) + 3)
@@ -1915,7 +1936,7 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
         }
 
         data = TUPLE2(hp, caller, data);
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_RECEIVE,
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_RECEIVE,
                            am_receive, data, THE_NON_VALUE);
 
         if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
@@ -1937,8 +1958,8 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, op))
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_SEND,
+    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, op))
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND,
                            op, msg, receiver);
 }
 
@@ -1948,7 +1969,7 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, am_send)) {
+    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, am_send)) {
         Eterm msg;
         Binary* bptr = NULL;
 #define LOCAL_HEAP_SIZE (3 + 3 + heap_bin_size(ERL_ONHEAP_BIN_LIMIT))
@@ -1967,7 +1988,7 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
         msg = TUPLE2(hp, t_p->common.id, msg);
         hp += 3;
 
-        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_SEND,
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND,
                            am_send, msg, to);
         if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
             erts_bin_free(bptr);
@@ -1996,9 +2017,9 @@ trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) {
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-    if (is_tracer_proc_enabled(NULL, 0, &t_p->common, &tnif, what))
+    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_RUNNING, what))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
-                           tnif, TRACE_FUN_RUNNING,
+                           tnif, TRACE_FUN_T_RUNNING,
                            what, where, THE_NON_VALUE);
 }
 
@@ -2570,33 +2591,62 @@ static void init_tracer_template(ErtsTracerNif *tnif) {
     tnif->tracers[TRACE_FUN_ENABLED].cb    = NULL;
 
     /* specific tracer functions */
-    tnif->tracers[TRACE_FUN_SEND].name  = "trace_send";
-    tnif->tracers[TRACE_FUN_SEND].arity = 6;
-    tnif->tracers[TRACE_FUN_SEND].cb    = NULL;
+    tnif->tracers[TRACE_FUN_T_SEND].name  = "trace_send";
+    tnif->tracers[TRACE_FUN_T_SEND].arity = 6;
+    tnif->tracers[TRACE_FUN_T_SEND].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_T_RECEIVE].name  = "trace_receive";
+    tnif->tracers[TRACE_FUN_T_RECEIVE].arity = 6;
+    tnif->tracers[TRACE_FUN_T_RECEIVE].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_T_CALL].name  = "trace_call";
+    tnif->tracers[TRACE_FUN_T_CALL].arity = 6;
+    tnif->tracers[TRACE_FUN_T_CALL].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_T_RUNNING].name  = "trace_running";
+    tnif->tracers[TRACE_FUN_T_RUNNING].arity = 6;
+    tnif->tracers[TRACE_FUN_T_RUNNING].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_RECEIVE].name  = "trace_receive";
-    tnif->tracers[TRACE_FUN_RECEIVE].arity = 6;
-    tnif->tracers[TRACE_FUN_RECEIVE].cb    = NULL;
+    tnif->tracers[TRACE_FUN_T_GC].name  = "trace_garbage_collection";
+    tnif->tracers[TRACE_FUN_T_GC].arity = 6;
+    tnif->tracers[TRACE_FUN_T_GC].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_CALL].name  = "trace_call";
-    tnif->tracers[TRACE_FUN_CALL].arity = 6;
-    tnif->tracers[TRACE_FUN_CALL].cb    = NULL;
+    tnif->tracers[TRACE_FUN_T_PROCS].name  = "trace_procs";
+    tnif->tracers[TRACE_FUN_T_PROCS].arity = 6;
+    tnif->tracers[TRACE_FUN_T_PROCS].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_RUNNING].name  = "trace_running";
-    tnif->tracers[TRACE_FUN_RUNNING].arity = 6;
-    tnif->tracers[TRACE_FUN_RUNNING].cb    = NULL;
+    tnif->tracers[TRACE_FUN_T_PORTS].name  = "trace_ports";
+    tnif->tracers[TRACE_FUN_T_PORTS].arity = 6;
+    tnif->tracers[TRACE_FUN_T_PORTS].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_GC].name  = "trace_garbage_collection";
-    tnif->tracers[TRACE_FUN_GC].arity = 6;
-    tnif->tracers[TRACE_FUN_GC].cb    = NULL;
+    /* specific enabled functions */
+    tnif->tracers[TRACE_FUN_E_SEND].name  = "enabled_send";
+    tnif->tracers[TRACE_FUN_E_SEND].arity = 3;
+    tnif->tracers[TRACE_FUN_E_SEND].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_PROCS].name  = "trace_procs";
-    tnif->tracers[TRACE_FUN_PROCS].arity = 6;
-    tnif->tracers[TRACE_FUN_PROCS].cb    = NULL;
+    tnif->tracers[TRACE_FUN_E_RECEIVE].name  = "enabled_receive";
+    tnif->tracers[TRACE_FUN_E_RECEIVE].arity = 3;
+    tnif->tracers[TRACE_FUN_E_RECEIVE].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_PORTS].name  = "trace_ports";
-    tnif->tracers[TRACE_FUN_PORTS].arity = 6;
-    tnif->tracers[TRACE_FUN_PORTS].cb    = NULL;
+    tnif->tracers[TRACE_FUN_E_CALL].name  = "enabled_call";
+    tnif->tracers[TRACE_FUN_E_CALL].arity = 3;
+    tnif->tracers[TRACE_FUN_E_CALL].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_E_RUNNING].name  = "enabled_running";
+    tnif->tracers[TRACE_FUN_E_RUNNING].arity = 3;
+    tnif->tracers[TRACE_FUN_E_RUNNING].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_E_GC].name  = "enabled_garbage_collection";
+    tnif->tracers[TRACE_FUN_E_GC].arity = 3;
+    tnif->tracers[TRACE_FUN_E_GC].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_E_PROCS].name  = "enabled_procs";
+    tnif->tracers[TRACE_FUN_E_PROCS].arity = 3;
+    tnif->tracers[TRACE_FUN_E_PROCS].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_E_PORTS].name  = "enabled_ports";
+    tnif->tracers[TRACE_FUN_E_PORTS].arity = 3;
+    tnif->tracers[TRACE_FUN_E_PORTS].cb    = NULL;
 }
 
 static Hash *tracer_hash = NULL;
@@ -2739,6 +2789,7 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee,
         flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1));
         Eterm *map_values = flatmap_get_values(map);
         enum ErtsTracerOpt topt = (tnif->tracers[topt_arg].cb) ? topt_arg : TRACE_FUN_DEFAULT;
+        ASSERT(topt < NIF_TRACER_TYPES);
 
         argv[0] = tag;
         argv[1] = ERTS_TRACER_STATE(tracer);
@@ -2808,26 +2859,29 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
 
 static ERTS_INLINE Eterm
 call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
-                    ErtsTracerNif **tnif_ret, Eterm tag, Eterm t_p_id)
-{
+                    ErtsTracerNif **tnif_ret,
+                    enum ErtsTracerOpt topt,
+                    Eterm tag, Eterm t_p_id) {
     ErtsTracerNif *tnif = lookup_tracer_nif(tracer);
     if (tnif) {
         Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id};
+        topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_ENABLED;
+        ASSERT(topt < NIF_TRACER_TYPES);
+        ASSERT(tnif->tracers[topt].cb != NULL);
         if (tnif_ret) *tnif_ret = tnif;
-        return erts_nif_call_function(
-            c_p, NULL, tnif->nif_mod,
-            tnif->tracers[TRACE_FUN_ENABLED].cb,
-            tnif->tracers[TRACE_FUN_ENABLED].arity,
-            argv);
+        return erts_nif_call_function(c_p, NULL, tnif->nif_mod,
+                                      tnif->tracers[topt].cb,
+                                      tnif->tracers[topt].arity,
+                                      argv);
     }
     return am_remove;
 }
 
 static int
-is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
-                       ErtsPTabElementCommon *t_p,
-                       ErtsTracerNif **tnif_ret, Eterm tag)
-{
+is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
+                  ErtsPTabElementCommon *t_p,
+                  ErtsTracerNif **tnif_ret,
+                  enum ErtsTracerOpt topt, Eterm tag) {
     Eterm nif_result;
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
@@ -2845,7 +2899,7 @@ is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
     }
 #endif
 
-    nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, tag, t_p->id);
+    nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, topt, tag, t_p->id);
     switch (nif_result) {
     case am_discard: return 0;
     case am_trace: return 1;
@@ -2878,14 +2932,13 @@ is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
             erts_smp_proc_unlock(c_p, c_p_xlocks);
     }
 
-
     return 0;
 }
 
 int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
                                 ErtsPTabElementCommon *t_p, Eterm type)
 {
-    return is_tracer_proc_enabled(c_p, c_p_locks, t_p, NULL, am_trace_status);
+    return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_ENABLED, am_trace_status);
 }
 
 int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer)
@@ -2893,7 +2946,7 @@ int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer)
     ErtsTracerNif *tnif = lookup_tracer_nif(tracer);
     if (tnif) {
         Eterm nif_result = call_enabled_tracer(c_p, tracer, &tnif,
-                                               am_trace_status,
+                                               TRACE_FUN_ENABLED, am_trace_status,
                                                c_p->common.id);
         switch (nif_result) {
         case am_discard:
-- 
cgit v1.2.3


From 4e2a61519f022a85e7774d9be65067d33256cb01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 20 Apr 2016 11:58:01 +0200
Subject: erts: Extend 'enabled' and 'trace' tracer callbacks

Adds the following capabilities to a tracer backend

* enabled_running_procs/3 changed from enabled_running/3
* enabled_running_ports/3
* trace_running_procs/6 changed from trace_running/6
* trace_running_ports/6
---
 erts/emulator/beam/erl_trace.c | 59 +++++++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index a2f94677bb..7b52959dd8 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -367,20 +367,22 @@ enum ErtsTracerOpt {
     TRACE_FUN_T_SEND    = 2,
     TRACE_FUN_T_RECEIVE = 3,
     TRACE_FUN_T_CALL    = 4,
-    TRACE_FUN_T_RUNNING = 5,
-    TRACE_FUN_T_GC      = 6,
-    TRACE_FUN_T_PROCS   = 7,
-    TRACE_FUN_T_PORTS   = 8,
-    TRACE_FUN_E_SEND    = 9,
-    TRACE_FUN_E_RECEIVE = 10,
-    TRACE_FUN_E_CALL    = 11,
-    TRACE_FUN_E_RUNNING = 12,
-    TRACE_FUN_E_GC      = 13,
-    TRACE_FUN_E_PROCS   = 14,
-    TRACE_FUN_E_PORTS   = 15
+    TRACE_FUN_T_SCHED_PROC = 5,
+    TRACE_FUN_T_SCHED_PORT = 6,
+    TRACE_FUN_T_GC      = 7,
+    TRACE_FUN_T_PROCS   = 8,
+    TRACE_FUN_T_PORTS   = 9,
+    TRACE_FUN_E_SEND    = 10,
+    TRACE_FUN_E_RECEIVE = 11,
+    TRACE_FUN_E_CALL    = 12,
+    TRACE_FUN_E_SCHED_PROC = 13,
+    TRACE_FUN_E_SCHED_PORT = 14,
+    TRACE_FUN_E_GC      = 15,
+    TRACE_FUN_E_PROCS   = 16,
+    TRACE_FUN_E_PORTS   = 17
 };
 
-#define NIF_TRACER_TYPES (16)
+#define NIF_TRACER_TYPES (18)
 
 
 static ERTS_INLINE int
@@ -768,7 +770,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	break;
     }
 
-    if (!is_tracer_enabled(p, locks, &p->common, &tnif, TRACE_FUN_E_RUNNING, what))
+    if (!is_tracer_enabled(p, locks, &p->common, &tnif, TRACE_FUN_E_SCHED_PROC, what))
         return;
 
     if (ERTS_PROC_IS_EXITING(p))
@@ -787,7 +789,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	hp += 4;
     }
 
-    send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_RUNNING,
+    send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SCHED_PROC,
                        what, tmp, THE_NON_VALUE);
 }
 
@@ -2017,9 +2019,9 @@ trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) {
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_RUNNING, what))
+    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SCHED_PORT, what))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
-                           tnif, TRACE_FUN_T_RUNNING,
+                           tnif, TRACE_FUN_T_SCHED_PORT,
                            what, where, THE_NON_VALUE);
 }
 
@@ -2603,9 +2605,13 @@ static void init_tracer_template(ErtsTracerNif *tnif) {
     tnif->tracers[TRACE_FUN_T_CALL].arity = 6;
     tnif->tracers[TRACE_FUN_T_CALL].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_T_RUNNING].name  = "trace_running";
-    tnif->tracers[TRACE_FUN_T_RUNNING].arity = 6;
-    tnif->tracers[TRACE_FUN_T_RUNNING].cb    = NULL;
+    tnif->tracers[TRACE_FUN_T_SCHED_PROC].name  = "trace_running_procs";
+    tnif->tracers[TRACE_FUN_T_SCHED_PROC].arity = 6;
+    tnif->tracers[TRACE_FUN_T_SCHED_PROC].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_T_SCHED_PORT].name  = "trace_running_ports";
+    tnif->tracers[TRACE_FUN_T_SCHED_PORT].arity = 6;
+    tnif->tracers[TRACE_FUN_T_SCHED_PORT].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_GC].name  = "trace_garbage_collection";
     tnif->tracers[TRACE_FUN_T_GC].arity = 6;
@@ -2632,9 +2638,13 @@ static void init_tracer_template(ErtsTracerNif *tnif) {
     tnif->tracers[TRACE_FUN_E_CALL].arity = 3;
     tnif->tracers[TRACE_FUN_E_CALL].cb    = NULL;
 
-    tnif->tracers[TRACE_FUN_E_RUNNING].name  = "enabled_running";
-    tnif->tracers[TRACE_FUN_E_RUNNING].arity = 3;
-    tnif->tracers[TRACE_FUN_E_RUNNING].cb    = NULL;
+    tnif->tracers[TRACE_FUN_E_SCHED_PROC].name  = "enabled_running_procs";
+    tnif->tracers[TRACE_FUN_E_SCHED_PROC].arity = 3;
+    tnif->tracers[TRACE_FUN_E_SCHED_PROC].cb    = NULL;
+
+    tnif->tracers[TRACE_FUN_E_SCHED_PORT].name  = "enabled_running_ports";
+    tnif->tracers[TRACE_FUN_E_SCHED_PORT].arity = 3;
+    tnif->tracers[TRACE_FUN_E_SCHED_PORT].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_E_GC].name  = "enabled_garbage_collection";
     tnif->tracers[TRACE_FUN_E_GC].arity = 3;
@@ -2780,7 +2790,7 @@ static ERTS_INLINE int
 send_to_tracer_nif_raw(Process *c_p, Process *tracee,
                        const ErtsTracer tracer, Uint tracee_flags,
                        Eterm t_p_id, ErtsTracerNif *tnif,
-                       enum ErtsTracerOpt topt_arg,
+                       enum ErtsTracerOpt topt,
                        Eterm tag, Eterm msg, Eterm extra, Eterm pam_result)
 {
     if (tnif || (tnif = lookup_tracer_nif(tracer)) != NULL) {
@@ -2788,7 +2798,8 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee,
         Eterm argv[6], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)];
         flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1));
         Eterm *map_values = flatmap_get_values(map);
-        enum ErtsTracerOpt topt = (tnif->tracers[topt_arg].cb) ? topt_arg : TRACE_FUN_DEFAULT;
+
+        topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_DEFAULT;
         ASSERT(topt < NIF_TRACER_TYPES);
 
         argv[0] = tag;
-- 
cgit v1.2.3


From a1f50deb9c87cbb7117baf22928c6cd7075ac2e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 20 Apr 2016 20:23:56 +0200
Subject: erts: Fix return_to trace callback

---
 erts/emulator/beam/erl_trace.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 7b52959dd8..56899f574a 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -406,10 +406,6 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
                   ErtsTracerNif **tnif_ret,
                   enum ErtsTracerOpt topt, Eterm tag);
 
-#define SEND_TO_TRACER(c_p, tag, msg)                                 \
-    send_to_tracer_nif(c_p, &(c_p)->common, (c_p)->common.id, NULL,   \
-            TRACE_FUN_DEFAULT, tag, msg, THE_NON_VALUE)
-
 static Uint active_sched;
 
 void
@@ -966,7 +962,8 @@ erts_trace_return_to(Process *p, BeamInstr *pc)
 	mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2]));
     }
 
-    SEND_TO_TRACER(p, am_return_to, mfa);
+    send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL,
+                       am_return_to, mfa, THE_NON_VALUE);
 }
 
 
-- 
cgit v1.2.3


From 4050fc5665a4819b15f204543c28c59726b4aac9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 12 Apr 2016 07:07:17 +0200
Subject: Refactor erts_finish_loading() and insert_new_code()

As a preparation for fixing some issues with -on_load(), we will
integrate insert_new_code() into erts_finish_loading. Also rename
insert_new_code() to stub_insert_new_code() to make it clear that
it will only be used when making a stub module
---
 erts/emulator/beam/beam_load.c | 53 +++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 19 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2e21a553ed..9bd9b4bfc0 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -479,9 +479,9 @@ static void free_loader_state(Binary* magic);
 static ErlHeapFragment* new_literal_fragment(Uint size);
 static void free_literal_fragment(ErlHeapFragment*);
 static void loader_state_dtor(Binary* magic);
-static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
-			     Eterm group_leader, Eterm module,
-			     BeamCodeHeader* code, Uint size);
+static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
+				  Eterm group_leader, Eterm module,
+				  BeamCodeHeader* code, Uint size);
 static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
@@ -513,7 +513,7 @@ static GenOp* gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
 
 static int freeze_code(LoaderState* stp);
 
-static void final_touch(LoaderState* stp);
+static void final_touch(LoaderState* stp, struct erl_module_instance* inst_p);
 static void short_file(int line, LoaderState* stp, unsigned needed);
 static void load_printf(int line, LoaderState* context, char *fmt, ...);
 static int transform_engine(LoaderState* st);
@@ -768,6 +768,9 @@ erts_finish_loading(Binary* magic, Process* c_p,
 {
     Eterm retval;
     LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
+    Module* mod_tab_p;
+    struct erl_module_instance* inst_p;
+    Uint size;
 
     /*
      * No other process may run since we will update the export
@@ -784,11 +787,25 @@ erts_finish_loading(Binary* magic, Process* c_p,
      */
 
     CHKBLK(ERTS_ALC_T_CODE,stp->code);
-    retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module,
-			     stp->hdr, stp->loaded_size);
-    if (retval != NIL) {
-	goto load_error;
-    }
+    retval = beam_make_current_old(c_p, c_p_locks, stp->module);
+    ASSERT(retval == NIL);
+
+    /*
+     * Update module table.
+     */
+
+    size = stp->loaded_size;
+    erts_total_code_size += size;
+    mod_tab_p = erts_put_module(stp->module);
+    inst_p = &mod_tab_p->curr;
+    inst_p->code_hdr = stp->hdr;
+    inst_p->code_length = size;
+
+    /*
+     * Update ranges (used for finding a function from a PC value).
+     */
+
+    erts_update_ranges((BeamInstr*)inst_p->code_hdr, size);
 
     /*
      * Ready for the final touch: fixing the export table entries for
@@ -796,7 +813,7 @@ erts_finish_loading(Binary* magic, Process* c_p,
      */
     
     CHKBLK(ERTS_ALC_T_CODE,stp->code);
-    final_touch(stp);
+    final_touch(stp, inst_p);
 
     /*
      * Loading succeded.
@@ -820,7 +837,6 @@ erts_finish_loading(Binary* magic, Process* c_p,
 	retval = am_on_load;
     }
 
- load_error:
     free_loader_state(magic);
     return retval;
 }
@@ -1026,9 +1042,9 @@ loader_state_dtor(Binary* magic)
 }
 
 static Eterm
-insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
-		Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr,
-		Uint size)
+stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
+		     Eterm group_leader, Eterm module,
+		     BeamCodeHeader* code_hdr, Uint size)
 {
     Module* modp;
     Eterm retval;
@@ -4700,14 +4716,13 @@ freeze_code(LoaderState* stp)
 }
 
 static void
-final_touch(LoaderState* stp)
+final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 {
     int i;
     int on_load = stp->on_load;
     unsigned catches;
     Uint index;
     BeamInstr* codev = stp->codev;
-    Module* modp;
 
     /*
      * Allocate catch indices and fix up all catch_yf instructions.
@@ -4722,8 +4737,7 @@ final_touch(LoaderState* stp)
 	codev[index+2] = make_catch(catches);
 	index = next;
     }
-    modp = erts_put_module(stp->module);
-    modp->curr.catches = catches;
+    inst_p->catches = catches;
 
     /*
      * Export functions.
@@ -6421,7 +6435,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Insert the module in the module table.
      */
 
-    rval = insert_new_code(p, 0, p->group_leader, Mod, code_hdr, code_size);
+    rval = stub_insert_new_code(p, 0, p->group_leader, Mod,
+				code_hdr, code_size);
     if (rval != NIL) {
 	goto error;
     }
-- 
cgit v1.2.3


From 74eafcdaff43d7d84264110066c52fa01e65f159 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 2 May 2016 17:36:33 +0200
Subject: erts: Change argument order for LTTng driver_stop

* Be consistent.
---
 erts/emulator/beam/erlang_lttng.h | 6 +++---
 erts/emulator/beam/io.c           | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index 43ceeda671..f7d751252d 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -218,13 +218,13 @@ TRACEPOINT_EVENT(
     driver_stop,
     TP_ARGS(
         char*, pid,
-        char*, driver,
-        char*, port
+        char*, port,
+        char*, driver
     ),
     TP_FIELDS(
         ctf_string(pid, pid)
-        ctf_string(driver, driver)
         ctf_string(port, port)
+        ctf_string(driver, driver)
     )
 )
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b14ca77a04..5c2595c69d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3861,7 +3861,7 @@ terminate_port(Port *prt)
             lttng_decl_procbuf(proc_str);
             lttng_pid_to_str(connected_id, proc_str);
             lttng_port_to_str(prt, port_str);
-            LTTNG3(driver_stop, proc_str, drv->name, port_str);
+            LTTNG3(driver_stop, proc_str, port_str, drv->name);
         }
 #endif
 
-- 
cgit v1.2.3


From 95194afd63545c7d60e31b5148584f0707034347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 2 May 2016 17:44:25 +0200
Subject: erts: Rename LTTng tracepoint aio_pool_add to aio_pool_put

* Be consistent.
---
 erts/emulator/beam/erl_async.c    | 4 ++--
 erts/emulator/beam/erlang_lttng.h | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index 4d6d699d9f..84254af0c2 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -283,10 +283,10 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
 
     erts_thr_q_enqueue(&q->thr_q, a);
 #ifdef USE_LTTNG_VM_TRACEPOINTS
-    if (LTTNG_ENABLED(aio_pool_add)) {
+    if (LTTNG_ENABLED(aio_pool_put)) {
         lttng_decl_portbuf(port_str);
         lttng_portid_to_str(a->port, port_str);
-        LTTNG2(aio_pool_add, port_str, -1);
+        LTTNG2(aio_pool_put, port_str, -1);
     }
 #endif
 #ifdef USE_VM_PROBES
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index f7d751252d..12f68e477b 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -324,7 +324,7 @@ TRACEPOINT_EVENT(
 
 TRACEPOINT_EVENT(
     com_ericsson_otp,
-    aio_pool_add,
+    aio_pool_put,
     TP_ARGS(
         char*, port,
         int, length
-- 
cgit v1.2.3


From f77ea1c049c585db93618991eb02d9afbfa7ad21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 12 Apr 2016 07:14:57 +0200
Subject: Reimplement -on_load()

Load the module as old code; swap old and new code if the
-on_load function succeeds. That way, a failed update attempt
for a module that has an -on_load function will preserve the
previous version of the code.
---
 erts/emulator/beam/beam_bif_load.c | 63 ++++++++++++++++++++++----------------
 erts/emulator/beam/beam_load.c     | 58 +++++++++++++++++++++++++++++------
 erts/emulator/beam/erl_nif.c       | 62 +++++++++++++++++++++----------------
 3 files changed, 122 insertions(+), 61 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 87508dcf5f..b6f5d66166 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -625,8 +625,8 @@ BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1)
 {
     Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
 
-    if (modp && modp->curr.code_hdr) {
-	BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code_hdr->on_load_function_ptr);
+    if (modp && modp->old.code_hdr) {
+	BIF_TRAP_CODE_PTR_0(BIF_P, modp->old.code_hdr->on_load_function_ptr);
     }
     else {
 	BIF_ERROR(BIF_P, BADARG);
@@ -651,14 +651,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
     code_ix = erts_active_code_ix();
     modp = erts_get_module(BIF_ARG_1, code_ix);
 
-    if (!modp || !modp->curr.code_hdr) {
+    if (!modp || !modp->old.code_hdr) {
     error:
 	erts_smp_thr_progress_unblock();
         erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_release_code_write_permission();
 	BIF_ERROR(BIF_P, BADARG);
     }
-    if (modp->curr.code_hdr->on_load_function_ptr == NULL) {
+    if (modp->old.code_hdr->on_load_function_ptr == NULL) {
 	goto error;
     }
     if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) {
@@ -667,44 +667,55 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 
     if (BIF_ARG_2 == am_true) {
 	int i;
+	struct erl_module_instance t;
+
+	/*
+	 * Swap old and new code.
+	 */
+	t = modp->curr;
+	modp->curr = modp->old;
+	modp->old = t;
 
 	/*
 	 * The on_load function succeded. Fix up export entries.
 	 */
 	for (i = 0; i < export_list_size(code_ix); i++) {
 	    Export *ep = export_list(i,code_ix);
-	    if (ep != NULL &&
-		ep->code[0] == BIF_ARG_1 &&
-		ep->code[4] != 0) {
+	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
+		continue;
+	    }
+	    if (ep->code[4] != 0) {
 		ep->addressv[code_ix] = (void *) ep->code[4];
 		ep->code[4] = 0;
+	    } else {
+		if (ep->addressv[code_ix] == ep->code+3 &&
+		    ep->code[3] == (BeamInstr) em_apply_bif) {
+		    continue;
+		}
+		ep->addressv[code_ix] = ep->code+3;
+		ep->code[3] = (BeamInstr) em_call_error_handler;
 	    }
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
 	set_default_trace_pattern(BIF_ARG_1);
     } else if (BIF_ARG_2 == am_false) {
-	BeamInstr* code;
-	BeamInstr* end;
+	int i;
 
 	/*
-	 * The on_load function failed. Remove the loaded code.
-	 * This is an combination of delete and purge. We purge
-	 * the current code; the old code is not touched.
+	 * The on_load function failed. Remove references to the
+	 * code that is about to be purged from the export entries.
 	 */
-	erts_total_code_size -= modp->curr.code_length;
-	code = (BeamInstr*) modp->curr.code_hdr;
-	end = (BeamInstr *) ((char *)code + modp->curr.code_length);
-	erts_cleanup_funs_on_purge(code, end);
-	beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length,
-			    erts_active_code_ix());
-        if (modp->curr.code_hdr->literals_start) {
-            erts_free(ERTS_ALC_T_LITERAL, modp->curr.code_hdr->literals_start);
-        }
-	erts_free(ERTS_ALC_T_CODE, modp->curr.code_hdr);
-	modp->curr.code_hdr = NULL;
-	modp->curr.code_length = 0;
-	modp->curr.catches = BEAM_CATCHES_NIL;
-	erts_remove_from_ranges(code);
+
+	for (i = 0; i < export_list_size(code_ix); i++) {
+	    Export *ep = export_list(i,code_ix);
+	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
+		continue;
+	    }
+	    if (ep->code[3] == (BeamInstr) em_apply_bif) {
+		continue;
+	    }
+	    ep->code[4] = 0;
+	}
     }
     erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9bd9b4bfc0..95489d9a63 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -32,6 +32,7 @@
 #include "bif.h"
 #include "external.h"
 #include "beam_load.h"
+#include "beam_bp.h"
 #include "big.h"
 #include "erl_bits.h"
 #include "beam_catches.h"
@@ -766,7 +767,7 @@ Eterm
 erts_finish_loading(Binary* magic, Process* c_p,
 		    ErtsProcLocks c_p_locks, Eterm* modp)
 {
-    Eterm retval;
+    Eterm retval = NIL;
     LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
     Module* mod_tab_p;
     struct erl_module_instance* inst_p;
@@ -779,16 +780,52 @@ erts_finish_loading(Binary* magic, Process* c_p,
 
     ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() ||
 		       erts_smp_thr_progress_is_blocking());
-
     /*
      * Make current code for the module old and insert the new code
      * as current.  This will fail if there already exists old code
      * for the module.
      */
 
+    mod_tab_p = erts_put_module(stp->module);
     CHKBLK(ERTS_ALC_T_CODE,stp->code);
-    retval = beam_make_current_old(c_p, c_p_locks, stp->module);
-    ASSERT(retval == NIL);
+    if (!stp->on_load) {
+	/*
+	 * Normal case -- no -on_load() function.
+	 */
+	retval = beam_make_current_old(c_p, c_p_locks, stp->module);
+	ASSERT(retval == NIL);
+    } else {
+	ErtsCodeIndex code_ix = erts_staging_code_ix();
+	Eterm module = stp->module;
+	int i;
+
+	/*
+	 * There is an -on_load() function. We will keep the current
+	 * code, but we must turn off any tracing.
+	 */
+
+	for (i = 0; i < export_list_size(code_ix); i++) {
+	    Export *ep = export_list(i, code_ix);
+	    if (ep == NULL || ep->code[0] != module) {
+		continue;
+	    }
+	    if (ep->addressv[code_ix] == ep->code+3) {
+		if (ep->code[3] == (BeamInstr) em_apply_bif) {
+		    continue;
+		} else if (ep->code[3] ==
+			   (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+		    ASSERT(mod_tab_p->curr.num_traced_exports > 0);
+		    erts_clear_export_break(mod_tab_p, ep->code+3);
+		    ep->addressv[code_ix] = (BeamInstr *) ep->code[4];
+		    ep->code[4] = 0;
+		}
+		ASSERT(ep->code[4] == 0);
+	    }
+	}
+	ASSERT(mod_tab_p->curr.num_breakpoints == 0);
+	ASSERT(mod_tab_p->curr.num_traced_exports == 0);
+    }
 
     /*
      * Update module table.
@@ -796,8 +833,11 @@ erts_finish_loading(Binary* magic, Process* c_p,
 
     size = stp->loaded_size;
     erts_total_code_size += size;
-    mod_tab_p = erts_put_module(stp->module);
-    inst_p = &mod_tab_p->curr;
+    if (stp->on_load) {
+	inst_p = &mod_tab_p->old;
+    } else {
+	inst_p = &mod_tab_p->curr;
+    }
     inst_p->code_hdr = stp->hdr;
     inst_p->code_length = size;
 
@@ -4757,10 +4797,10 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	    ep->addressv[erts_staging_code_ix()] = address;
 	} else {
 	    /*
-	     * Don't make any of the exported functions
-	     * callable yet.
+	     * on_load: Don't make any of the exported functions
+	     * callable yet. Keep any function in the current
+	     * code callable.
 	     */
-	    ep->addressv[erts_staging_code_ix()] = ep->code+3;
 	    ep->code[4] = (BeamInstr) address;
 	}
     }
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 941f44b9ec..dc83a780a2 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2780,7 +2780,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     ErlNifEntry* entry = NULL;
     ErlNifEnv env;
     int i, err, encoding;
-    Module* mod;
+    Module* module_p;
     Eterm mod_atom;
     const Atom* mod_atomp;
     Eterm f_atom;
@@ -2790,6 +2790,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     int veto;
     struct erl_module_nif* lib = NULL;
     int reload_warning = 0;
+    struct erl_module_instance* this_mi;
+    struct erl_module_instance* prev_mi;
 
     encoding = erts_get_native_filename_encoding();
     if (encoding == ERL_FILENAME_WIN_WCHAR) {
@@ -2823,21 +2825,29 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     ASSERT(caller != NULL);
     mod_atom = caller[0];
     ASSERT(is_atom(mod_atom));
-    mod=erts_get_module(mod_atom, erts_active_code_ix());
-    ASSERT(mod != NULL);
+    module_p = erts_get_module(mod_atom, erts_active_code_ix());
+    ASSERT(module_p != NULL);
 
     mod_atomp = atom_tab(atom_val(mod_atom));
     init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len);
     if (init_func != NULL)
       handle = init_func;
 
-    if (!in_area(caller, mod->curr.code_hdr, mod->curr.code_length)) {
-	ASSERT(in_area(caller, mod->old.code_hdr, mod->old.code_length));
+    if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) {
+	if (module_p->old.code_hdr->on_load_function_ptr) {
+	    this_mi = &module_p->old;
+	    prev_mi = &module_p->curr;
+	} else {
+	    ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old "
+				 "module '%T' not allowed", mod_atom);
+	    goto error;
+	}
+    } else {
+	this_mi = &module_p->curr;
+	prev_mi = &module_p->old;
+    }
 
-	ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old "
-			     "module '%T' not allowed", mod_atom);
-    }    
-    else if (init_func == NULL &&
+    if (init_func == NULL &&
 	     (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
 	const char slogan[] = "Failed to load NIF library";
 	if (strstr(errdesc.str, lib_name) != NULL) {
@@ -2886,7 +2896,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
 	    BeamInstr** code_pp;
 	    if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1)
-		|| (code_pp = get_func_pp(mod->curr.code_hdr, f_atom, f->arity))==NULL) {
+		|| (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) {
 		ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u",
 				     mod_atom, f->name, f->arity);
 	    }
@@ -2937,9 +2947,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     erts_refc_init(&lib->rt_cnt, 0);
     erts_refc_init(&lib->rt_dtor_cnt, 0);
     ASSERT(opened_rt_list == NULL);
-    lib->mod = mod;
+    lib->mod = module_p;
     env.mod_nif = lib;
-    if (mod->curr.nif != NULL) { /*************** Reload ******************/
+    if (this_mi->nif != NULL) { /*************** Reload ******************/
 	/*
 	 * Repeated load_nif calls from same Erlang module instance ("reload")
 	 * is deprecated and was only ment as a development feature not to
@@ -2947,16 +2957,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	 */
 	int k, old_incr = 0;
 	ErlNifFunc* old_func;
-        lib->priv_data = mod->curr.nif->priv_data;
+        lib->priv_data = this_mi->nif->priv_data;
 
-	ASSERT(mod->curr.nif->entry != NULL);
+	ASSERT(this_mi->nif->entry != NULL);
 	if (entry->reload == NULL) {
 	    ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library.");
 	    goto error;
 	}
 	/* Check that no NIF is removed */
-	old_func = mod->curr.nif->entry->funcs;
-	for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) {
+	old_func = this_mi->nif->entry->funcs;
+	for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) {
 	    int incr = 0;
 	    ErlNifFunc* f = entry->funcs;
 	    for (i=0; i < entry->num_of_funcs; i++) {
@@ -2972,7 +2982,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 				     old_func->name, old_func->arity);
 		goto error;
 	    }
-	    old_func = next_func(mod->curr.nif->entry, &old_incr, old_func);
+	    old_func = next_func(this_mi->nif->entry, &old_incr, old_func);
 	}
 	erts_pre_nif(&env, BIF_P, lib, NULL);
 	veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2);
@@ -2982,24 +2992,24 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	}
 	else {
 	    commit_opened_resource_types(lib);
-	    mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */
-	    erts_unload_nif(mod->curr.nif);
+	    this_mi->nif->entry = NULL; /* to prevent 'unload' callback */
+	    erts_unload_nif(this_mi->nif);
 	    reload_warning = 1;
 	}
     }
     else {
 	lib->priv_data = NULL;
-	if (mod->old.nif != NULL) { /**************** Upgrade ***************/
-	    void* prev_old_data = mod->old.nif->priv_data;
+	if (prev_mi->nif != NULL) { /**************** Upgrade ***************/
+	    void* prev_old_data = prev_mi->nif->priv_data;
 	    if (entry->upgrade == NULL) {
 		ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library.");
 		goto error;
 	    }
-	    erts_pre_nif(&env, BIF_P, lib, NULL);
-	    veto = entry->upgrade(&env, &lib->priv_data, &mod->old.nif->priv_data, BIF_ARG_2);
+		erts_pre_nif(&env, BIF_P, lib, NULL);
+	    veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2);
 	    erts_post_nif(&env);
 	    if (veto) {
-		mod->old.nif->priv_data = prev_old_data;
+		prev_mi->nif->priv_data = prev_old_data;
 		ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful.");
 	    }
 	    else
@@ -3024,12 +3034,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	int incr = 0;
 	ErlNifFunc* f = entry->funcs;
 
-	mod->curr.nif = lib;
+	this_mi->nif = lib;
 	for (i=0; i < entry->num_of_funcs; i++)
 	{
 	    BeamInstr* code_ptr;
 	    erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1);
-	    code_ptr = *get_func_pp(mod->curr.code_hdr, f_atom, f->arity);
+	    code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity);
 
 	    if (code_ptr[1] == 0) {
 		code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif);
-- 
cgit v1.2.3


From 19932cade31a7973120eba8db9a3b57b925f674a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 27 Apr 2016 10:36:10 +0200
Subject: Ensure correct reduction counting

---
 erts/emulator/beam/beam_bif_load.c | 12 ++---
 erts/emulator/beam/beam_emu.c      | 94 +++++++++++++++++++++++++++++++-------
 erts/emulator/beam/bif.c           | 48 +++++++++++++++----
 erts/emulator/beam/bif.h           |  1 +
 erts/emulator/beam/erl_gc.c        | 40 ++++++++++------
 erts/emulator/beam/erl_gc.h        |  2 +-
 erts/emulator/beam/erl_init.c      | 10 ++--
 erts/emulator/beam/erl_process.c   | 62 +++++++++++++++++--------
 erts/emulator/beam/erl_process.h   | 19 ++++++++
 erts/emulator/beam/erl_trace.c     | 22 +++++----
 erts/emulator/beam/global.h        |  2 +-
 11 files changed, 231 insertions(+), 81 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 87508dcf5f..801218774b 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -38,7 +38,7 @@
 #include "erl_thr_progress.h"
 
 static void set_default_trace_pattern(Eterm module);
-static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp);
+static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls);
 static void delete_code(Module* modp);
 static void decrement_refc(BeamCodeHeader*);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
@@ -467,7 +467,7 @@ check_old_code_1(BIF_ALIST_1)
 }
 
 Eterm
-erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp)
+erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls)
 {
     Module* modp;
     Eterm res;
@@ -483,7 +483,7 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp)
 	return am_false;
     erts_rlock_old_code(code_ix);
     res = (!modp->old.code_hdr ? am_false :
-           check_process_code(c_p, modp, flags, redsp));
+           check_process_code(c_p, modp, flags, redsp, fcalls));
     erts_runlock_old_code(code_ix);
 
     return res;
@@ -506,7 +506,7 @@ BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2)
         goto badarg;
     }
 
-    res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds);
+    res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls);
 
     ASSERT(is_value(res));
 
@@ -753,7 +753,7 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size)
 
 
 static Eterm
-check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
+check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls)
 {
     BeamInstr* start;
     char* literals;
@@ -955,7 +955,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
 
 	if (need_gc & ERTS_ORDINARY_GC__) {
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
-	    *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity);
+	    *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls);
 	    done_gc |= ERTS_ORDINARY_GC__;
 	}
 	if (need_gc & ERTS_LITERAL_GC__) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 72526bca5e..59112e1e43 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -308,7 +308,8 @@ void** beam_ops;
      if (E - HTOP < (needed + (HeapNeed))) { \
            SWAPOUT; \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
-           FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \
+           FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \
+						 reg, (M), FCALLS);	\
            ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            SWAPIN; \
@@ -360,7 +361,7 @@ void** beam_ops;
     if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
        SWAPOUT;                                                 		\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));  		\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS);	\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        SWAPIN;                                                  		\
@@ -381,7 +382,7 @@ void** beam_ops;
     if (E - HTOP < need) {                                      \
        SWAPOUT;                                                 \
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        SWAPIN;                                                  \
@@ -402,7 +403,7 @@ void** beam_ops;
        SWAPOUT;								\
        reg[Live] = Extra;						\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1);	\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        Extra = reg[Live];						\
@@ -1196,6 +1197,25 @@ init_emulator(void)
 #define DTRACE_NIF_RETURN(p, m, f, a)        do {} while (0)
 #endif /* USE_VM_PROBES */
 
+#ifdef DEBUG
+#define ERTS_DBG_CHK_REDS(P, FC)					\
+    do {								\
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) {			\
+	    ASSERT(FC <= 0);						\
+	    ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds		\
+		   <= 0 - (FC));					\
+	}								\
+	else {								\
+	    ASSERT(FC <= CONTEXT_REDS);					\
+	    ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds		\
+		   <= CONTEXT_REDS - (FC));				\
+	}								\
+} while (0)
+#else
+#define ERTS_DBG_CHK_REDS(P, FC)
+#endif
+
+
 /*
  * process_main() is called twice:
  * The first call performs some initialisation, including exporting
@@ -1290,7 +1310,12 @@ void process_main(void)
     goto do_schedule1;
 
  do_schedule:
-    reds_used = REDS_IN(c_p) - FCALLS;
+    ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	reds_used = REDS_IN(c_p) - FCALLS;
+    else
+	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+    ASSERT(reds_used >= 0);
  do_schedule1:
 
     if (start_time != 0) {
@@ -1310,6 +1335,7 @@ void process_main(void)
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     c_p = schedule(c_p, reds_used);
+    ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     start_time = 0;
 #ifdef DEBUG
@@ -1348,16 +1374,21 @@ void process_main(void)
 
 	SET_I(c_p->i);
 
-	reds = c_p->fcalls;
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
-	    && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
-	    neg_o_reds = -reds;
-	    FCALLS = REDS_IN(c_p) = 0;
+	REDS_IN(c_p) = reds = c_p->fcalls;
+#ifdef DEBUG
+	c_p->debug_reds_in = reds;
+#endif
+
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+	    neg_o_reds = -CONTEXT_REDS;
+	    FCALLS = neg_o_reds + reds;
 	} else {
 	    neg_o_reds = 0;
-	    FCALLS = REDS_IN(c_p) = reds;
+	    FCALLS = reds;
 	}
 
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
 	next = (BeamInstr *) *I;
 	SWAPIN;
 	ASSERT(VALID_INSTR(next));
@@ -1770,7 +1801,7 @@ void process_main(void)
 	     if (E - HTOP < 3) {
 		 SWAPOUT;
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
-		 FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1);
+		 FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS);
 		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 SWAPIN;
@@ -1865,6 +1896,7 @@ void process_main(void)
      c_p->flags |= F_DELAY_GC;
 
  loop_rec__:
+
      PROCESS_MAIN_CHK_LOCKS(c_p);
 
      msgp = PEEK_MESSAGE(c_p);
@@ -2007,6 +2039,8 @@ void process_main(void)
 	 ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
      }
 
+     ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
      ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
      NextPF(0, next);
@@ -2535,6 +2569,7 @@ do {						\
 
 	GetArg1(2, tmp_reg[0]);
 	bf = (BifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2544,6 +2579,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(3, result);
 	}
@@ -2564,6 +2600,7 @@ do {						\
 
 	GetArg1(1, tmp_reg[0]);
 	bf = (BifFunction) Arg(0);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2573,6 +2610,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(2, result);
 	}
@@ -2591,6 +2629,7 @@ do {						\
 
 	GetArg1(2, x(live));
 	bf = (GcBifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2602,6 +2641,7 @@ do {						\
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(4, result);
 	}
@@ -2630,6 +2670,7 @@ do {						\
 	 */
 	live++;
 	bf = (GcBifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2641,6 +2682,7 @@ do {						\
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(5, result);
 	}
@@ -2671,6 +2713,7 @@ do {						\
 	 */
 	live += 2;
 	bf = (GcBifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2682,6 +2725,7 @@ do {						\
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(5, result);
 	}
@@ -2708,6 +2752,7 @@ do {						\
 
 	GetArg2(2, tmp_reg[0], tmp_reg[1]);
 	bf = (BifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2717,6 +2762,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(4, result);
 	}
@@ -2775,6 +2821,7 @@ do {						\
 	bf = GET_BIF_ADDRESS(Arg(0));
 
 	PRE_BIF_SWAPOUT(c_p);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS - 1;
 	if (FCALLS <= 0) {
 	   save_calls(c_p, (Export *) Arg(0));
@@ -2796,6 +2843,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	HTOP = HEAP_TOP(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
         /* We have to update the cache if we are enabled in order
            to make sure no book keeping is done after we disabled
            msacc. We don't always do this as it is quite expensive. */
@@ -3324,8 +3372,13 @@ do {						\
       * (beacuse the code for the Dispatch() macro becomes shorter that way).
       */
 
-     reds_used = REDS_IN(c_p) - FCALLS + 1;
-     
+     ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	reds_used = REDS_IN(c_p) - FCALLS;
+    else
+	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+    ASSERT(reds_used >= 0);
+
      /*
       * Save the argument registers and everything else.
       */
@@ -3514,6 +3567,7 @@ do {						\
 	    DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 
 	    SWAPOUT;
+	    ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
 	    c_p->fcalls = FCALLS - 1;
 	    vbf = (BifFunction) Arg(0);
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -3549,6 +3603,7 @@ do {						\
 	    }
 	    SWAPIN;  /* There might have been a garbage collection. */
 	    FCALLS = c_p->fcalls;
+	    ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	    if (is_value(nif_bif_result)) {
 		r(0) = nif_bif_result;
 		CHECK_TERM(r(0));
@@ -4611,9 +4666,9 @@ do {						\
  OpCase(i_generic_breakpoint): {
      BeamInstr real_I;
      ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      real_I = erts_generic_breakpoint(c_p, I, reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      ASSERT(VALID_INSTR(real_I));
      Goto(real_I);
  }
@@ -4790,6 +4845,7 @@ do {						\
  {
 #define HIPE_MODE_SWITCH(Cmd)			\
      SWAPOUT;					\
+     ERTS_DBG_CHK_REDS(c_p, FCALLS);		\
      c_p->fcalls = FCALLS;			\
      c_p->def_arg_reg[4] = -neg_o_reds;		\
      c_p = hipe_mode_switch(c_p, Cmd, reg);     \
@@ -4828,6 +4884,9 @@ do {						\
 #undef HIPE_MODE_SWITCH
 
  L_post_hipe_mode_switch:
+#ifdef DEBUG
+     pid = c_p->common.id; /* may have switched process... */
+#endif
      reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
      freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
      ERL_BITS_RELOAD_STATEP(c_p);
@@ -4835,6 +4894,7 @@ do {						\
      neg_o_reds = -c_p->def_arg_reg[4];
      FCALLS = c_p->fcalls;
      SWAPIN;
+     ERTS_DBG_CHK_REDS(c_p, FCALLS);
      switch( c_p->def_arg_reg[3] ) {
        case HIPE_MODE_SWITCH_RES_RETURN:
 	 ASSERT(is_value(reg[0]));
@@ -6839,7 +6899,7 @@ erts_current_reductions(Process *current, Process *p)
     if (current != p) {
 	return 0;
     } else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) {
-	return -current->fcalls;
+	return current->fcalls + CONTEXT_REDS;
     } else {
 	return REDS_IN(current) - current->fcalls;
     }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ed5b2983dd..fa5c51ef88 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1569,7 +1569,32 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
 	   scb->n = 0;
        }
 
-       scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
+#ifdef HIPE
+       if (rp->flags & F_HIPE_MODE) {
+	   ASSERT(!ERTS_PROC_GET_SAVED_CALLS_BUF(rp));
+	   scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(rp, scb);
+       }
+       else
+#endif
+       {
+#ifdef HIPE
+	   ASSERT(!ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(rp));
+#endif
+	   scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
+	   if (rp == BIF_P && ((scb && i == 0) || (!scb && i != 0))) {
+	       /* Adjust fcalls to match save calls setting... */
+	       if (i == 0)
+		   BIF_P->fcalls += CONTEXT_REDS; /* disabled it */
+	       else
+		   BIF_P->fcalls -= CONTEXT_REDS; /* enabled it */
+
+	       /*
+		* Make sure we reschedule immediately so the
+		* change take effect at once.
+		*/
+	       ERTS_VBUMP_ALL_REDS(BIF_P);
+	   }
+       }
 
        if (!scb)
 	   old_value = make_small(0);
@@ -1578,12 +1603,7 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
 	   erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
        }
 
-       /* Make sure the process in question is rescheduled
-	  immediately, if it's us, so the call saving takes effect. */
-       if (rp == BIF_P)
-	   BIF_RET2(old_value, CONTEXT_REDS);
-       else
-	   BIF_RET(old_value);
+       BIF_RET(old_value);
    }
 
  error:
@@ -1775,10 +1795,18 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3)
    Process *rp;
    Eterm res;
 
-   if ((rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-			   BIF_ARG_1, ERTS_PROC_LOCK_MAIN)) == NULL) {
+#ifdef ERTS_SMP
+   rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
+				  BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
+   if (rp == ERTS_PROC_LOCK_BUSY)
+       ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P,
+		       BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+#else
+   rp = erts_proc_lookup(BIF_ARG_1);
+#endif
+
+   if (!rp)
        BIF_ERROR(BIF_P, BADARG);
-   }
 
    res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3);
 
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 546e3830b9..5d751dd67d 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -52,6 +52,7 @@ extern Export *erts_convert_time_unit_trap;
 	(p)->fcalls = 0; 			\
     else 					\
 	(p)->fcalls = -CONTEXT_REDS;		\
+    ASSERT(ERTS_BIF_REDS_LEFT((p)) == 0);	\
 } while(0)
 
 #define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls)				\
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 4698458521..5a180d7e48 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -116,7 +116,7 @@ static Eterm *full_sweep_heaps(Process *p,
 			       char *oh, Uint oh_size,
 			       Eterm *objv, int nobj);
 static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-			   int need, Eterm* objv, int nobj);
+			   int need, Eterm* objv, int nobj, int fcalls);
 static int major_collection(Process* p, ErlHeapFragment *live_hf_end,
 			    int need, Eterm* objv, int nobj, Uint *recl);
 static int minor_collection(Process* p, ErlHeapFragment *live_hf_end,
@@ -392,15 +392,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 		regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array;
 	    }
 	  #endif
-	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls);
 	} else {
-	    cost = garbage_collect(p, live_hf_end, 0, regs, arity);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls);
 	}
     } else {
 	Eterm val[1];
 
 	val[0] = result;
-	cost = garbage_collect(p, live_hf_end, 0, val, 1);
+	cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls);
 	result = val[0];
     }
     BUMP_REDS(p, cost);
@@ -431,7 +431,7 @@ static ERTS_INLINE void reset_active_writer(Process *p)
 #define ERTS_ABANDON_HEAP_COST 10
 
 static int
-delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
+delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int fcalls)
 {
     ErlHeapFragment *hfrag;
     Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
@@ -506,12 +506,16 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
 
     /* Make sure that we do a proper GC as soon as possible... */
     p->flags |= F_FORCE_GC;
-    reds_left = ERTS_BIF_REDS_LEFT(p);
+    reds_left = ERTS_REDS_LEFT(p, fcalls);
+    ASSERT(CONTEXT_REDS - reds_left >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+
     if (reds_left > ERTS_ABANDON_HEAP_COST) {
 	int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
-	ERTS_VBUMP_REDS(p, vreds);
+	ERTS_PROC_GET_SCHDATA((p))->virtual_reds += vreds;
     }
-    return ERTS_ABANDON_HEAP_COST;
+
+    ASSERT(CONTEXT_REDS >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+    return reds_left;
 }
 
 static ERTS_FORCE_INLINE Uint
@@ -570,7 +574,7 @@ young_gen_usage(Process *p)
  */
 static int
 garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-		int need, Eterm* objv, int nobj)
+		int need, Eterm* objv, int nobj, int fcalls)
 {
     Uint reclaimed_now = 0;
     int reds;
@@ -581,8 +585,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
+    ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
+	   >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+
     if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
-	return delay_garbage_collection(p, live_hf_end, need);
+	return delay_garbage_collection(p, live_hf_end, need, fcalls);
 
     if (p->abandoned_heap)
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -700,16 +707,23 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 }
 
 int
-erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj)
+erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
 {
-    return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls);
+    int reds_left = ERTS_REDS_LEFT(p, fcalls);
+    if (reds > reds_left)
+	reds = reds_left;
+    ASSERT(CONTEXT_REDS - (reds_left - reds) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+    return reds;
 }
 
 void
 erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls);
     BUMP_REDS(p, reds);
+    ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
+	   >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
 }
 
 /*
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 40b7c5d12c..8a6ff99990 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -139,7 +139,7 @@ Eterm erts_process_gc_info(struct process*, Uint *, Eterm **);
 
 void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
-int erts_garbage_collect_nobump(struct process*, int, Eterm*, int);
+int erts_garbage_collect_nobump(struct process*, int, Eterm*, int, int);
 void erts_garbage_collect(struct process*, int, Eterm*, int);
 void erts_garbage_collect_hibernate(struct process* p);
 Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index dec9bdfedc..4d27530866 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -206,15 +206,15 @@ int erts_no_line_info = 0;	/* -L: Don't load line information */
 
 ErtsModifiedTimings erts_modified_timings[] = {
     /* 0 */	{make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS},
-    /* 1 */	{make_small(0), 2*CONTEXT_REDS, 2*INPUT_REDUCTIONS},
+    /* 1 */	{make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS},
     /* 2 */	{make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2},
-    /* 3 */	{make_small(0), 3*CONTEXT_REDS, 3*INPUT_REDUCTIONS},
+    /* 3 */	{make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS},
     /* 4 */	{make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS},
-    /* 5 */	{make_small(0), 4*CONTEXT_REDS, INPUT_REDUCTIONS/2},
+    /* 5 */	{make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2},
     /* 6 */	{make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS},
-    /* 7 */	{make_small(1), 5*CONTEXT_REDS, INPUT_REDUCTIONS/3},
+    /* 7 */	{make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3},
     /* 8 */	{make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS},
-    /* 9 */	{make_small(10), 6*CONTEXT_REDS, INPUT_REDUCTIONS/4}
+    /* 9 */	{make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4}
 };
 
 #define ERTS_MODIFIED_TIMING_LEVELS \
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d485affa3b..c6c3fd64a1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9240,6 +9240,21 @@ erts_set_process_priority(Process *p, Eterm value)
     }
 }
 
+static int
+scheduler_gc_proc(Process *c_p, int reds_left)
+{
+    int fcalls, reds;
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	fcalls = reds_left;
+    else
+	fcalls = reds_left - CONTEXT_REDS;
+    reds = erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls);
+    ASSERT(reds_left >= reds);
+    return reds;
+}
+
+
+
 /*
  * schedule() is called from BEAM (process_main()) or HiPE
  * (hipe_mode_switch()) when the current process is to be
@@ -9318,6 +9333,7 @@ Process *schedule(Process *p, int calls)
 #endif
 
 	reds = actual_reds = calls - esdp->virtual_reds;
+	ASSERT(actual_reds >= 0);
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
@@ -9740,7 +9756,7 @@ Process *schedule(Process *p, int calls)
 
 	    esdp->current_process = p;
 
-
+	    calls = 0;
 	    reds = context_reds;
 
 #ifdef ERTS_SMP
@@ -9819,9 +9835,7 @@ Process *schedule(Process *p, int calls)
 		/* Migrate to dirty scheduler... */
 	    sunlock_sched_out_proc:
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-		p->fcalls = reds;
 		goto sched_out_proc;
-
 	    }
 	}
 	else {
@@ -9856,8 +9870,6 @@ Process *schedule(Process *p, int calls)
 
 #endif /* ERTS_SMP */
 
-	p->fcalls = reds;
-
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
 	if (IS_TRACED(p)) {
@@ -9890,14 +9902,15 @@ Process *schedule(Process *p, int calls)
 	     * not allowed to execute system tasks.
 	     */
 	    if (!(p->flags & F_DELAY_GC)) {
-		reds -= execute_sys_tasks(p, &state, reds);
+		int cost = execute_sys_tasks(p, &state, reds);
+		calls += cost;
+		reds -= cost;
 		if (reds <= 0
 #ifdef ERTS_DIRTY_SCHEDULERS
 		    || ERTS_SCHEDULER_IS_DIRTY(esdp)
 		    || (state & ERTS_PSFLGS_DIRTY_WORK)
 #endif
 		    ) {
-		    p->fcalls = reds;
 		    goto sched_out_proc;
 		}
 	    }
@@ -9910,8 +9923,9 @@ Process *schedule(Process *p, int calls)
 
 		if (((state & (ERTS_PSFLG_SUSPENDED
 			       | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
-		    && !(state & ERTS_PSFLG_EXITING))
+		    && !(state & ERTS_PSFLG_EXITING)) {
 		    goto sched_out_proc;
+		}
 
 		n = e = state;
 		n &= ~ERTS_PSFLG_RUNNING_SYS;
@@ -9930,11 +9944,11 @@ Process *schedule(Process *p, int calls)
 
 	if (ERTS_IS_GC_DESIRED(p)) {
 	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
-		reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity);
-		if (reds <= 0) {
-		    p->fcalls = reds;
+		int cost = scheduler_gc_proc(p, reds);
+		calls += cost;
+		reds -= cost;
+		if (reds <= 0)
 		    goto sched_out_proc;
-		}
 	    }
 	}
 
@@ -9942,6 +9956,8 @@ Process *schedule(Process *p, int calls)
 	    free_proxy_proc(proxy_p);
 	    proxy_p = NULL;
 	}
+
+	p->fcalls = reds;
 	    
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
@@ -10211,21 +10227,24 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    else {
 		if (!garbage_collected) {
 		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
-		    reds -= erts_garbage_collect_nobump(c_p,
-							0,
-							c_p->arg_reg,
-							c_p->arity);
+		    reds -= scheduler_gc_proc(c_p, reds);
 		    garbage_collected = 1;
 		}
 		st_res = am_true;
 	    }
 	    break;
 	case ERTS_PSTT_CPC: {
+	    int fcalls;
             int cpc_reds = 0;
+	    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+		fcalls = reds;
+	    else
+		fcalls = reds - CONTEXT_REDS;
 	    st_res = erts_check_process_code(c_p,
 					     st->arg[0],
                                              unsigned_val(st->arg[1]),
-					     &cpc_reds);
+					     &cpc_reds,
+					     fcalls);
             reds -= cpc_reds;
 	    if (is_non_value(st_res)) {
 		/* Needed gc, but gc was disabled */
@@ -10257,6 +10276,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 
     *statep = state;
 
+    if (in_reds < reds)
+	return in_reds;
+
     return in_reds - reds;
 }
 
@@ -11482,8 +11504,10 @@ delete_process(Process* p)
 
     psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
 
-    if (psd)
+    if (psd) {
+	erts_smp_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */
 	erts_free(ERTS_ALC_T_PSD, psd);
+    }
 
     /* Clean binaries and funs */
     erts_cleanup_offheap(&p->off_heap);
@@ -12577,6 +12601,8 @@ erts_continue_exit_process(Process *p)
     
     dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
     scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
+    if (scb)
+	p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */
     pbt = ERTS_PROC_SET_CALL_TIME(p, NULL);
     nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 61acf5924b..8c7dfc8d2f 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -805,8 +805,15 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_CALL_TIME_BP			3
 #define ERTS_PSD_DELAYED_GC_TASK_QS		4
 #define ERTS_PSD_NIF_TRAP_EXPORT		5
+#ifdef HIPE
+#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	6
+#endif
 
+#ifdef HIPE
+#define ERTS_PSD_SIZE				7
+#else
 #define ERTS_PSD_SIZE				6
+#endif
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -1071,6 +1078,10 @@ struct process {
     Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
     Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
 #endif
+
+#ifdef DEBUG
+    Uint debug_reds_in;
+#endif
 };
 
 extern const Process erts_invalid_process;
@@ -1342,6 +1353,7 @@ extern int erts_system_profile_ts_type;
 #define F_DELAY_GC           (1 << 16) /* Similar to disable GC (see below) */
 #define F_SCHDLR_ONLN_WAITQ  (1 << 17) /* Process enqueued waiting to change schedulers online */
 #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
+#define F_HIPE_MODE          (1 << 19)
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1999,6 +2011,13 @@ erts_psd_set(Process *p, int ix, void *data)
 #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \
     erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
 
+#ifdef HIPE
+#define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \
+  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF))
+#define ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(P, SCB) \
+  ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
+#endif
+
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index bd88769dfc..00e7cb23ba 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1282,16 +1282,18 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
 
 void save_calls(Process *p, Export *e)
 {
-    struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
-    if (scb) {
-	Export **ct = &scb->ct[0];
-	int len = scb->len;
-
-	ct[scb->cur] = e;
-	if (++scb->cur >= len)
-	    scb->cur = 0;
-	if (scb->n < len)
-	    scb->n++;
+    if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) {
+	struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
+	if (scb) {
+	    Export **ct = &scb->ct[0];
+	    int len = scb->len;
+
+	    ct[scb->cur] = e;
+	    if (++scb->cur >= len)
+		scb->cur = 0;
+	    if (scb->n < len)
+		scb->n++;
+	}
     }
 }
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index a7bc990deb..49eec44053 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -997,7 +997,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 #define ERTS_CPC_ALLOW_GC      (1 << 0)
 #define ERTS_CPC_COPY_LITERALS (1 << 1)
 #define ERTS_CPC_ALL           (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS)
-Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp);
+Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls);
 
 typedef struct {
     Eterm *ptr;
-- 
cgit v1.2.3


From dd5087bf582bba0879fa352bdc0ddcbf06d943d2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 4 May 2016 17:44:07 +0200
Subject: erts: Add macro HAVE_ERTS_MMAP

and make erts_mmap unavailable at compile time
if not supported.
---
 erts/emulator/beam/erl_alloc.c    | 23 +++++++++++++++--------
 erts/emulator/beam/erl_bif_info.c |  2 +-
 2 files changed, 16 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 01db65d539..f571515def 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2963,10 +2963,11 @@ erts_allocator_options(void *proc)
     atoms[length] = am_atom_put("alloc_util", 10); 
     terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp);
 
-
+#if HAVE_ERTS_MMAP
     atoms[length] = ERTS_MAKE_AM("erts_mmap");
     terms[length++] = erts_mmap_info_options(&erts_dflt_mmapper, NULL, NULL,
                                              NULL, hpp, szp);
+#endif
     {
 	Eterm o[3], v[3];
 	o[0] = am_atom_put("m", 1);
@@ -3093,12 +3094,14 @@ reply_alloc_info(void *vair)
     Uint sz, *szp;
     ErlOffHeap *ohp = NULL;
     ErtsMessage *mp = NULL;
+#if HAVE_ERTS_MMAP
     struct erts_mmap_info_struct mmap_info_dflt;
-#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     struct erts_mmap_info_struct mmap_info_literal;
-#endif
-#ifdef ERTS_ALC_A_EXEC
+# endif
+# ifdef ERTS_ALC_A_EXEC
     struct erts_mmap_info_struct mmap_info_exec;
+# endif
 #endif
     int i;
     Eterm (*info_func)(Allctr_t *,
@@ -3207,7 +3210,7 @@ reply_alloc_info(void *vair)
 		    break;
                 case ERTS_ALC_INFO_A_ERTS_MMAP:
                     alloc_atom = erts_bld_atom(hpp, szp, "erts_mmap");
-
+#if HAVE_ERTS_MMAP
                     ainfo = (air->only_sz ? NIL :
                              erts_mmap_info(&erts_dflt_mmapper, NULL, NULL,
                                             hpp, szp, &mmap_info_dflt));
@@ -3215,7 +3218,7 @@ reply_alloc_info(void *vair)
                                             alloc_atom,
                                             erts_bld_atom(hpp,szp,"default_mmap"),
                                             ainfo);
-#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+#  if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
                     ai_list = erts_bld_cons(hpp, szp,
                                             ainfo, ai_list);
                     ainfo = (air->only_sz ? NIL :
@@ -3225,8 +3228,8 @@ reply_alloc_info(void *vair)
                                             alloc_atom,
                                             erts_bld_atom(hpp,szp,"literal_mmap"),
                                             ainfo);
-#endif
-#ifdef ERTS_ALC_A_EXEC
+#  endif
+#  ifdef ERTS_ALC_A_EXEC
                     ai_list = erts_bld_cons(hpp, szp,
                                             ainfo, ai_list);
                     ainfo = (air->only_sz ? NIL :
@@ -3236,6 +3239,10 @@ reply_alloc_info(void *vair)
                                             alloc_atom,
                                             erts_bld_atom(hpp,szp,"exec_mmap"),
                                             ainfo);
+#  endif
+#else  /* !HAVE_ERTS_MMAP */
+                    ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
+                                            am_false);
 #endif
                     break;
 		case ERTS_ALC_INFO_A_MSEG_ALLOC:
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index c9741b361f..95d1fef3d3 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3554,7 +3554,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    BIF_RET(res);
 	}
         else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) {
-            BIF_RET(erts_mmap_debug_info(&erts_dflt_mmapper, BIF_P));
+            BIF_RET(erts_mmap_debug_info(BIF_P));
         }
 	else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) {
 	    BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P));
-- 
cgit v1.2.3


From 03afaae12811632cf975586728826af14a634ad1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 22 Jan 2016 20:12:40 +0100
Subject: erts: Beautify some code with container_of macro

---
 erts/emulator/beam/erl_bif_trace.c | 11 ++++-------
 erts/emulator/beam/sys.h           |  3 +++
 2 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index ff2018aa27..09c82ad206 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1307,7 +1307,7 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 
     for (i = 0; i < n; i++) {
 	BeamInstr* pc = fp[i].pc;
-	Export* ep = (Export *)(((char *)(pc-3)) - offsetof(Export, code));
+	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
 
 	if (on && !flags.breakpoint) {
 	    /* Turn on global call tracing */
@@ -1556,11 +1556,10 @@ install_exp_breakpoints(BpFunctions* f)
     BpFunction* fp = f->matching;
     Uint ne = f->matched;
     Uint i;
-    Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr);
 
     for (i = 0; i < ne; i++) {
 	BeamInstr* pc = fp[i].pc;
-	Export* ep = (Export *) (((char *)pc)-offset);
+	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
 
 	ep->addressv[code_ix] = pc;
     }
@@ -1573,11 +1572,10 @@ uninstall_exp_breakpoints(BpFunctions* f)
     BpFunction* fp = f->matching;
     Uint ne = f->matched;
     Uint i;
-    Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr);
 
     for (i = 0; i < ne; i++) {
 	BeamInstr* pc = fp[i].pc;
-	Export* ep = (Export *) (((char *)pc)-offset);
+	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
 
 	if (ep->addressv[code_ix] != pc) {
 	    continue;
@@ -1594,11 +1592,10 @@ clean_export_entries(BpFunctions* f)
     BpFunction* fp = f->matching;
     Uint ne = f->matched;
     Uint i;
-    Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr);
 
     for (i = 0; i < ne; i++) {
 	BeamInstr* pc = fp[i].pc;
-	Export* ep = (Export *) (((char *)pc)-offset);
+	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
 
 	if (ep->addressv[code_ix] == pc) {
 	    continue;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 748fba15c7..dda3ba565c 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -92,6 +92,9 @@
 #define ErtsInArea(ptr,start,nbytes) \
     ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
+#define ErtsContainerStruct(ptr, type, member) \
+  (type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))
+
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
 #else
-- 
cgit v1.2.3


From af2a4c5a40c62da775caa92df5164fbee08b9245 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 2 Feb 2016 17:30:51 +0100
Subject: erts: Rename constants in enum erts_break_op

with uppercase for constants
and why not call them 'RESTART' and 'PAUSE' as the API.
---
 erts/emulator/beam/beam_bp.c       | 6 +++---
 erts/emulator/beam/beam_bp.h       | 8 ++++----
 erts/emulator/beam/erl_bif_trace.c | 4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 2ee98ed7b5..c4b9259d64 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1432,7 +1432,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
     g = (GenericBp *) pc[-4];
     if (g == 0) {
 	int i;
-	if (count_op == erts_break_reset || count_op == erts_break_stop) {
+	if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) {
 	    /* Do not insert a new breakpoint */
 	    return;
 	}
@@ -1456,7 +1456,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	MatchSetUnref(bp->meta_ms);
 	bp_meta_unref(bp->meta_tracer);
     } else if (common & ERTS_BPF_COUNT) {
-	if (count_op == erts_break_stop) {
+	if (count_op == ERTS_BREAK_PAUSE) {
 	    bp->flags &= ~ERTS_BPF_COUNT_ACTIVE;
 	} else {
 	    bp->flags |= ERTS_BPF_COUNT_ACTIVE;
@@ -1468,7 +1468,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	BpDataTime* bdt = bp->time;
 	Uint i = 0;
 
-	if (count_op == erts_break_stop) {
+	if (count_op == ERTS_BREAK_PAUSE) {
 	    bp->flags &= ~ERTS_BPF_TIME_TRACE_ACTIVE;
 	} else {
 	    bp->flags |= ERTS_BPF_TIME_TRACE_ACTIVE;
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index bb171be8a6..fc17f95b5d 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -86,10 +86,10 @@ typedef struct generic_bp {
 #endif
 
 enum erts_break_op{
-    erts_break_nop   =  0, /* Must be false */
-    erts_break_set   = !0, /* Must be true */
-    erts_break_reset,
-    erts_break_stop
+    ERTS_BREAK_NOP   =  0, /* Must be false */
+    ERTS_BREAK_SET   = !0, /* Must be true */
+    ERTS_BREAK_RESTART,
+    ERTS_BREAK_PAUSE
 };
 
 typedef Uint32 ErtsBpIndex;
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 09c82ad206..ef01aa3340 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -137,10 +137,10 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	on = 1;
     } else if (Pattern == am_restart) {
 	match_prog_set = NULL;
-	on = erts_break_reset;
+	on = ERTS_BREAK_RESTART;
     } else if (Pattern == am_pause) {
 	match_prog_set = NULL;
-	on = erts_break_stop;
+	on = ERTS_BREAK_PAUSE;
     } else if ((match_prog_set = erts_match_set_compile(p, Pattern)) != NULL) {
 	MatchSetRef(match_prog_set);
 	on = 1;
-- 
cgit v1.2.3


From 4a4475dea3ab0daf29f9a49ce845fa062387362a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 3 Mar 2016 19:34:50 +0100
Subject: erts: Add match spec for send trace

---
 erts/emulator/beam/beam_bp.c       |  5 +++-
 erts/emulator/beam/erl_bif_trace.c | 57 ++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_trace.c     | 59 +++++++++++++++++++++++++++-----------
 erts/emulator/beam/erl_trace.h     |  8 ++++++
 4 files changed, 111 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index c4b9259d64..1e30e8d8d1 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -248,7 +248,10 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified)
 void
 erts_bp_free_matched_functions(BpFunctions* f)
 {
-    Free(f->matching);
+    if (f->matching) {
+	Free(f->matching);
+    }
+    else ASSERT(f->matched == 0);
 }
 
 void
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index ef01aa3340..6b5bb2f889 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -68,6 +68,9 @@ static struct {			/* Protected by code write permission */
 
 static Eterm
 trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist);
+static void
+erts_set_trace_send_pattern(Process*, Binary*, int on);
+
 #ifdef ERTS_SMP
 static void smp_bp_finisher(void* arg);
 #endif
@@ -85,14 +88,23 @@ static void install_exp_breakpoints(BpFunctions* f);
 static void uninstall_exp_breakpoints(BpFunctions* f);
 static void clean_export_entries(BpFunctions* f);
 
+ErtsTracingEvent erts_send_tracing[ERTS_NUM_BP_IX];
+
 void
 erts_bif_trace_init(void)
 {
+    int i;
+
     erts_default_trace_pattern_is_on = 0;
     erts_default_match_spec = NULL;
     erts_default_meta_match_spec = NULL;
     erts_default_trace_pattern_flags = erts_trace_pattern_flags_off;
     erts_default_meta_tracer = erts_tracer_nil;
+
+    for (i=0; i<ERTS_NUM_BP_IX; i++) {
+        erts_send_tracing[i].on = 1;
+        erts_send_tracing[i].match_spec = NULL;
+    }
 }
 
 /*
@@ -314,8 +326,15 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	matches = erts_set_trace_pattern(p, mfa, specified,
 					 match_prog_set, match_prog_set,
 					 on, flags, meta_tracer, 0);
+    } else if (MFA == am_send) {
+        if (is_global || flags.breakpoint || on > ERTS_BREAK_SET) {
+            goto error;
+        }
+        erts_set_trace_send_pattern(p, match_prog_set, on);
+        matches = 1;
     }
 
+
  error:
     MatchSetUnref(match_prog_set);
     UnUseTmpHeap(3,p);
@@ -1467,6 +1486,43 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
     return matches;
 }
 
+void
+erts_set_trace_send_pattern(Process*p, Binary* match_spec, int on)
+{
+    ErtsTracingEvent* st = &erts_send_tracing[erts_staging_bp_ix()];
+
+    MatchSetUnref(st->match_spec);
+
+    st->on = on;
+    st->match_spec = match_spec;
+    MatchSetRef(match_spec);
+
+    finish_bp.current = 1;  /* prepare phase not needed for send trace */
+    finish_bp.install = on;
+    finish_bp.e.matched = 0;
+    finish_bp.e.matching = NULL;
+    finish_bp.f.matched = 0;
+    finish_bp.f.matching = NULL;
+
+#ifndef ERTS_SMP
+    while (erts_finish_breakpointing()) {
+        /* Empty loop body */
+    }
+#endif
+}
+
+static void
+consolidate_send_tracing(void)
+{
+    ErtsTracingEvent* src = &erts_send_tracing[erts_active_bp_ix()];
+    ErtsTracingEvent* dst = &erts_send_tracing[erts_staging_bp_ix()];
+
+    MatchSetUnref(dst->match_spec);
+    dst->on = src->on;
+    dst->match_spec = src->match_spec;
+    MatchSetRef(dst->match_spec);
+}
+
 int
 erts_finish_breakpointing(void)
 {
@@ -1542,6 +1598,7 @@ erts_finish_breakpointing(void)
 	erts_consolidate_bp_data(&finish_bp.f, 1);
 	erts_bp_free_matched_functions(&finish_bp.e);
 	erts_bp_free_matched_functions(&finish_bp.f);
+        consolidate_send_tracing();
 	return 0;
     default:
 	ASSERT(0);
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 56899f574a..e8516cd09c 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -394,7 +394,8 @@ static ERTS_INLINE int
 send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
                    Eterm t_p_id, ErtsTracerNif *tnif,
                    enum ErtsTracerOpt topt,
-                   Eterm tag, Eterm msg, Eterm extra);
+                   Eterm tag, Eterm msg, Eterm extra,
+                   Eterm pam_result);
 static ERTS_INLINE Eterm
 call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
                     ErtsTracerNif **tnif_ref,
@@ -786,7 +787,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
     }
 
     send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SCHED_PROC,
-                       what, tmp, THE_NON_VALUE);
+                       what, tmp, THE_NON_VALUE, am_true);
 }
 
 /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
@@ -811,9 +812,31 @@ trace_send(Process *p, Eterm to, Eterm msg)
 {
     Eterm operation = am_send;
     ErtsTracerNif *tnif = NULL;
+    ErtsTracingEvent* te;
+    Eterm pam_result;
 
     ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND));
 
+    te = &erts_send_tracing[erts_active_bp_ix()];
+    if (!te->on) {
+	return;
+    }
+    if (te->match_spec) {
+	Eterm args[2];
+	Uint32 return_flags;
+	args[0] = to;
+	args[1] = msg;
+	pam_result = erts_match_set_run(p, te->match_spec, args, 2,
+					ERTS_PAM_TMP_RESULT, &return_flags);
+	if (is_non_value(pam_result)
+	    || pam_result == am_false
+	    || (ERTS_TRACE_FLAGS(p) & F_TRACE_SILENT)) {
+            erts_match_set_release_result(p);
+	    return;
+	}
+    } else
+        pam_result = am_true;
+
     if (is_internal_pid(to)) {
 	if (!erts_proc_lookup(to))
 	    goto send_to_non_existing_process;
@@ -825,9 +848,11 @@ trace_send(Process *p, Eterm to, Eterm msg)
     }
 
     if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif,
-                TRACE_FUN_E_SEND, operation))
+                          TRACE_FUN_E_SEND, operation)) {
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND,
-                operation, msg, to);
+                operation, msg, to, pam_result);
+    }
+    erts_match_set_release_result(p);
 }
 
 /* Send {trace_ts, Pid, receive, Msg, Timestamp}
@@ -841,7 +866,7 @@ trace_receive(Process *c_p, Eterm msg)
                 TRACE_FUN_E_RECEIVE, am_receive))
         send_to_tracer_nif(NULL, &c_p->common, c_p->common.id,
                            tnif, TRACE_FUN_T_RECEIVE,
-                           am_receive, msg, THE_NON_VALUE);
+                           am_receive, msg, THE_NON_VALUE, am_true);
 }
 
 int
@@ -963,7 +988,7 @@ erts_trace_return_to(Process *p, BeamInstr *pc)
     }
 
     send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL,
-                       am_return_to, mfa, THE_NON_VALUE);
+                       am_return_to, mfa, THE_NON_VALUE, am_true);
 }
 
 
@@ -1289,7 +1314,7 @@ trace_proc(Process *c_p, ErtsProcLocks c_p_locks,
     if (is_tracer_enabled(c_p, c_p_locks, &t_p->common, &tnif,
                 TRACE_FUN_E_PROCS, what))
         send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS,
-                           what, data, THE_NON_VALUE);
+                           what, data, THE_NON_VALUE, am_true);
 }
 
 
@@ -1315,7 +1340,7 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
         mfa = TUPLE3(hp, mod, func, args);
         hp += 4;
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS,
-                           what, pid, mfa);
+                           what, pid, mfa, am_true);
     }
 }
 
@@ -1365,7 +1390,7 @@ trace_gc(Process *p, Eterm what, Uint size)
         msg = CONS(hp, tup, msg); hp += 2;
 
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC,
-                what, msg, am_undefined);
+                what, msg, am_undefined, am_true);
     }
 }
 
@@ -1780,7 +1805,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &p->common, &tnif, TRACE_FUN_E_PORTS, am_open))
         send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PORTS,
-                am_open, calling_pid, drv_name);
+                am_open, calling_pid, drv_name, am_true);
 }
 
 /* Sends trace message:
@@ -1799,7 +1824,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_PORTS, what))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PORTS,
-                           what, data, THE_NON_VALUE);
+                           what, data, THE_NON_VALUE, am_true);
 }
 
 
@@ -1936,7 +1961,7 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
 
         data = TUPLE2(hp, caller, data);
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_RECEIVE,
-                           am_receive, data, THE_NON_VALUE);
+                           am_receive, data, THE_NON_VALUE, am_true);
 
         if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
             erts_bin_free(bptr);
@@ -1959,7 +1984,7 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists)
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, op))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND,
-                           op, msg, receiver);
+                           op, msg, receiver, am_true);
 }
 
 void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
@@ -1988,7 +2013,7 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
         hp += 3;
 
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND,
-                           am_send, msg, to);
+                           am_send, msg, to, am_true);
         if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
             erts_bin_free(bptr);
 
@@ -2019,7 +2044,7 @@ trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) {
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SCHED_PORT, what))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
                            tnif, TRACE_FUN_T_SCHED_PORT,
-                           what, where, THE_NON_VALUE);
+                           what, where, THE_NON_VALUE, am_true);
 }
 
 /* Port profiling */
@@ -2842,7 +2867,7 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee,
 static ERTS_INLINE int
 send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
                    Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt topt,
-                   Eterm tag, Eterm msg, Eterm extra)
+                   Eterm tag, Eterm msg, Eterm extra, Eterm pam_result)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     if (c_p) {
@@ -2862,7 +2887,7 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
                                   is_internal_pid(t_p->id) ? (Process*)t_p : NULL,
                                   t_p->tracer, t_p->trace_flags,
                                   t_p_id, tnif, topt, tag, msg, extra,
-                                  am_true);
+                                  pam_result);
 }
 
 static ERTS_INLINE Eterm
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 9a007e62ec..e8e968c0cf 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -220,4 +220,12 @@ ERTS_DECLARE_DUMMY(erts_tracer_nil) = NIL;
      && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN,          \
                                     &(PROC)->common, am_trace_status))
 
+typedef struct
+{
+    int on;
+    struct binary* match_spec;
+} ErtsTracingEvent;
+
+extern ErtsTracingEvent erts_send_tracing[];
+
 #endif /* ERL_TRACE_H__ */
-- 
cgit v1.2.3


From 3261d580be2263b7b4b70ec3c7670a5de963e91c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 4 Mar 2016 19:25:38 +0100
Subject: erts: Remove multi scheduler blocking in match specs

for enable_trace and disable_trace operations.

Instead seize needed locks while updating trace flags.
---
 erts/emulator/beam/erl_db_util.c | 51 +++++++++++++---------------------------
 1 file changed, 16 insertions(+), 35 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 76b96637ae..a1f458038e 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1758,7 +1758,6 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
     Eterm (*bif)(Process*, ...);
     Eterm bif_args[3];
     int fail_label;
-    int atomic_trace;
 #ifdef DMC_DEBUG
     Uint *heap_fence;
     Uint *stack_fence;
@@ -1776,28 +1775,6 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
     current_scheduled = esdp->current_process;
     /* SMP: psp->scheduler_data is set by get_match_pseudo_process */
 
-    atomic_trace = 0;
-#define BEGIN_ATOMIC_TRACE(p)                               \
-    do {                                                    \
-	if (! atomic_trace) {                               \
-            erts_refc_inc(&bprog->refc, 2);                 \
-	    erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \
-	    erts_smp_thr_progress_block();                  \
-            atomic_trace = !0;                              \
-	}                                                   \
-    } while (0)
-#define END_ATOMIC_TRACE(p)                               \
-    do {                                                  \
-	if (atomic_trace) {                               \
-            erts_smp_thr_progress_unblock();              \
-            erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \
-            if (erts_refc_dectest(&bprog->refc, 0) == 0) {\
-                erts_bin_free(bprog);                     \
-	    }                                             \
-            atomic_trace = 0;                             \
-	}                                                 \
-    } while (0)
-
 #ifdef DMC_DEBUG
     save_op = 0;
     heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1;
@@ -2334,8 +2311,9 @@ restart:
 	    break;
 	case matchEnableTrace:
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
-		BEGIN_ATOMIC_TRACE(c_p);
+                erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n);
+                erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		esp[-1] = am_true;
 	    } else {
 		esp[-1] = FAIL_TERM;
@@ -2345,18 +2323,22 @@ restart:
 	    n = erts_trace_flag2bit((--esp)[-1]);
 	    esp[-1] = FAIL_TERM;
 	    if (n) {
-		BEGIN_ATOMIC_TRACE(c_p);
-		if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) {
+		if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) {
 		    /* Always take over the tracer of the current process */
 		    set_tracee_flags(tmpp, ERTS_TRACER(c_p), 0, n);
-		    esp[-1] = am_true;
+                    if (tmpp == c_p)
+                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR);
+                    else
+                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
+                    esp[-1] = am_true;
 		}
 	    }
 	    break;
 	case matchDisableTrace:
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
-		BEGIN_ATOMIC_TRACE(c_p);
+                erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0);
+                erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		esp[-1] = am_true;
 	    } else {
 		esp[-1] = FAIL_TERM;
@@ -2366,11 +2348,14 @@ restart:
 	    n = erts_trace_flag2bit((--esp)[-1]);
 	    esp[-1] = FAIL_TERM;
 	    if (n) {
-		BEGIN_ATOMIC_TRACE(c_p);
-		if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) {
+		if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) {
 		    /* Always take over the tracer of the current process */
 		    set_tracee_flags(tmpp, ERTS_TRACER(c_p), n, 0);
-		    esp[-1] = am_true;
+                    if (tmpp == c_p)
+                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR);
+                    else
+                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
+                    esp[-1] = am_true;
 		}
 	    }
 	    break;
@@ -2508,13 +2493,9 @@ success:
 
     esdp->current_process = current_scheduled;
 
-    END_ATOMIC_TRACE(c_p);
-
     return ret;
 #undef FAIL
 #undef FAIL_TERM
-#undef BEGIN_ATOMIC_TRACE
-#undef END_ATOMIC_TRACE
 }
 
 
-- 
cgit v1.2.3


From 8b2906d9974decf9e8bab24a8f753ba81a025410 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Mar 2016 19:53:42 +0100
Subject: erts: Add matchspec to 'receive' trace

---
 erts/emulator/beam/beam_emu.c      |  2 +-
 erts/emulator/beam/erl_bif_trace.c | 40 +++++++++++++++---------
 erts/emulator/beam/erl_db.c        |  3 +-
 erts/emulator/beam/erl_db_util.c   | 62 ++++++++++++++++++++++++++------------
 erts/emulator/beam/erl_db_util.h   |  3 +-
 erts/emulator/beam/erl_message.c   | 37 +++++++++++++----------
 erts/emulator/beam/erl_trace.c     | 48 +++++++++++++++++++++++------
 erts/emulator/beam/erl_trace.h     | 19 ++++++------
 erts/emulator/beam/global.h        |  4 ++-
 9 files changed, 145 insertions(+), 73 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 72526bca5e..7eecd059a5 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2166,7 +2166,7 @@ void process_main(void)
 
      PreFetch(0, next);
      if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
-	 trace_receive(c_p, am_timeout);
+	 trace_receive(c_p, c_p, am_timeout, NULL);
      }
      if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
 	 save_calls(c_p, &exp_timeout);
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 6b5bb2f889..7b21ff0110 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -68,8 +68,8 @@ static struct {			/* Protected by code write permission */
 
 static Eterm
 trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist);
-static void
-erts_set_trace_send_pattern(Process*, Binary*, int on);
+static int
+erts_set_tracing_event_pattern(Eterm event, Binary*, int on);
 
 #ifdef ERTS_SMP
 static void smp_bp_finisher(void* arg);
@@ -89,6 +89,7 @@ static void uninstall_exp_breakpoints(BpFunctions* f);
 static void clean_export_entries(BpFunctions* f);
 
 ErtsTracingEvent erts_send_tracing[ERTS_NUM_BP_IX];
+ErtsTracingEvent erts_receive_tracing[ERTS_NUM_BP_IX];
 
 void
 erts_bif_trace_init(void)
@@ -104,6 +105,8 @@ erts_bif_trace_init(void)
     for (i=0; i<ERTS_NUM_BP_IX; i++) {
         erts_send_tracing[i].on = 1;
         erts_send_tracing[i].match_spec = NULL;
+	erts_receive_tracing[i].on = 1;
+	erts_receive_tracing[i].match_spec = NULL;
     }
 }
 
@@ -326,15 +329,13 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	matches = erts_set_trace_pattern(p, mfa, specified,
 					 match_prog_set, match_prog_set,
 					 on, flags, meta_tracer, 0);
-    } else if (MFA == am_send) {
+    } else if (is_atom(MFA)) {
         if (is_global || flags.breakpoint || on > ERTS_BREAK_SET) {
             goto error;
         }
-        erts_set_trace_send_pattern(p, match_prog_set, on);
-        matches = 1;
+        matches = erts_set_tracing_event_pattern(MFA, match_prog_set, on);
     }
 
-
  error:
     MatchSetUnref(match_prog_set);
     UnUseTmpHeap(3,p);
@@ -1486,10 +1487,17 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
     return matches;
 }
 
-void
-erts_set_trace_send_pattern(Process*p, Binary* match_spec, int on)
+int
+erts_set_tracing_event_pattern(Eterm event, Binary* match_spec, int on)
 {
-    ErtsTracingEvent* st = &erts_send_tracing[erts_staging_bp_ix()];
+    ErtsBpIndex ix = erts_staging_bp_ix();
+    ErtsTracingEvent* st;
+
+    switch (event) {
+    case am_send: st = &erts_send_tracing[ix]; break;
+    case am_receive: st = &erts_receive_tracing[ix]; break;
+    default: return -1;
+    }
 
     MatchSetUnref(st->match_spec);
 
@@ -1497,7 +1505,7 @@ erts_set_trace_send_pattern(Process*p, Binary* match_spec, int on)
     st->match_spec = match_spec;
     MatchSetRef(match_spec);
 
-    finish_bp.current = 1;  /* prepare phase not needed for send trace */
+    finish_bp.current = 1;  /* prepare phase not needed for event trace */
     finish_bp.install = on;
     finish_bp.e.matched = 0;
     finish_bp.e.matching = NULL;
@@ -1509,13 +1517,14 @@ erts_set_trace_send_pattern(Process*p, Binary* match_spec, int on)
         /* Empty loop body */
     }
 #endif
+    return 1;
 }
 
 static void
-consolidate_send_tracing(void)
+consolidate_event_tracing(ErtsTracingEvent te[])
 {
-    ErtsTracingEvent* src = &erts_send_tracing[erts_active_bp_ix()];
-    ErtsTracingEvent* dst = &erts_send_tracing[erts_staging_bp_ix()];
+    ErtsTracingEvent* src = &te[erts_active_bp_ix()];
+    ErtsTracingEvent* dst = &te[erts_staging_bp_ix()];
 
     MatchSetUnref(dst->match_spec);
     dst->on = src->on;
@@ -1529,7 +1538,7 @@ erts_finish_breakpointing(void)
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
 
     /*
-     * Memory barriers will be issued for all processes *before*
+     * Memory barriers will be issued for all schedulers *before*
      * each of the stages below. (Unless the other schedulers
      * are blocked, in which case memory barriers will be issued
      * when they are awaken.)
@@ -1598,7 +1607,8 @@ erts_finish_breakpointing(void)
 	erts_consolidate_bp_data(&finish_bp.f, 1);
 	erts_bp_free_matched_functions(&finish_bp.e);
 	erts_bp_free_matched_functions(&finish_bp.f);
-        consolidate_send_tracing();
+        consolidate_event_tracing(erts_send_tracing);
+	consolidate_event_tracing(erts_receive_tracing);
 	return 0;
     default:
 	ASSERT(0);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 615d23402b..acaca54e9a 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2833,7 +2833,8 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
 	    BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3],
 		      BIF_P,lst,BIF_ARG_2,ret);
 	}
-	res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0,
+	res = db_prog_match(BIF_P, BIF_P,
+                            mp, CAR(list_val(lst)), NULL, 0,
 			    ERTS_PAM_COPY_RESULT, &dummy);
 	if (is_value(res)) {
 	    hp = HAlloc(BIF_P, 2);
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index a1f458038e..72d59df813 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1202,14 +1202,16 @@ done:
     return ret;
 }
     
-Eterm erts_match_set_run(Process *p, Binary *mpsp, 
+Eterm erts_match_set_run(Process *c_p,
+                         Process *self,
+                         Binary *mpsp,
 			 Eterm *args, int num_args,
 			 enum erts_pam_run_flags in_flags,
 			 Uint32 *return_flags) 
 {
     Eterm ret;
 
-    ret = db_prog_match(p, mpsp, NIL, args, num_args,
+    ret = db_prog_match(c_p, self, mpsp, NIL, args, num_args,
 			in_flags, return_flags);
 #if defined(HARDDEBUG)
     if (is_non_value(ret)) {
@@ -1234,7 +1236,8 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp,
 {
     Eterm ret;
 
-    ret = db_prog_match(p, mpsp, args, NULL, num_args,
+    ret = db_prog_match(p, p,
+                        mpsp, args, NULL, num_args,
 			ERTS_PAM_COPY_RESULT,
 			return_flags);
 #if defined(HARDDEBUG)
@@ -1730,7 +1733,9 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
 ** the parameter 'arity' is only used if 'term' is actually an array,
 ** i.e. 'DCOMP_TRACE' was specified 
 */
-Eterm db_prog_match(Process *c_p, Binary *bprog,
+Eterm db_prog_match(Process *c_p,
+                    Process *self,
+                    Binary *bprog,
 		    Eterm term,
 		    Eterm *termp,
 		    int arity,
@@ -1746,7 +1751,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
     UWord *pc = prog->text;
     Eterm *ehp;
     Eterm ret;
-    Uint n = 0; /* To avoid warning. */
+    Uint n;
     int i;
     unsigned do_catch;
     ErtsMatchPseudoProcess *mpsp;
@@ -1764,6 +1769,8 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
     Uint save_op;
 #endif /* DMC_DEBUG */
 
+    ERTS_UNDEF(n,0);
+
     mpsp = get_match_pseudo_process(c_p, prog->heap_size);
     psp = &mpsp->process;
 
@@ -2233,7 +2240,7 @@ restart:
 	    pc += n;
 	    break;
 	case matchSelf:
-	    *esp++ = c_p->common.id;
+	    *esp++ = self->common.id;
 	    break;
 	case matchWaste:
 	    --esp;
@@ -2243,6 +2250,7 @@ restart:
 	    break;
 	case matchProcessDump: {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
+            ASSERT(c_p == self);
 	    print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p);
 	    *esp++ = new_binary(build_proc, (byte *)dsbufp->str,
 				dsbufp->str_len);
@@ -2261,14 +2269,16 @@ restart:
 	    *return_flags |= MATCH_SET_EXCEPTION_TRACE;
 	    *esp++ = am_true;
 	    break;
-	case matchIsSeqTrace:
+        case matchIsSeqTrace:
+            ASSERT(c_p == self);
             if (have_seqtrace(SEQ_TRACE_TOKEN(c_p)))
 		*esp++ = am_true;
 	    else
 		*esp++ = am_false;
 	    break;
 	case matchSetSeqToken:
-	    t = erts_seq_trace(c_p, esp[-1], esp[-2], 0);
+            ASSERT(c_p == self);
+            t = erts_seq_trace(c_p, esp[-1], esp[-2], 0);
 	    if (is_non_value(t)) {
 		esp[-2] = FAIL_TERM;
 	    } else {
@@ -2276,7 +2286,8 @@ restart:
 	    }
 	    --esp;
 	    break;
-	case matchSetSeqTokenFake:
+        case matchSetSeqTokenFake:
+            ASSERT(c_p == self);
 	    t = seq_trace_fake(c_p, esp[-1]);
 	    if (is_non_value(t)) {
 		esp[-2] = FAIL_TERM;
@@ -2285,7 +2296,8 @@ restart:
 	    }
 	    --esp;
 	    break;
-	case matchGetSeqToken:
+        case matchGetSeqToken:
+            ASSERT(c_p == self);
             if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p)))
 		*esp++ = NIL;
 	    else {
@@ -2309,7 +2321,8 @@ restart:
 		ASSERT(is_immed(ehp[5]));
 	    } 
 	    break;
-	case matchEnableTrace:
+        case matchEnableTrace:
+            ASSERT(c_p == self);
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
                 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n);
@@ -2319,7 +2332,8 @@ restart:
 		esp[-1] = FAIL_TERM;
 	    }
 	    break;
-	case matchEnableTrace2:
+        case matchEnableTrace2:
+            ASSERT(c_p == self);
 	    n = erts_trace_flag2bit((--esp)[-1]);
 	    esp[-1] = FAIL_TERM;
 	    if (n) {
@@ -2334,7 +2348,8 @@ restart:
 		}
 	    }
 	    break;
-	case matchDisableTrace:
+        case matchDisableTrace:
+            ASSERT(c_p == self);
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
                 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0);
@@ -2344,7 +2359,8 @@ restart:
 		esp[-1] = FAIL_TERM;
 	    }
 	    break;
-	case matchDisableTrace2:
+        case matchDisableTrace2:
+            ASSERT(c_p == self);
 	    n = erts_trace_flag2bit((--esp)[-1]);
 	    esp[-1] = FAIL_TERM;
 	    if (n) {
@@ -2359,7 +2375,8 @@ restart:
 		}
 	    }
 	    break;
- 	case matchCaller:
+        case matchCaller:
+            ASSERT(c_p == self);
 	    if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) {
  		*esp++ = am_undefined;
  	    } else {
@@ -2371,7 +2388,8 @@ restart:
 		ehp[3] = make_small((Uint) cp[2]);
 	    }
 	    break;
-	case matchSilent:
+        case matchSilent:
+            ASSERT(c_p == self);
 	    --esp;
 	    if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT)
 	      break;
@@ -2386,7 +2404,8 @@ restart:
 		erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	    }
 	    break;
-	case matchTrace2:
+        case matchTrace2:
+            ASSERT(c_p == self);
 	    {
 		/*    disable         enable                                */
 		Uint  d_flags  = 0,   e_flags  = 0;  /* process trace flags */
@@ -2418,7 +2437,8 @@ restart:
                 ERTS_TRACER_CLEAR(&tracer);
 	    }
 	    break;
-	case matchTrace3:
+        case matchTrace3:
+            ASSERT(c_p == self);
 	    {
 		/*    disable         enable                                */
 		Uint  d_flags  = 0,   e_flags  = 0;  /* process trace flags */
@@ -5080,7 +5100,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 	    }
 	    save_cp = p->cp;
 	    p->cp = NULL;
-	    res = erts_match_set_run(p, mps, arr, n,
+	    res = erts_match_set_run(p, p,
+                      mps, arr, n,
 		      ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT,
 		      &ret_flags);
 	    p->cp = save_cp;
@@ -5163,7 +5184,8 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
 	obj = db_alloc_tmp_uncompressed(tb, obj);
     }
 
-    res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0,
+    res = db_prog_match(c_p, c_p,
+                        bprog, make_tuple(obj->tpl), NULL, 0,
 			ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
 
     if (is_value(res) && hpp!=NULL) {
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 5d7946a525..c3eb82a44a 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -435,7 +435,8 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards,
 Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
 		      int all, DbTerm* obj, Eterm** hpp, Uint extra);
 
-Eterm db_prog_match(Process *p, Binary *prog, Eterm term,
+Eterm db_prog_match(Process *p, Process *self,
+                    Binary *prog, Eterm term,
 		    Eterm *termp, int arity,
 		    enum erts_pam_run_flags in_flags,
 		    Uint32 *return_flags /* Zeroed on enter */);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 9beff52835..ee8a7702ce 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -32,6 +32,7 @@
 #include "erl_process.h"
 #include "erl_binary.h"
 #include "dtrace-wrapper.h"
+#include "beam_bp.h"
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref,
 				 ErtsMessageRef,
@@ -369,11 +370,12 @@ static Sint
 queue_messages(Process *c_p,
                Process* receiver,
                erts_aint32_t *receiver_state,
-               ErtsProcLocks *receiver_locks,
+               ErtsProcLocks receiver_locks,
                ErtsMessage* first,
                ErtsMessage** last,
                Uint len)
 {
+    ErtsTracingEvent* te;
     Sint res;
     int locked_msgq = 0;
     erts_aint32_t state;
@@ -386,12 +388,12 @@ queue_messages(Process *c_p,
 #ifdef ERTS_SMP
 #ifdef ERTS_ENABLE_LOCK_CHECK
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
-                       *receiver_locks == erts_proc_lc_my_proc_locks(receiver));
+                       receiver_locks == erts_proc_lc_my_proc_locks(receiver));
 #endif
 
-    if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
+    if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
 	if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
-            ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
+            ErtsProcLocks need_locks;
 
 	    if (receiver_state)
 		state = *receiver_state;
@@ -400,10 +402,11 @@ queue_messages(Process *c_p,
 	    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
 		goto exiting;
 
-	    if (*receiver_locks & ERTS_PROC_LOCK_STATUS) {
-		erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS);
-		need_locks |= ERTS_PROC_LOCK_STATUS;
+            need_locks = receiver_locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+	    if (need_locks) {
+		erts_smp_proc_unlock(receiver, need_locks);
 	    }
+            need_locks |= ERTS_PROC_LOCK_MSGQ;
 	    erts_smp_proc_lock(receiver, need_locks);
 	}
 	locked_msgq = 1;
@@ -426,7 +429,7 @@ queue_messages(Process *c_p,
 
     res = receiver->msg.len;
 #ifdef ERTS_SMP
-    if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
+    if (receiver_locks & ERTS_PROC_LOCK_MAIN) {
 	/*
 	 * We move 'in queue' to 'private queue' and place
 	 * message at the end of 'private queue' in order
@@ -445,7 +448,10 @@ queue_messages(Process *c_p,
 	LINK_MESSAGE(receiver, first, last, len);
     }
 
-    if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
+    if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)
+        && (te = &erts_receive_tracing[erts_active_bp_ix()],
+            te->on)) {
+
         ErtsMessage *msg = first;
 
 #ifdef USE_VM_PROBES
@@ -468,19 +474,18 @@ queue_messages(Process *c_p,
                     tok_label, tok_lastcnt, tok_serial);
         }
 #endif
-
         while (msg) {
-            trace_receive(receiver, ERL_MESSAGE_TERM(msg));
+            trace_receive(c_p, receiver, ERL_MESSAGE_TERM(msg), te);
             msg = msg->next;
         }
 
     }
-
-    if (locked_msgq)
+    if (locked_msgq) {
 	erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
+    }
 
 #ifdef ERTS_SMP
-    erts_proc_notify_new_message(receiver, *receiver_locks);
+    erts_proc_notify_new_message(receiver, receiver_locks);
 #else
     erts_proc_notify_new_message(receiver, 0);
     ERTS_HOLE_CHECK(receiver);
@@ -496,7 +501,7 @@ queue_message(Process *c_p,
               ErtsMessage* mp, Eterm msg)
 {
     ERL_MESSAGE_TERM(mp) = msg;
-    return queue_messages(c_p, receiver, receiver_state, receiver_locks,
+    return queue_messages(c_p, receiver, receiver_state, *receiver_locks,
                           mp, &mp->next, 1 );
 }
 
@@ -512,7 +517,7 @@ Sint
 erts_queue_messages(Process* receiver, ErtsProcLocks *receiver_locks,
                     ErtsMessage* first, ErtsMessage** last, Uint len)
 {
-    return queue_messages(NULL, receiver, NULL, receiver_locks,
+    return queue_messages(NULL, receiver, NULL, *receiver_locks,
                           first, last, len);
 }
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index e8516cd09c..755deda9c1 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -238,7 +238,6 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp)
 }
 
 #ifdef ERTS_SMP
-#define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p))
 
 static ERTS_INLINE Uint
 patch_ts_size(int ts_type)
@@ -258,7 +257,7 @@ patch_ts_size(int ts_type)
 	return 0;
     }
 }
-#endif
+#endif /* ERTS_SMP */
 
 /*
  * Write a timestamp. The timestamp MUST be the last
@@ -826,7 +825,8 @@ trace_send(Process *p, Eterm to, Eterm msg)
 	Uint32 return_flags;
 	args[0] = to;
 	args[1] = msg;
-	pam_result = erts_match_set_run(p, te->match_spec, args, 2,
+	pam_result = erts_match_set_run(p, p,
+                                        te->match_spec, args, 2,
 					ERTS_PAM_TMP_RESULT, &return_flags);
 	if (is_non_value(pam_result)
 	    || pam_result == am_false
@@ -859,14 +859,43 @@ trace_send(Process *p, Eterm to, Eterm msg)
  * or   {trace, Pid, receive, Msg}
  */
 void
-trace_receive(Process *c_p, Eterm msg)
+trace_receive(Process *c_p,
+              Process* receiver,
+              Eterm msg, ErtsTracingEvent* te)
 {
     ErtsTracerNif *tnif = NULL;
-    if (is_tracer_enabled(NULL, 0, &c_p->common, &tnif,
-                TRACE_FUN_E_RECEIVE, am_receive))
-        send_to_tracer_nif(NULL, &c_p->common, c_p->common.id,
+    Eterm pam_result;
+
+    if (!te) {
+        te = &erts_receive_tracing[erts_active_bp_ix()];
+        if (!te->on)
+            return;
+    }
+
+    if (te->match_spec) {
+        Eterm args[2];
+        Uint32 return_flags;
+        args[0] = am_undefined; /* ToDo: from who? */
+        args[1] = msg;
+        pam_result = erts_match_set_run(c_p, receiver,
+                                        te->match_spec, args, 2,
+                                        ERTS_PAM_TMP_RESULT, &return_flags);
+        if (is_non_value(pam_result)
+            || pam_result == am_false
+            || (ERTS_TRACE_FLAGS(receiver) & F_TRACE_SILENT)) {
+            erts_match_set_release_result(c_p);
+            return;
+        }
+    } else
+        pam_result = am_true;
+
+    if (is_tracer_enabled(NULL, 0, &receiver->common, &tnif,
+                          TRACE_FUN_E_RECEIVE, am_receive)) {
+        send_to_tracer_nif(NULL, &receiver->common, receiver->common.id,
                            tnif, TRACE_FUN_T_RECEIVE,
-                           am_receive, msg, THE_NON_VALUE, am_true);
+                           am_receive, msg, THE_NON_VALUE, pam_result);
+    }
+    erts_match_set_release_result(c_p);
 }
 
 int
@@ -1226,7 +1255,8 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
            may remove it, and we still want to generate a trace message */
         erts_tracer_update(&pre_ms_tracer, *tracer);
         tracer = &pre_ms_tracer;
-        pam_result = erts_match_set_run(p, match_spec, args, arity,
+        pam_result = erts_match_set_run(p, p,
+                                        match_spec, args, arity,
                                         ERTS_PAM_TMP_RESULT, &return_flags);
         if (is_non_value(pam_result)) {
             erts_match_set_release_result(p);
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index e8e968c0cf..040aa296a1 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -56,6 +56,15 @@
 
 struct binary;
 
+typedef struct
+{
+    int on;
+    struct binary* match_spec;
+} ErtsTracingEvent;
+
+extern ErtsTracingEvent erts_send_tracing[];
+extern ErtsTracingEvent erts_receive_tracing[];
+
 /* erl_bif_trace.c */
 Eterm erl_seq_trace_info(Process *p, Eterm arg1);
 void erts_system_monitor_clear(Process *c_p);
@@ -91,7 +100,7 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
 #endif
 
 void trace_send(Process*, Eterm, Eterm);
-void trace_receive(Process *, Eterm);
+void trace_receive(Process*, Process*, Eterm, ErtsTracingEvent*);
 Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec,
                        Eterm* args, int local, ErtsTracer *tracer);
 void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval,
@@ -220,12 +229,4 @@ ERTS_DECLARE_DUMMY(erts_tracer_nil) = NIL;
      && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN,          \
                                     &(PROC)->common, am_trace_status))
 
-typedef struct
-{
-    int on;
-    struct binary* match_spec;
-} ErtsTracingEvent;
-
-extern ErtsTracingEvent erts_send_tracing[];
-
 #endif /* ERL_TRACE_H__ */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index a7bc990deb..916c6277c1 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1493,7 +1493,9 @@ enum erts_pam_run_flags {
     ERTS_PAM_CONTIGUOUS_TUPLE=4,
     ERTS_PAM_IGNORE_TRACE_SILENT=8
 };
-extern Eterm erts_match_set_run(Process *p, Binary *mpsp, 
+extern Eterm erts_match_set_run(Process *p,
+                                Process *self,
+                                Binary *mpsp,
 				Eterm *args, int num_args,
 				enum erts_pam_run_flags in_flags,
 				Uint32 *return_flags);
-- 
cgit v1.2.3


From 5cb62b003d082c5a32ef5b12a89d872a317fd05f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 17 Mar 2016 16:35:08 +0100
Subject: erts: Add matchspec restrictions for 'receive' trace

and non-call-trace.

This is the easy way out to avoid difficult locking
scenarios when accessing tracing flags on another process.
---
 erts/emulator/beam/erl_bif_trace.c |  13 ++-
 erts/emulator/beam/erl_db_util.c   | 223 ++++++++++++++++---------------------
 erts/emulator/beam/erl_db_util.h   |   5 +
 erts/emulator/beam/global.h        |   2 +-
 4 files changed, 110 insertions(+), 133 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 7b21ff0110..c604053caa 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -156,11 +156,14 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     } else if (Pattern == am_pause) {
 	match_prog_set = NULL;
 	on = ERTS_BREAK_PAUSE;
-    } else if ((match_prog_set = erts_match_set_compile(p, Pattern)) != NULL) {
-	MatchSetRef(match_prog_set);
-	on = 1;
-    } else{
-	goto error;
+    } else {
+	match_prog_set = erts_match_set_compile(p, Pattern, MFA);
+	if (match_prog_set) {
+	    MatchSetRef(match_prog_set);
+	    on = 1;
+	} else{
+	    goto error;
+	}
     }
     
     is_global = 0;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 72d59df813..ae2cc40bfa 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -123,6 +123,9 @@ do {									\
 
 #define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm))))
 
+#define add_dmc_err(EINFO, STR, VAR, TERM, SEV) \
+       vadd_dmc_err(EINFO, SEV, VAR, STR, TERM)
+
 
 static ERTS_INLINE Process *
 get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks)
@@ -889,11 +892,7 @@ void db_match_dis(Binary *prog);
 #define TRACE /* Nothing */
 #define FENCE_PATTERN_SIZE 0
 #endif
-static void add_dmc_err(DMCErrInfo *err_info, 
-			   char *str,
-			   int variable,
-			   Eterm term,
-			   DMCErrorSeverity severity);
+static void vadd_dmc_err(DMCErrInfo*, DMCErrorSeverity, int var, const char *str, ...);
 
 static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity);
 
@@ -989,12 +988,16 @@ Eterm erts_match_set_get_source(Binary *mpsp)
 }
 
 /* This one is for the tracing */
-Binary *erts_match_set_compile(Process *p, Eterm matchexpr) {
+Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA) {
     Binary *bin;
     Uint sz;
     Eterm *hp;
+    Uint flags = DCOMP_TRACE;
+
+    if (is_tuple(MFA)) flags |= DCOMP_CALL_TRACE;
+    if (MFA != am_receive) flags |= DCOMP_ALLOW_TRACE_OPS;
     
-    bin = db_match_set_compile(p, matchexpr, DCOMP_TRACE);
+    bin = db_match_set_compile(p, matchexpr, flags);
     if (bin != NULL) {
 	MatchProg *prog = Binary2MatchProg(bin);
 	sz = size_object(matchexpr);
@@ -1124,8 +1127,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags)
     int i;
 
     if (!is_list(matchexpr)) {
-	add_dmc_err(err_info, "Match programs are not in a list.", 
-		    -1, 0UL, dmcError);
+	add_dmc_err(err_info, "Match programs are not in a list.",
+                    -1, 0UL, dmcError);
 	goto done;
     }
     num_heads = 0;
@@ -1133,9 +1136,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags)
 	++num_heads;
 
     if (l != NIL)  { /* proper list... */
-	add_dmc_err(err_info, "Match programs are not in a proper "
-		    "list.", 
-		    -1, 0UL, dmcError);
+	add_dmc_err(err_info, "Match programs are not in a proper list.",
+                     -1, 0UL, dmcError);
 	goto done;
     }
 
@@ -3260,20 +3262,20 @@ int erts_db_is_compiled_ms(Eterm term)
 ** Utility to add an error
 */
 
-static void add_dmc_err(DMCErrInfo *err_info, 
-			char *str,
-			int variable,
-			Eterm term,
-			DMCErrorSeverity severity)
+static void vadd_dmc_err(DMCErrInfo *err_info,
+                         DMCErrorSeverity severity,
+                         int variable,
+                         const char *str,
+                         ...)
 {
+    DMCError *e;
+    va_list args;
+    va_start(args, str);
+
+
     /* Linked in in reverse order, to ease the formatting */
-    DMCError *e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError));
-    if (term != 0UL) {
-	erts_snprintf(e->error_string, DMC_ERR_STR_LEN, str, term);
-    } else {
-	strncpy(e->error_string, str, DMC_ERR_STR_LEN);
-	e->error_string[DMC_ERR_STR_LEN] ='\0';
-    }
+    e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError));
+    erts_vsnprintf(e->error_string, DMC_ERR_STR_LEN, str, args);
     e->variable = variable;
     e->severity = severity;
     e->next = err_info->first;
@@ -3283,8 +3285,11 @@ static void add_dmc_err(DMCErrInfo *err_info,
     err_info->first = e;
     if (severity >= dmcError)
 	err_info->error_added = 1;
+
+    va_end(args);
 }
     
+
 /*
 ** Handle one term in the match expression (not the guard) 
 */
@@ -3483,24 +3488,21 @@ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
 	    context->stack_need = context->stack_used;
 }
 
-#define RETURN_ERROR_X(String, X, Y, ContextP, ConstantF)        \
-do {                                                            \
-if ((ContextP)->err_info != NULL) {				\
-    (ConstantF) = 0;						\
-    add_dmc_err((ContextP)->err_info, String, X, Y, dmcError);  \
-    return retOk;						\
-} else 								\
-  return retFail;                                                \
-} while(0)
+#define RETURN_ERROR_X(VAR, ContextP, ConstantF, String, ARG)            \
+    (((ContextP)->err_info != NULL)				         \
+     ? ((ConstantF) = 0,						 \
+        vadd_dmc_err((ContextP)->err_info, dmcError, VAR, String, ARG),  \
+        retOk)						                 \
+     : retFail)
 
 #define RETURN_ERROR(String, ContextP, ConstantF) \
-     RETURN_ERROR_X(String, -1, 0UL, ContextP, ConstantF)
+     return RETURN_ERROR_X(-1, ContextP, ConstantF, String, 0)
 
 #define RETURN_VAR_ERROR(String, N, ContextP, ConstantF) \
-     RETURN_ERROR_X(String, N, 0UL, ContextP, ConstantF)
+     return RETURN_ERROR_X(N, ContextP, ConstantF, String, 0)
 
 #define RETURN_TERM_ERROR(String, T, ContextP, ConstantF) \
-     RETURN_ERROR_X(String, -1, T, ContextP, ConstantF)
+     return RETURN_ERROR_X(-1, ContextP, ConstantF, String, T)
 
 #define WARNING(String, ContextP) \
 add_dmc_err((ContextP)->err_info, String, -1, 0UL, dmcWarning)
@@ -3766,7 +3768,7 @@ static DMCRet dmc_variable(DMCContext *context,
     Uint n = db_is_variable(t);
 
     if (n >= heap->vars_used || !heap->vars[n].is_bound) {
-	RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant);
+	RETURN_VAR_ERROR("Variable $%%d is unbound.", n, context, *constant);
     }
 
     dmc_add_pushv_variant(context, heap, text, n);
@@ -4098,7 +4100,30 @@ static DMCRet dmc_exception_trace(DMCContext *context,
     return retOk;
 }
 
-
+static int check_trace(const char* op,
+                       DMCContext *context,
+                       int *constant,
+                       int need_cflags,
+                       int allow_in_guard,
+                       DMCRet* retp)
+{
+    if (!(context->cflags & DCOMP_TRACE)) {
+	*retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' "
+                               "used in wrong dialect.", op);
+        return 0;
+    }
+    if ((context->cflags & need_cflags) != need_cflags) {
+        *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' "
+                               "not allow for this trace event.", op);
+        return 0;
+    }
+    if (context->is_guard && !allow_in_guard) {
+        *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' "
+                               "called in guard context.", op);
+        return 0;
+    }
+    return 1;
+}
 
 static DMCRet dmc_is_seq_trace(DMCContext *context,
 			       DMCHeap *heap,
@@ -4108,12 +4133,11 @@ static DMCRet dmc_is_seq_trace(DMCContext *context,
 {
     Eterm *p = tuple_val(t);
     Uint a = arityval(*p);
+    DMCRet ret;
     
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'is_seq_trace' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
+    if (!check_trace("is_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 1, &ret))
+        return ret;
+
     if (a != 1) {
 	RETURN_TERM_ERROR("Special form 'is_seq_trace' called with "
 			  "arguments in %T.", t, context, *constant);
@@ -4137,16 +4161,8 @@ static DMCRet dmc_set_seq_token(DMCContext *context,
     DMCRet ret;
     int c;
     
-
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'set_seq_token' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
-	RETURN_ERROR("Special form 'set_seq_token' called in "
-		     "guard context.", context, *constant);
-    }
+    if (!check_trace("set_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret))
+        return ret;
 
     if (a != 3) {
 	RETURN_TERM_ERROR("Special form 'set_seq_token' called with wrong "
@@ -4183,16 +4199,11 @@ static DMCRet dmc_get_seq_token(DMCContext *context,
 {
     Eterm *p = tuple_val(t);
     Uint a = arityval(*p);
+    DMCRet ret;
+
+    if (!check_trace("get_seq_token", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret))
+        return ret;
 
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'get_seq_token' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
-	RETURN_ERROR("Special form 'get_seq_token' called in "
-		     "guard context.", context, *constant);
-    }
     if (a != 1) {
 	RETURN_TERM_ERROR("Special form 'get_seq_token' called with "
 			  "arguments in %T.", t, context, 
@@ -4256,16 +4267,10 @@ static DMCRet dmc_process_dump(DMCContext *context,
 {
     Eterm *p = tuple_val(t);
     Uint a = arityval(*p);
-    
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'process_dump' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
-	RETURN_ERROR("Special form 'process_dump' called in "
-		     "guard context.", context, *constant);
-    }
+    DMCRet ret;
+
+    if (!check_trace("process_dump", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret))
+        return ret;
 
     if (a != 1) {
 	RETURN_TERM_ERROR("Special form 'process_dump' called with "
@@ -4289,17 +4294,8 @@ static DMCRet dmc_enable_trace(DMCContext *context,
     DMCRet ret;
     int c;
     
-
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'enable_trace' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
-	RETURN_ERROR("Special form 'enable_trace' called in guard context.",
-		     context, 
-		     *constant);
-    }
+    if (!check_trace("enable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret))
+        return ret;
 
     switch (a) {
     case 2:
@@ -4348,18 +4344,9 @@ static DMCRet dmc_disable_trace(DMCContext *context,
     Uint a = arityval(*p);
     DMCRet ret;
     int c;
-    
 
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'disable_trace' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
-	RETURN_ERROR("Special form 'disable_trace' called in guard context.",
-		     context, 
-		     *constant);
-    }
+    if (!check_trace("disable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret))
+        return ret;
 
     switch (a) {
     case 2:
@@ -4409,17 +4396,8 @@ static DMCRet dmc_trace(DMCContext *context,
     DMCRet ret;
     int c;
     
-
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'trace' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
-	RETURN_ERROR("Special form 'trace' called in guard context.",
-		     context, 
-		     *constant);
-    }
+    if (!check_trace("trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret))
+        return ret;
 
     switch (a) {
     case 3:
@@ -4480,16 +4458,11 @@ static DMCRet dmc_caller(DMCContext *context,
 {
     Eterm *p = tuple_val(t);
     Uint a = arityval(*p);
+    DMCRet ret;
      
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'caller' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
- 	RETURN_ERROR("Special form 'caller' called in "
- 		     "guard context.", context, *constant);
-    }
+    if (!check_trace("caller", context, constant,
+                     (DCOMP_CALL_TRACE|DCOMP_ALLOW_TRACE_OPS), 0, &ret))
+        return ret;
   
     if (a != 1) {
  	RETURN_TERM_ERROR("Special form 'caller' called with "
@@ -4515,15 +4488,8 @@ static DMCRet dmc_silent(DMCContext *context,
     DMCRet ret;
     int c;
      
-    if (!(context->cflags & DCOMP_TRACE)) {
-	RETURN_ERROR("Special form 'silent' used in wrong dialect.",
-		     context, 
-		     *constant);
-    }
-    if (context->is_guard) {
- 	RETURN_ERROR("Special form 'silent' called in "
- 		     "guard context.", context, *constant);
-    }
+    if (!check_trace("silent", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret))
+        return ret;
   
     if (a != 2) {
 	RETURN_TERM_ERROR("Special form 'silent' called with wrong "
@@ -5063,11 +5029,14 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 	return THE_NON_VALUE;
     }
     if (trace) {
-	lint_res = db_match_set_lint(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE);
-	mps = db_match_set_compile(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE);
+        const Uint cflags = (DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE |
+                             DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS);
+	lint_res = db_match_set_lint(p, spec, cflags);
+	mps = db_match_set_compile(p, spec, cflags);
     } else {
-	lint_res = db_match_set_lint(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE);
-	mps = db_match_set_compile(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE);
+        const Uint cflags = (DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE);
+	lint_res = db_match_set_lint(p, spec, cflags);
+	mps = db_match_set_compile(p, spec, cflags);
     }
 
     if (mps == NULL) {
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index c3eb82a44a..60f7067d70 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -425,6 +425,11 @@ typedef struct dmc_err_info {
 #define DCOMP_FAKE_DESTRUCTIVE ((Uint) 8) /* When this is active, no setting of
 					     trace control words or seq_trace tokens will be done. */
 
+/* Allow lock seizing operations on the tracee and 3rd party processes */
+#define DCOMP_ALLOW_TRACE_OPS ((Uint) 0x10)
+
+/* This is call trace */
+#define DCOMP_CALL_TRACE ((Uint) 0x20)
 
 Binary *db_match_compile(Eterm *matchexpr, Eterm *guards,
 			 Eterm *body, int num_matches, 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 916c6277c1..328bfe344c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1483,7 +1483,7 @@ do {								\
 
 #define MatchSetGetSource(MPSP) erts_match_set_get_source(MPSP)
 
-extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr);
+extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA);
 Eterm erts_match_set_lint(Process *p, Eterm matchexpr); 
 extern void erts_match_set_release_result(Process* p);
 
-- 
cgit v1.2.3


From 9627711cc39fd311a573a172e6d0e8a6b55dbd17 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 1 Apr 2016 15:38:16 +0200
Subject: erts: Add Sender in 'receive' trace matchspec

All 'EXIT' and monitor messages are sent from 'system'

Timeouts are "sent" from 'clock_service'
---
 erts/emulator/beam/beam_emu.c      |  2 +-
 erts/emulator/beam/bif.c           |  2 +-
 erts/emulator/beam/dist.c          |  8 ++---
 erts/emulator/beam/erl_alloc.c     |  2 +-
 erts/emulator/beam/erl_bif_ddll.c  |  2 +-
 erts/emulator/beam/erl_bif_trace.c |  2 +-
 erts/emulator/beam/erl_db_util.c   | 14 +++++---
 erts/emulator/beam/erl_gc.c        |  2 +-
 erts/emulator/beam/erl_hl_timer.c  |  8 ++---
 erts/emulator/beam/erl_message.c   | 72 +++++++++++++++++++------------------
 erts/emulator/beam/erl_message.h   |  8 ++---
 erts/emulator/beam/erl_msacc.c     |  2 +-
 erts/emulator/beam/erl_nif.c       | 27 +++++++-------
 erts/emulator/beam/erl_process.c   | 10 +++---
 erts/emulator/beam/erl_time_sup.c  |  2 +-
 erts/emulator/beam/erl_trace.c     | 26 +++++++-------
 erts/emulator/beam/erl_trace.h     |  2 +-
 erts/emulator/beam/io.c            | 74 ++++++++++++++++++++++----------------
 erts/emulator/beam/utils.c         |  2 +-
 19 files changed, 144 insertions(+), 123 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 7eecd059a5..aab0baebea 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2166,7 +2166,7 @@ void process_main(void)
 
      PreFetch(0, next);
      if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
-	 trace_receive(c_p, c_p, am_timeout, NULL);
+	 trace_receive(c_p, am_clock_service, am_timeout, NULL);
      }
      if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
 	 save_calls(c_p, &exp_timeout);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ed5b2983dd..ac77fa96e1 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -611,7 +611,7 @@ erts_queue_monitor_message(Process *p,
     ref_copy    = copy_struct(ref, ref_size, &hp, ohp);
 
     tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy);
-    erts_queue_message(p, p_locksp, msgp, tup);
+    erts_queue_message(p, *p_locksp, msgp, tup, am_system);
 }
 
 static BIF_RETTYPE
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 52fd57e101..b020f16805 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -397,7 +397,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	    msgp = erts_alloc_message_heap(rp, &rp_locks,
 					   3, &hp, &ohp);
 	    tup = TUPLE2(hp, am_nodedown, name);
-	    erts_queue_message(rp, &rp_locks, msgp, tup);
+	    erts_queue_message(rp, rp_locks, msgp, tup, am_system);
 	}
 	erts_smp_proc_unlock(rp, rp_locks);
     }
@@ -1456,7 +1456,7 @@ int erts_net_message(Port *prt,
 		token = copy_struct(token, token_size, &hp, ohp);
 	    }
 
-	    erts_queue_dist_message(rp, &locks, ede_copy, token);
+	    erts_queue_dist_message(rp, locks, ede_copy, token, from);
 	    if (locks)
 		erts_smp_proc_unlock(rp, locks);
 	}
@@ -1505,7 +1505,7 @@ int erts_net_message(Port *prt,
 		token = copy_struct(token, token_size, &hp, ohp);
 	    }
 
-	    erts_queue_dist_message(rp, &locks, ede_copy, token);
+	    erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]);
 	    if (locks)
 		erts_smp_proc_unlock(rp, locks);
 	}
@@ -3317,7 +3317,7 @@ send_nodes_mon_msg(Process *rp,
     }
 
     ASSERT(hend == hp);
-    erts_queue_message(rp, rp_locksp, mp, msg);
+    erts_queue_message(rp, *rp_locksp, mp, msg, am_system);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d04977b9ae..3dc12bac51 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3286,7 +3286,7 @@ reply_alloc_info(void *vair)
     if (hp != hp_end)
 	erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1);
 
-    erts_queue_message(rp, &rp_locks, mp, msg);
+    erts_queue_message(rp, rp_locks, mp, msg, am_system);
 
     if (air->req_sched == sched_id)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 1e2db38442..ef77201544 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1737,7 +1737,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	hp += REF_THING_SIZE;
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
     }
-    erts_queue_message(proc, &rp_locks, mp, mess);
+    erts_queue_message(proc, rp_locks, mp, mess, am_system);
     erts_smp_proc_unlock(proc, rp_locks);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 }
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index c604053caa..ffc0afda58 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -2344,7 +2344,7 @@ reply_trace_delivered_all(void *vtdarp)
 #ifdef ERTS_SMP
         erts_send_sys_msg_proc(rp->common.id, rp->common.id, msg, bp);
 #else
-        erts_queue_message(rp, &rp_locks, mp, msg);
+        erts_queue_message(rp, rp_locks, mp, msg, am_system);
 #endif
 
 	erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp);
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index ae2cc40bfa..7d64529250 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -414,17 +414,19 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
 {
     ErtsMatchPseudoProcess *mpsp;
 #ifdef ERTS_SMP
-    mpsp = (ErtsMatchPseudoProcess *) c_p->scheduler_data->match_pseudo_process;
+    ErtsSchedulerData *esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data();
+
+    mpsp = (ErtsMatchPseudoProcess *) esdp->match_pseudo_process;
     if (mpsp)
 	cleanup_match_pseudo_process(mpsp, 0);
     else {
 	ASSERT(erts_smp_tsd_get(match_pseudo_process_key) == NULL);
 	mpsp = create_match_pseudo_process();
-	c_p->scheduler_data->match_pseudo_process = (void *) mpsp;
+	esdp->match_pseudo_process = (void *) mpsp;
 	erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp);
     }
     ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key));
-    mpsp->process.scheduler_data = c_p->scheduler_data;
+    mpsp->process.scheduler_data = esdp;
 #else
     mpsp = match_pseudo_process;
     cleanup_match_pseudo_process(mpsp, 0);
@@ -1750,7 +1752,7 @@ Eterm db_prog_match(Process *c_p,
     Eterm *esp;
     MatchVariable* variables;
     BeamInstr *cp;
-    UWord *pc = prog->text;
+    const UWord *pc = prog->text;
     Eterm *ehp;
     Eterm ret;
     Uint n;
@@ -1773,13 +1775,15 @@ Eterm db_prog_match(Process *c_p,
 
     ERTS_UNDEF(n,0);
 
+    ASSERT(c_p || !(in_flags & ERTS_PAM_COPY_RESULT));
+
     mpsp = get_match_pseudo_process(c_p, prog->heap_size);
     psp = &mpsp->process;
 
     /* We need to lure the scheduler into believing in the pseudo process, 
        because of floating point exceptions. Do *after* mpsp is set!!! */
 
-    esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p);
+    esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(psp);
     ASSERT(esdp != NULL);
     current_scheduled = esdp->current_process;
     /* SMP: psp->scheduler_data is set by get_match_pseudo_process */
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index df5d0f4918..85a1467a52 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2924,7 +2924,7 @@ reply_gc_info(void *vgcirp)
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, mp, msg);
+    erts_queue_message(rp, rp_locks, mp, msg, am_system);
 
     if (gcirp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 8e201d5711..c418762578 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1247,8 +1247,8 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
 	if (!ERTS_PROC_IS_EXITING(proc)) {
 	    ErtsMessage *mp = erts_alloc_message(0, NULL);
 	    mp->data.heap_frag = tmr->btm.bp;
-	    erts_queue_message(proc, &proc_locks, mp,
-			       tmr->btm.message);
+	    erts_queue_message(proc, proc_locks, mp,
+			       tmr->btm.message, am_clock_service);
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND);
 	    queued_message = 1;
 	    proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND;
@@ -1980,7 +1980,7 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 
 	ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end);
 
-	erts_queue_message(proc, &proc_locks, mp, msg);
+	erts_queue_message(proc, proc_locks, mp, msg, am_clock_service);
 
 	if (c_p)
 	    proc_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -2111,7 +2111,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 
 	msg = TUPLE3(hp, tag, tref, res);
 
-	erts_queue_message(c_p, &proc_locks, mp, msg);
+        erts_queue_message(c_p, proc_locks, mp, msg, am_clock_service);
 
 	proc_locks &= ~ERTS_PROC_LOCK_MAIN;
 	if (proc_locks)
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ee8a7702ce..9bb6e40a11 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -252,9 +252,10 @@ erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_s
 
 void
 erts_queue_dist_message(Process *rcvr,
-			ErtsProcLocks *rcvr_locks,
+			ErtsProcLocks rcvr_locks,
 			ErtsDistExternal *dist_ext,
-			Eterm token)
+			Eterm token,
+                        Eterm from)
 {
     ErtsMessage* mp;
 #ifdef USE_VM_PROBES
@@ -266,7 +267,7 @@ erts_queue_dist_message(Process *rcvr,
     erts_aint_t state;
 #endif
 
-    ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
+    ERTS_SMP_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
 
     mp = erts_alloc_message(0, NULL);
     mp->data.dist_ext = dist_ext;
@@ -281,10 +282,10 @@ erts_queue_dist_message(Process *rcvr,
 	ERL_MESSAGE_TOKEN(mp) = token;
 
 #ifdef ERTS_SMP
-    if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
+    if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
 	if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 	    ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
-	    if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) {
+	    if (rcvr_locks & ERTS_PROC_LOCK_STATUS) {
 		erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
 		need_locks |= ERTS_PROC_LOCK_STATUS;
 	    }
@@ -294,7 +295,7 @@ erts_queue_dist_message(Process *rcvr,
 
     state = erts_smp_atomic32_read_acqb(&rcvr->state);
     if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
-	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
+	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 	/* Drop message if receiver is exiting or has a pending exit ... */
 	erts_cleanup_messages(mp);
@@ -302,10 +303,13 @@ erts_queue_dist_message(Process *rcvr,
     else
 #endif
     if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
+        if (from == am_Empty)
+            from = dist_ext->dep->sysname;
+
 	/* Ahh... need to decode it in order to trace it... */
-	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
+	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
-	if (!erts_decode_dist_message(rcvr, *rcvr_locks, mp, 0))
+	if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0))
 	    erts_free_message(mp);
 	else {
 	    Eterm msg = ERL_MESSAGE_TERM(mp);
@@ -325,7 +329,7 @@ erts_queue_dist_message(Process *rcvr,
 			tok_label, tok_lastcnt, tok_serial);
 	    }
 #endif
-	    erts_queue_message(rcvr, rcvr_locks, mp, msg);
+	    erts_queue_message(rcvr, rcvr_locks, mp, msg, from);
 	}
     }
     else {
@@ -352,12 +356,12 @@ erts_queue_dist_message(Process *rcvr,
 
 	LINK_MESSAGE(rcvr, mp, &mp->next, 1);
 
-	if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ))
+	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 
 	erts_proc_notify_new_message(rcvr,
 #ifdef ERTS_SMP
-				     *rcvr_locks
+				     rcvr_locks
 #else
 				     0
 #endif
@@ -367,13 +371,13 @@ erts_queue_dist_message(Process *rcvr,
 
 /* Add messages last in message queue */
 static Sint
-queue_messages(Process *c_p,
-               Process* receiver,
+queue_messages(Process* receiver,
                erts_aint32_t *receiver_state,
                ErtsProcLocks receiver_locks,
                ErtsMessage* first,
                ErtsMessage** last,
-               Uint len)
+               Uint len,
+               Eterm from)
 {
     ErtsTracingEvent* te;
     Sint res;
@@ -475,7 +479,7 @@ queue_messages(Process *c_p,
         }
 #endif
         while (msg) {
-            trace_receive(c_p, receiver, ERL_MESSAGE_TERM(msg), te);
+            trace_receive(receiver, from, ERL_MESSAGE_TERM(msg), te);
             msg = msg->next;
         }
 
@@ -494,31 +498,31 @@ queue_messages(Process *c_p,
 }
 
 static Sint
-queue_message(Process *c_p,
-              Process* receiver,
+queue_message(Process* receiver,
               erts_aint32_t *receiver_state,
-              ErtsProcLocks *receiver_locks,
-              ErtsMessage* mp, Eterm msg)
+              ErtsProcLocks receiver_locks,
+              ErtsMessage* mp, Eterm msg, Eterm from)
 {
     ERL_MESSAGE_TERM(mp) = msg;
-    return queue_messages(c_p, receiver, receiver_state, *receiver_locks,
-                          mp, &mp->next, 1 );
+    return queue_messages(receiver, receiver_state, receiver_locks,
+                          mp, &mp->next, 1, from);
 }
 
 Sint
-erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks,
-                   ErtsMessage* mp, Eterm msg)
+erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks,
+                   ErtsMessage* mp, Eterm msg, Eterm from)
 {
-    return queue_message(NULL, receiver, NULL, receiver_locks, mp, msg);
+    return queue_message(receiver, NULL, receiver_locks, mp, msg, from);
 }
 
 
 Sint
-erts_queue_messages(Process* receiver, ErtsProcLocks *receiver_locks,
-                    ErtsMessage* first, ErtsMessage** last, Uint len)
+erts_queue_messages(Process* receiver, ErtsProcLocks receiver_locks,
+                    ErtsMessage* first, ErtsMessage** last, Uint len,
+                    Eterm from)
 {
-    return queue_messages(NULL, receiver, NULL, *receiver_locks,
-                          first, last, len);
+    return queue_messages(receiver, NULL, receiver_locks,
+                          first, last, len, from);
 }
 
 void
@@ -835,11 +839,11 @@ erts_send_message(Process* sender,
 #ifdef USE_VM_PROBES
     ERL_MESSAGE_DT_UTAG(mp) = utag;
 #endif
-    res = queue_message(sender,
-			receiver,
+    res = queue_message(receiver,
 			&receiver_state,
-			receiver_locks,
-			mp, message);
+			*receiver_locks,
+			mp, message,
+                        sender->common.id);
 
     BM_SWAP_TIMER(send,system);
     
@@ -896,7 +900,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL);
 	temptoken = copy_struct(token, sz_token, &hp, ohp);
         ERL_MESSAGE_TOKEN(mp) = temptoken;
-	erts_queue_message(to, to_locksp, mp, save);
+	erts_queue_message(to, *to_locksp, mp, save, am_system);
     } else {
 	sz_from = IS_CONST(from) ? 0 : size_object(from);
 #ifdef SHCOPY_SEND
@@ -918,7 +922,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 		     ? from
 		     : copy_struct(from, sz_from, &hp, ohp));
 	save = TUPLE3(hp, am_EXIT, from_copy, mess);
-	erts_queue_message(to, to_locksp, mp, save);
+	erts_queue_message(to, *to_locksp, mp, save, am_system);
     }
 }
 
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 608cf552a2..851ac37fda 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -295,10 +295,10 @@ ErlHeapFragment* new_message_buffer(Uint);
 ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
 					    Eterm *, Uint);
 void free_message_buffer(ErlHeapFragment *);
-void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm);
-Sint erts_queue_message(Process*, ErtsProcLocks*,ErtsMessage*, Eterm);
-Sint erts_queue_messages(Process*, ErtsProcLocks*,
-                         ErtsMessage*, ErtsMessage**, Uint);
+void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm);
+Sint erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm);
+Sint erts_queue_messages(Process*, ErtsProcLocks,
+                         ErtsMessage*, ErtsMessage**, Uint, Eterm);
 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
 Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index d0f305900a..0e625f213b 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -257,7 +257,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
 
     if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
 
-    erts_queue_message(rp, &rp_locks, msgp, msg);
+    erts_queue_message(rp, rp_locks, msgp, msg, am_system);
 
     if (esdp && msaccrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 941f44b9ec..bd3e7ebe54 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -366,7 +366,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
             rp_locks = 0;
             if (rp->common.id == c_p->common.id)
                 rp_locks = c_p_locks;
-            erts_queue_messages(rp, &rp_locks, first, last, len);
+            erts_queue_messages(rp, rp_locks, first, last, len, c_p->common.id);
             if (rp->common.id == c_p->common.id)
                 rp_locks &= ~c_p_locks;
             if (rp_locks)
@@ -405,7 +405,10 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
           ErlNifEnv* msg_env, ERL_NIF_TERM msg)
 {
     struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
-    ErtsProcLocks rp_locks = 0, lc_locks = 0, c_p_locks = ERTS_PROC_LOCK_MAIN;
+    ErtsProcLocks rp_locks = 0;
+#ifdef ERTS_SMP
+    ErtsProcLocks lc_locks = 0;
+#endif
     Process* rp;
     Process* c_p;
     ErtsMessage *mp;
@@ -417,7 +420,7 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     if (env != NULL) {
 	c_p = env->proc;
 	if (receiver == c_p->common.id) {
-	    rp_locks = c_p_locks;
+	    rp_locks = ERTS_PROC_LOCK_MAIN;
 	    flush_me = 1;
 	}
     }
@@ -470,14 +473,8 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 
         if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND))
             trace_send(c_p, receiver, msg);
-
-#ifndef ERTS_SMP
     }
-#endif
-
-        erts_queue_message(rp, &rp_locks, mp, msg);
 #ifdef ERTS_SMP
-    }
     else {
         /* This clause is taken when the nif is called in the context
            of a traced process. We do not know which locks we have
@@ -502,8 +499,6 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 #ifdef ERTS_ENABLE_LOCK_CHECK
         lc_locks = erts_proc_lc_my_proc_locks(rp);
         rp_locks |= lc_locks;
-        if (receiver == c_p->common.id)
-            c_p_locks |= lc_locks;
 #endif
         if (ERTS_FORCE_ENIF_SEND_DELAY() || msgq ||
             rp_locks & ERTS_PROC_LOCK_MSGQ ||
@@ -532,19 +527,25 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
                 msgq->last = &mp->next;
                 erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
             }
+            goto done;
         } else {
             erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
             rp_locks &= ~ERTS_PROC_LOCK_TRACE;
             rp_locks |= ERTS_PROC_LOCK_MSGQ;
-            erts_queue_message(rp, &rp_locks, mp, msg);
         }
     }
-#endif
+#endif /* ERTS_SMP */
+
+    erts_queue_message(rp, rp_locks, mp, msg,
+                       c_p ? c_p->common.id : am_undefined);
 
+#ifdef ERTS_SMP
+done:
     if (c_p == rp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
     if (rp_locks & ~lc_locks)
 	erts_smp_proc_unlock(rp, rp_locks & ~lc_locks);
+#endif
     if (!scheduler)
 	erts_proc_dec_refc(rp);
     if (flush_me) {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d485affa3b..da2559839d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1139,7 +1139,7 @@ reply_sched_wall_time(void *vswtrp)
 	hpp = &hp;
     }
 
-    erts_queue_message(rp, &rp_locks, mp, msg);
+    erts_queue_message(rp, rp_locks, mp, msg, am_system);
 
     if (swtrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -1218,7 +1218,7 @@ reply_system_check(void *vscrp)
     hpp = &hp;
     msg = STORE_NC(hpp, ohp, scrp->ref);
 
-    erts_queue_message(rp, &rp_locks, mp, msg);
+    erts_queue_message(rp, rp_locks, mp, msg, am_system);
 
     if (scrp->req_sched == esdp->no)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -10010,7 +10010,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 	ASSERT(hp_start + hsz == hp);
 #endif
 
-	erts_queue_message(rp, &rp_locks, mp, msg);
+	erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id);
 
 	if (c_p == rp)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -11744,7 +11744,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
 #endif
-	erts_queue_message(to, to_locksp, mp, mess);
+	erts_queue_message(to, *to_locksp, mp, mess, am_system);
     } else {
 	Eterm temp_token;
 	Uint sz_token;
@@ -11765,7 +11765,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, to);
 	temp_token = copy_struct(token, sz_token, &hp, ohp);
         ERL_MESSAGE_TOKEN(mp) = temp_token;
-	erts_queue_message(to, to_locksp, mp, mess);
+	erts_queue_message(to, *to_locksp, mp, mess, am_system);
     }
 }
 
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 346404fe2a..fadbd704bd 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1966,7 +1966,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 		    *patch_refp = ref;
 		    ASSERT(hsz == size_object(message_template));
 		    message = copy_struct(message_template, hsz, &hp, ohp);
-		    erts_queue_message(rp, &rp_locks, mp, message);
+		    erts_queue_message(rp, rp_locks, mp, message, am_clock_service);
 		}
 		erts_smp_proc_unlock(rp, rp_locks);
 	    }
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 755deda9c1..682f08f667 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -738,7 +738,7 @@ profile_send(Eterm from, Eterm message) {
 	else
 	    msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap);
 
-        erts_queue_message(profile_p, NULL, mp, msg);
+        erts_queue_message(profile_p, 0, mp, msg, from);
     }
 }
 
@@ -859,8 +859,8 @@ trace_send(Process *p, Eterm to, Eterm msg)
  * or   {trace, Pid, receive, Msg}
  */
 void
-trace_receive(Process *c_p,
-              Process* receiver,
+trace_receive(Process* receiver,
+              Eterm from,
               Eterm msg, ErtsTracingEvent* te)
 {
     ErtsTracerNif *tnif = NULL;
@@ -875,15 +875,15 @@ trace_receive(Process *c_p,
     if (te->match_spec) {
         Eterm args[2];
         Uint32 return_flags;
-        args[0] = am_undefined; /* ToDo: from who? */
+        args[0] = from;
         args[1] = msg;
-        pam_result = erts_match_set_run(c_p, receiver,
+        pam_result = erts_match_set_run(NULL, receiver,
                                         te->match_spec, args, 2,
                                         ERTS_PAM_TMP_RESULT, &return_flags);
         if (is_non_value(pam_result)
             || pam_result == am_false
             || (ERTS_TRACE_FLAGS(receiver) & F_TRACE_SILENT)) {
-            erts_match_set_release_result(c_p);
+            erts_match_set_release_result(NULL);
             return;
         }
     } else
@@ -895,7 +895,7 @@ trace_receive(Process *c_p,
                            tnif, TRACE_FUN_T_RECEIVE,
                            am_receive, msg, THE_NON_VALUE, pam_result);
     }
-    erts_match_set_release_result(c_p);
+    erts_match_set_release_result(NULL);
 }
 
 int
@@ -1483,7 +1483,7 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg);
+	erts_queue_message(monitor_p, 0, mp, msg, am_system);
     }
 #endif
 }
@@ -1548,7 +1548,7 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time)
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg);
+	erts_queue_message(monitor_p, 0, mp, msg, am_system);
     }
 #endif
 }
@@ -1623,7 +1623,7 @@ monitor_long_gc(Process *p, Uint time) {
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg);
+	erts_queue_message(monitor_p, 0, mp, msg, am_system);
     }
 #endif
 }
@@ -1698,7 +1698,7 @@ monitor_large_heap(Process *p) {
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg);
+	erts_queue_message(monitor_p, 0, mp, msg, am_system);
     }
 #endif
 }
@@ -1730,7 +1730,7 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, NULL, mp, msg);
+	erts_queue_message(monitor_p, 0, mp, msg, am_system);
     }
 #endif
 
@@ -2514,7 +2514,7 @@ sys_msg_dispatcher_func(void *unused)
 		queue_proc_msg:
 		    mp = erts_alloc_message(0, NULL);
 		    mp->data.heap_frag = smqp->bp;
-		    erts_queue_message(proc,&proc_locks,mp,smqp->msg);
+		    erts_queue_message(proc,proc_locks,mp,smqp->msg,am_system);
 #ifdef DEBUG_PRINTOUTS
 		    erts_fprintf(stderr, "delivered\n");
 #endif
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 040aa296a1..de4beadb7e 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -100,7 +100,7 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
 #endif
 
 void trace_send(Process*, Eterm, Eterm);
-void trace_receive(Process*, Process*, Eterm, ErtsTracingEvent*);
+void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*);
 Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec,
                        Eterm* args, int local, ErtsTracer *tracer);
 void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval,
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b14ca77a04..1d307ae15e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1432,10 +1432,11 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
 static ERTS_INLINE void
 queue_port_sched_op_reply(Process *rp,
-			  ErtsProcLocks *rp_locksp,
+			  ErtsProcLocks rp_locks,
                           ErtsHeapFactory* factory,
 			  Uint32 *ref_num,
-			  Eterm msg)
+			  Eterm msg,
+			  Port* prt)
 {
     Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0);
     Eterm ref;
@@ -1448,11 +1449,12 @@ queue_port_sched_op_reply(Process *rp,
 
     erts_factory_trim_and_close(factory, &msg, 1);
 
-    erts_queue_message(rp, rp_locksp, factory->message, msg);
+    erts_queue_message(rp, rp_locks, factory->message, msg,
+		       prt ? prt->common.id : am_undefined);
 }
 
 static void
-port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
+port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg, Port* prt)
 {
     Process *rp = erts_proc_lookup_raw(to);
     if (rp) {
@@ -1478,10 +1480,11 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
 				  factory.off_heap));
 
 	queue_port_sched_op_reply(rp,
-				  &rp_locks,
+				  rp_locks,
                                   &factory,
 				  ref_num,
-				  msg_copy);
+				  msg_copy,
+				  prt);
 
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
@@ -1651,7 +1654,7 @@ port_badsig(Port *prt, erts_aint32_t state, int op,
 			state,
 			sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT);
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt);
     return ERTS_PORT_REDS_BADSIG;
 } /* port_badsig */
 /* bad_port_signal() will
@@ -1820,7 +1823,7 @@ port_sig_outputv(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s
     }
 
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, reply);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, reply, prt);
 
     cleanup_scheduled_outputv(sigdp->u.outputv.evp,
 			      sigdp->u.outputv.cbinp);
@@ -1928,7 +1931,7 @@ port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si
     }
 
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, reply);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, reply, prt);
 
     cleanup_scheduled_output(sigdp->u.output.bufp);
 
@@ -2636,7 +2639,7 @@ port_sig_exit(Port *prt,
     if (sigdp->u.exit.bp)
 	free_message_buffer(sigdp->u.exit.bp);
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, msg);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, msg, prt);
 
     return ERTS_PORT_REDS_EXIT;
 }
@@ -2829,7 +2832,7 @@ port_sig_connect(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s
 	    msg = am_true;
     }
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, msg);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, msg, prt);
     return ERTS_PORT_REDS_CONNECT;
 }
 
@@ -2912,7 +2915,7 @@ port_sig_unlink(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si
     if (op == ERTS_PROC2PORT_SIG_EXEC)
 	port_unlink(prt, sigdp->u.unlink.from);
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
     return ERTS_PORT_REDS_UNLINK;
 }
 
@@ -3007,7 +3010,7 @@ port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigd
 	port_link_failure(sigdp->u.link.port, sigdp->u.link.to);
     }
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
     return ERTS_PORT_REDS_LINK;
 }
 
@@ -3064,7 +3067,8 @@ init_ack_send_reply(Port *port, Eterm resp)
     }
     port_sched_op_reply(port->async_open_port->to,
                         port->async_open_port->ref,
-                        resp);
+                        resp,
+			port);
 
     erts_free(ERTS_ALC_T_PRTSD, port->async_open_port);
     port->async_open_port = NULL;
@@ -3461,7 +3465,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res)
 				     sz_res + 3, &hp, &ohp);
 	res = copy_struct(res, sz_res, &hp, ohp);
 	tuple = TUPLE2(hp, sender, res);
-	erts_queue_message(rp, &rp_locks, mp, tuple);
+	erts_queue_message(rp, rp_locks, mp, tuple, sender);
 
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
@@ -3562,7 +3566,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
         trace_port_send(prt, to, tuple, 1);
 
     ERL_MESSAGE_TOKEN(mp) = am_undefined;
-    erts_queue_message(rp, &rp_locks, mp, tuple);
+    erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
@@ -3734,7 +3738,7 @@ deliver_vec_message(Port* prt,			/* Port */
         trace_port_send(prt, to, tuple, 1);
 
     ERL_MESSAGE_TOKEN(mp) = am_undefined;
-    erts_queue_message(rp, &rp_locks, mp, tuple);
+    erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
     erts_smp_proc_unlock(rp, rp_locks);
     if (!scheduler)
 	erts_proc_dec_refc(rp);
@@ -4388,10 +4392,11 @@ port_sig_control(Port *prt,
 					    &factory.hp,
 					    factory.off_heap);
 	    queue_port_sched_op_reply(rp,
-				      &rp_locks,
+				      rp_locks,
                                       &factory,
 				      sigdp->ref,
-				      msg);
+				      msg,
+				      prt);
 
 	    if (rp_locks)
 		erts_smp_proc_unlock(rp, rp_locks);
@@ -4402,7 +4407,7 @@ port_sig_control(Port *prt,
     /* failure */
 
     if (sigdp->caller != ERTS_INVALID_PID)
-        port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+        port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt);
 
 done:
 
@@ -4739,10 +4744,11 @@ port_sig_call(Port *prt,
 		    msg = TUPLE2(hp, am_ok, msg);
 
 		    queue_port_sched_op_reply(rp,
-					      &rp_locks,
+					      rp_locks,
                                               &factory,
 					      sigdp->ref,
-					      msg);
+					      msg,
+					      prt);
 
 		    if (rp_locks)
 			erts_smp_proc_unlock(rp, rp_locks);
@@ -4754,7 +4760,7 @@ port_sig_call(Port *prt,
 	}
     }
 
-    port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+    port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg, prt);
 
 done:
 
@@ -4969,7 +4975,7 @@ port_sig_info(Port *prt,
 {
     ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY);
     if (op != ERTS_PROC2PORT_SIG_EXEC)
-	port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined);
+	port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined, prt);
     else {
 	Eterm *hp, *hp_start;
 	Uint hsz;
@@ -4995,10 +5001,11 @@ port_sig_info(Port *prt,
 	    mp->data.heap_frag = bp;
             erts_factory_selfcontained_message_init(&factory, mp, hp);
 	    queue_port_sched_op_reply(rp,
-				      &rp_locks,
+				      rp_locks,
                                       &factory,
 				      sigdp->ref,
-				      value);
+				      value,
+				      prt);
 	}
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
@@ -5115,7 +5122,7 @@ reply_io_bytes(void *vreq)
 
 	msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout);
 
-	erts_queue_message(rp, &rp_locks, mp, msg);
+	erts_queue_message(rp, rp_locks, mp, msg, am_system);
 
 	if (req->sched_id == sched_id)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -5613,7 +5620,7 @@ void driver_report_exit(ErlDrvPort ix, int status)
         trace_port_send(prt, pid, tuple, 1);
 
    ERL_MESSAGE_TOKEN(mp) = am_undefined;
-   erts_queue_message(rp, &rp_locks, mp, tuple);
+   erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
 
    erts_smp_proc_unlock(rp, rp_locks);
    if (!scheduler)
@@ -6217,15 +6224,20 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len)
  done:
 
     if (res > 0) {
+	Eterm from = am_undefined;
 	mess = ESTACK_POP(stack);  /* get resulting value */
 	erts_factory_trim_and_close(&factory, &mess, 1);
 
-        if (prt && IS_TRACED_FL(prt, F_TRACE_SEND))
-            trace_port_send(prt, to, mess, 1);
+	if (prt) {
+	    if (IS_TRACED_FL(prt, F_TRACE_SEND)) {
+		trace_port_send(prt, to, mess, 1);
+	    }
+	    from = prt->common.id;
+	}
 
 	/* send message */
         ERL_MESSAGE_TOKEN(factory.message) = am_undefined;
-	erts_queue_message(rp, &rp_locks, factory.message, mess);
+	erts_queue_message(rp, rp_locks, factory.message, mess, from);
     }
     else if (res == -2) {
         /* this clause only happens when we were requested to
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index b280995488..74114626be 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2298,7 +2298,7 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *
     {
 	ErtsMessage *mp = erts_alloc_message(0, NULL);
 	mp->data.heap_frag = bp;
-	erts_queue_message(p, NULL /* only used for smp build */, mp, message);
+	erts_queue_message(p, 0, mp, message, am_system);
     }
 #endif
 }
-- 
cgit v1.2.3


From da75310ce8973221ac90fd34f6375bfc17ae751b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 31 Mar 2016 21:20:24 +0200
Subject: erts: Change receive matchspec to [NodeOrOther, Pid, Msg]

---
 erts/emulator/beam/erl_trace.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 682f08f667..be0d2f0939 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -871,14 +871,23 @@ trace_receive(Process* receiver,
         if (!te->on)
             return;
     }
+    else ASSERT(te->on);
 
     if (te->match_spec) {
-        Eterm args[2];
+        Eterm args[3];
         Uint32 return_flags;
-        args[0] = from;
-        args[1] = msg;
+        if (is_pid(from)) {
+            args[0] = pid_node_name(from);
+            args[1] = from;
+        }
+        else {
+            ASSERT(is_atom(from));
+            args[0] = from;  /* node name or other atom (e.g 'system') */
+            args[1] = am_undefined;
+        }
+        args[2] = msg;
         pam_result = erts_match_set_run(NULL, receiver,
-                                        te->match_spec, args, 2,
+                                        te->match_spec, args, 3,
                                         ERTS_PAM_TMP_RESULT, &return_flags);
         if (is_non_value(pam_result)
             || pam_result == am_false
-- 
cgit v1.2.3


From 14b81c60f0c6403e9371be420a6df4d414180c68 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 29 Apr 2016 17:50:18 +0200
Subject: erts: Fix PAM to be callable from non-scheduler thread

also simplified the interface to to run PAM from trace
---
 erts/emulator/beam/erl_db_util.c | 73 ++++++++++++++++++++++++----------------
 erts/emulator/beam/erl_trace.c   | 61 +++++++++++++++------------------
 erts/emulator/beam/global.h      | 22 ++++++++----
 3 files changed, 88 insertions(+), 68 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 7d64529250..051c107d1f 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -414,19 +414,27 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
 {
     ErtsMatchPseudoProcess *mpsp;
 #ifdef ERTS_SMP
-    ErtsSchedulerData *esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data();
+    ErtsSchedulerData *esdp;
+
+    esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data();
+
+    mpsp = esdp ? esdp->match_pseudo_process :
+        (ErtsMatchPseudoProcess*) erts_smp_tsd_get(match_pseudo_process_key);
 
-    mpsp = (ErtsMatchPseudoProcess *) esdp->match_pseudo_process;
-    if (mpsp)
+    if (mpsp) {
+        ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key));
+        ASSERT(mpsp->process.scheduler_data == esdp);
 	cleanup_match_pseudo_process(mpsp, 0);
+    }
     else {
 	ASSERT(erts_smp_tsd_get(match_pseudo_process_key) == NULL);
 	mpsp = create_match_pseudo_process();
-	esdp->match_pseudo_process = (void *) mpsp;
+        if (esdp) {
+            esdp->match_pseudo_process = (void *) mpsp;
+        }
+        mpsp->process.scheduler_data = esdp;
 	erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp);
     }
-    ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key));
-    mpsp->process.scheduler_data = esdp;
 #else
     mpsp = match_pseudo_process;
     cleanup_match_pseudo_process(mpsp, 0);
@@ -1206,32 +1214,37 @@ done:
     return ret;
 }
     
-Eterm erts_match_set_run(Process *c_p,
-                         Process *self,
-                         Binary *mpsp,
-			 Eterm *args, int num_args,
-			 enum erts_pam_run_flags in_flags,
-			 Uint32 *return_flags) 
+/* Returns
+ *   am_false      if no match or
+ *                 if {message,false} has been called,
+ *   am_true       if {message,_} has NOT been called or
+ *                 if {message,true} has been called,
+ *   Msg           if {message,Msg} has been called.
+ *
+ *   If return value is_not_immed
+ *   then erts_match_set_release_result_trace() must be called to release it.
+ */
+Eterm erts_match_set_run_trace(Process *c_p,
+                               Process *self,
+                               Binary *mpsp,
+                               Eterm *args, int num_args,
+                               enum erts_pam_run_flags in_flags,
+                               Uint32 *return_flags)
 {
     Eterm ret;
 
     ret = db_prog_match(c_p, self, mpsp, NIL, args, num_args,
 			in_flags, return_flags);
-#if defined(HARDDEBUG)
-    if (is_non_value(ret)) {
-	erts_fprintf(stderr, "Failed\n");
-    } else {
-	erts_fprintf(stderr, "Returning : %T\n", ret);
+
+    ASSERT(!(is_non_value(ret) && *return_flags));
+
+    if (is_non_value(ret) || ret == am_false) {
+        erts_match_set_release_result(c_p);
+        return am_false;
     }
-#endif
+    if (is_immed(ret))
+        erts_match_set_release_result(c_p);
     return ret;
-    /* Returns 
-     *   THE_NON_VALUE if no match
-     *   am_false      if {message,false} has been called,
-     *   am_true       if {message,_} has not been called or
-     *                 if {message,true} has been called,
-     *   Msg           if {message,Msg} has been called.
-     */
 }
 
 static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp,
@@ -1774,6 +1787,7 @@ Eterm db_prog_match(Process *c_p,
 #endif /* DMC_DEBUG */
 
     ERTS_UNDEF(n,0);
+    ERTS_UNDEF(current_scheduled,NULL);
 
     ASSERT(c_p || !(in_flags & ERTS_PAM_COPY_RESULT));
 
@@ -1784,8 +1798,8 @@ Eterm db_prog_match(Process *c_p,
        because of floating point exceptions. Do *after* mpsp is set!!! */
 
     esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(psp);
-    ASSERT(esdp != NULL);
-    current_scheduled = esdp->current_process;
+    if (esdp)
+        current_scheduled = esdp->current_process;
     /* SMP: psp->scheduler_data is set by get_match_pseudo_process */
 
 #ifdef DMC_DEBUG
@@ -2517,7 +2531,8 @@ success:
     }
 #endif
 
-    esdp->current_process = current_scheduled;
+    if (esdp)
+        esdp->current_process = current_scheduled;
 
     return ret;
 #undef FAIL
@@ -5073,7 +5088,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 	    }
 	    save_cp = p->cp;
 	    p->cp = NULL;
-	    res = erts_match_set_run(p, p,
+	    res = erts_match_set_run_trace(p, p,
                       mps, arr, n,
 		      ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT,
 		      &ret_flags);
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index be0d2f0939..9f3341537d 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -825,13 +825,13 @@ trace_send(Process *p, Eterm to, Eterm msg)
 	Uint32 return_flags;
 	args[0] = to;
 	args[1] = msg;
-	pam_result = erts_match_set_run(p, p,
-                                        te->match_spec, args, 2,
-					ERTS_PAM_TMP_RESULT, &return_flags);
-	if (is_non_value(pam_result)
-	    || pam_result == am_false
-	    || (ERTS_TRACE_FLAGS(p) & F_TRACE_SILENT)) {
-            erts_match_set_release_result(p);
+	pam_result = erts_match_set_run_trace(p, p,
+                                              te->match_spec, args, 2,
+                                              ERTS_PAM_TMP_RESULT, &return_flags);
+	if (pam_result == am_false)
+            return;
+        if (ERTS_TRACE_FLAGS(p) & F_TRACE_SILENT) {
+            erts_match_set_release_result_trace(p, pam_result);
 	    return;
 	}
     } else
@@ -850,9 +850,9 @@ trace_send(Process *p, Eterm to, Eterm msg)
     if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif,
                           TRACE_FUN_E_SEND, operation)) {
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND,
-                operation, msg, to, pam_result);
+                           operation, msg, to, pam_result);
     }
-    erts_match_set_release_result(p);
+    erts_match_set_release_result_trace(p, pam_result);
 }
 
 /* Send {trace_ts, Pid, receive, Msg, Timestamp}
@@ -886,13 +886,13 @@ trace_receive(Process* receiver,
             args[1] = am_undefined;
         }
         args[2] = msg;
-        pam_result = erts_match_set_run(NULL, receiver,
-                                        te->match_spec, args, 3,
-                                        ERTS_PAM_TMP_RESULT, &return_flags);
-        if (is_non_value(pam_result)
-            || pam_result == am_false
-            || (ERTS_TRACE_FLAGS(receiver) & F_TRACE_SILENT)) {
-            erts_match_set_release_result(NULL);
+        pam_result = erts_match_set_run_trace(NULL, receiver,
+                                              te->match_spec, args, 3,
+                                              ERTS_PAM_TMP_RESULT, &return_flags);
+        if (pam_result == am_false)
+            return;
+        if (ERTS_TRACE_FLAGS(receiver) & F_TRACE_SILENT) {
+            erts_match_set_release_result_trace(NULL, pam_result);
             return;
         }
     } else
@@ -904,7 +904,7 @@ trace_receive(Process* receiver,
                            tnif, TRACE_FUN_T_RECEIVE,
                            am_receive, msg, THE_NON_VALUE, pam_result);
     }
-    erts_match_set_release_result(NULL);
+    erts_match_set_release_result_trace(NULL, pam_result);
 }
 
 int
@@ -1264,21 +1264,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
            may remove it, and we still want to generate a trace message */
         erts_tracer_update(&pre_ms_tracer, *tracer);
         tracer = &pre_ms_tracer;
-        pam_result = erts_match_set_run(p, p,
-                                        match_spec, args, arity,
-                                        ERTS_PAM_TMP_RESULT, &return_flags);
-        if (is_non_value(pam_result)) {
-            erts_match_set_release_result(p);
-            UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
-            ERTS_TRACER_CLEAR(&pre_ms_tracer);
-            return 0;
-        }
+        pam_result = erts_match_set_run_trace(p, p,
+                                              match_spec, args, arity,
+                                              ERTS_PAM_TMP_RESULT, &return_flags);
     }
 
     if (tracee_flags == &meta_flags) {
         /* Meta trace */
         if (pam_result == am_false) {
-            erts_match_set_release_result(p);
             UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
             ERTS_TRACER_CLEAR(&pre_ms_tracer);
             return return_flags;
@@ -1286,13 +1279,12 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
     } else {
         /* Non-meta trace */
         if (*tracee_flags & F_TRACE_SILENT) {
-            erts_match_set_release_result(p);
+            erts_match_set_release_result_trace(p, pam_result);
             UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
             ERTS_TRACER_CLEAR(&pre_ms_tracer);
             return 0;
         }
         if (pam_result == am_false) {
-            erts_match_set_release_result(p);
             UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
             ERTS_TRACER_CLEAR(&pre_ms_tracer);
             return return_flags;
@@ -1328,11 +1320,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
      * Build the trace tuple and send it to the port.
      */
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           tnif, TRACE_FUN_T_CALL, am_call, mfa_tuple, THE_NON_VALUE, pam_result);
-    erts_match_set_release_result(p);
+                           tnif, TRACE_FUN_T_CALL, am_call, mfa_tuple,
+                           THE_NON_VALUE, pam_result);
 
-    if (match_spec && tracer == &pre_ms_tracer)
-        ERTS_TRACER_CLEAR(&pre_ms_tracer);
+    if (match_spec) {
+        erts_match_set_release_result_trace(p, pam_result);
+        if (tracer == &pre_ms_tracer)
+            ERTS_TRACER_CLEAR(&pre_ms_tracer);
+    }
 
     return return_flags;
 }
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 328bfe344c..baa0a201d1 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1486,6 +1486,16 @@ do {								\
 extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA);
 Eterm erts_match_set_lint(Process *p, Eterm matchexpr); 
 extern void erts_match_set_release_result(Process* p);
+ERTS_GLB_INLINE void erts_match_set_release_result_trace(Process* p, Eterm);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE
+void erts_match_set_release_result_trace(Process* p, Eterm pam_result)
+{
+    if (is_not_immed(pam_result))
+        erts_match_set_release_result(p);
+}
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 enum erts_pam_run_flags {
     ERTS_PAM_TMP_RESULT=1,
@@ -1493,12 +1503,12 @@ enum erts_pam_run_flags {
     ERTS_PAM_CONTIGUOUS_TUPLE=4,
     ERTS_PAM_IGNORE_TRACE_SILENT=8
 };
-extern Eterm erts_match_set_run(Process *p,
-                                Process *self,
-                                Binary *mpsp,
-				Eterm *args, int num_args,
-				enum erts_pam_run_flags in_flags,
-				Uint32 *return_flags);
+extern Eterm erts_match_set_run_trace(Process *p,
+                                      Process *self,
+                                      Binary *mpsp,
+                                      Eterm *args, int num_args,
+                                      enum erts_pam_run_flags in_flags,
+                                      Uint32 *return_flags);
 extern Eterm erts_match_set_get_source(Binary *mpsp);
 extern void erts_match_prog_foreach_offheap(Binary *b,
 					    void (*)(ErlOffHeap *, void *),
-- 
cgit v1.2.3


From 6197aa2498bcb35e331c9112a70635f55047bf26 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 2 May 2016 18:40:10 +0200
Subject: erts: Fix bug in trace_pattern for 'on_load'

'on_load' is a call trace.
---
 erts/emulator/beam/erl_db_util.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 051c107d1f..95b1cd0148 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1002,10 +1002,14 @@ Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA) {
     Binary *bin;
     Uint sz;
     Eterm *hp;
-    Uint flags = DCOMP_TRACE;
+    Uint flags;
 
-    if (is_tuple(MFA)) flags |= DCOMP_CALL_TRACE;
-    if (MFA != am_receive) flags |= DCOMP_ALLOW_TRACE_OPS;
+    switch (MFA) {
+    case am_receive: flags = DCOMP_TRACE; break;
+    case am_send:    flags = DCOMP_TRACE | DCOMP_ALLOW_TRACE_OPS; break;
+    default:
+        flags = DCOMP_TRACE | DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS;
+    }
     
     bin = db_match_set_compile(p, matchexpr, flags);
     if (bin != NULL) {
-- 
cgit v1.2.3


From 54172674e71caf7da7a0b069c9bd92543e4f705d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 4 May 2016 14:45:05 +0200
Subject: erts: Add send and 'receive' to trace_info/2

to obtain match specs
---
 erts/emulator/beam/erl_bif_trace.c | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index ffc0afda58..b65c0e303f 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -81,6 +81,8 @@ static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/
 static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key);
 static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key);
 static Eterm trace_info_on_load(Process* p, Eterm key);
+static Eterm trace_info_event(Process* p, Eterm event, Eterm key);
+
 
 static void reset_bif_trace(void);
 static void setup_bif_trace(void);
@@ -814,6 +816,8 @@ Eterm trace_info_2(BIF_ALIST_2)
 
     if (What == am_on_load) {
 	res = trace_info_on_load(p, Key);
+    } else if (What == am_send || What == am_receive) {
+        res = trace_info_event(p, What, Key);
     } else if (is_atom(What) || is_pid(What) || is_port(What)) {
 	res = trace_info_pid(p, What, Key);
     } else if (is_tuple(What)) {
@@ -1303,6 +1307,42 @@ trace_info_on_load(Process* p, Eterm key)
     }
 }
 
+static Eterm
+trace_info_event(Process* p, Eterm event, Eterm key)
+{
+    ErtsTracingEvent* te;
+    Eterm retval;
+    Eterm* hp;
+
+    switch (event) {
+    case am_send:    te = erts_send_tracing;    break;
+    case am_receive: te = erts_receive_tracing; break;
+    default:
+        goto error;
+    }
+
+    if (key != am_match_spec)
+        goto error;
+
+    te = &te[erts_active_bp_ix()];
+
+    if (te->on) {
+        if (!te->match_spec)
+            retval = am_true;
+        else
+            retval = copy_object(MatchSetGetSource(te->match_spec), p);
+    }
+    else
+        retval = am_false;
+
+    hp = HAlloc(p, 3);
+    return TUPLE2(hp, key, retval);
+
+ error:
+    BIF_ERROR(p, BADARG);
+}
+
+
 #undef FUNC_TRACE_NOEXIST
 #undef FUNC_TRACE_UNTRACED
 #undef FUNC_TRACE_GLOBAL_TRACE
-- 
cgit v1.2.3


From 0371d8e6fb2545843e73125eb2d0edaf8fd3e778 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 4 May 2016 21:03:16 +0200
Subject: erts: Refactor ERTS_ALC_INFO_A_* constants

as an enum and replace _MAX with _END
---
 erts/emulator/beam/erl_alloc.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index f571515def..40b36e4b6c 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -137,10 +137,12 @@ static ErtsAllocatorState_t exec_alloc_state;
 #endif
 static ErtsAllocatorState_t test_alloc_state;
 
-#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1)
-#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2)
-#define ERTS_ALC_INFO_A_ERTS_MMAP  (ERTS_ALC_A_MAX + 3)
-#define ERTS_ALC_INFO_A_MAX        ERTS_ALC_INFO_A_ERTS_MMAP
+enum {
+    ERTS_ALC_INFO_A_ALLOC_UTIL = ERTS_ALC_A_MAX + 1,
+    ERTS_ALC_INFO_A_MSEG_ALLOC,
+    ERTS_ALC_INFO_A_ERTS_MMAP,
+    ERTS_ALC_INFO_A_END
+};
 
 typedef struct {
     erts_smp_atomic32_t refc;
@@ -150,7 +152,7 @@ typedef struct {
     Process *proc;
     Eterm ref;
     Eterm ref_heap[REF_THING_SIZE];
-    int allocs[ERTS_ALC_INFO_A_MAX - ERTS_ALC_A_MIN + 1 + 1];
+    int allocs[ERTS_ALC_INFO_A_END - ERTS_ALC_A_MIN + 1];
 } ErtsAllocInfoReq;
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
@@ -3359,7 +3361,7 @@ erts_request_alloc_info(struct process *c_p,
 			int internal)
 {
     ErtsAllocInfoReq *air = aireq_alloc();
-    Eterm req_ai[ERTS_ALC_INFO_A_MAX+1] = {0};
+    Eterm req_ai[ERTS_ALC_INFO_A_END] = {0};
     Eterm alist;
     Eterm *hp;
     int airix = 0, ai;
-- 
cgit v1.2.3


From c93da5e4fb98f92b9693586dda2ce1bf2d05ade1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 4 May 2016 21:13:08 +0200
Subject: erts: Show exec_alloc as disabled when not used

---
 erts/emulator/beam/erl_alloc.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 40b36e4b6c..d7919b9db0 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -141,6 +141,7 @@ enum {
     ERTS_ALC_INFO_A_ALLOC_UTIL = ERTS_ALC_A_MAX + 1,
     ERTS_ALC_INFO_A_MSEG_ALLOC,
     ERTS_ALC_INFO_A_ERTS_MMAP,
+    ERTS_ALC_INFO_A_DISABLED_EXEC,  /* fake a disabled "exec_alloc" */
     ERTS_ALC_INFO_A_END
 };
 
@@ -3262,6 +3263,12 @@ reply_alloc_info(void *vair)
                                             am_false);
 #endif
 		    break;
+#ifndef ERTS_ALC_A_EXEC
+		case ERTS_ALC_INFO_A_DISABLED_EXEC:
+		    alloc_atom = erts_bld_atom(hpp, szp, "exec_alloc");
+		    ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, am_false);
+		    break;
+#endif
 		default:
 		    alloc_atom = erts_bld_atom(hpp, szp,
 					       (char *) ERTS_ALC_A2AD(ai));
@@ -3289,7 +3296,8 @@ reply_alloc_info(void *vair)
 	    switch (ai) {
 	    case ERTS_ALC_A_SYSTEM:
             case ERTS_ALC_INFO_A_ALLOC_UTIL:
-            case ERTS_ALC_INFO_A_ERTS_MMAP:
+	    case ERTS_ALC_INFO_A_ERTS_MMAP:
+	    case ERTS_ALC_INFO_A_DISABLED_EXEC:
 		break;
 	    case ERTS_ALC_INFO_A_MSEG_ALLOC:
 #if HAVE_ERTS_MSEG && defined(ERTS_SMP)
@@ -3401,6 +3409,12 @@ erts_request_alloc_info(struct process *c_p,
             ai = ERTS_ALC_INFO_A_ERTS_MMAP;
             goto save_alloc;
         }
+#ifndef ERTS_ALC_A_EXEC
+	if (erts_is_atom_str("exec_alloc", alloc, 0)) {
+	    ai = ERTS_ALC_INFO_A_DISABLED_EXEC;
+	    goto save_alloc;
+	}
+#endif
 	if (erts_is_atom_str("alloc_util", alloc, 0)) {
 	    ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
 	save_alloc:
-- 
cgit v1.2.3


From 98855709622835602b4c0a2d37e3f06ceb003af7 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikael.pettersson@klarna.com>
Date: Sun, 24 Apr 2016 10:58:11 +0200
Subject: erts_new_mso_binary(): do not truncate len

---
 erts/emulator/beam/binary.c | 2 +-
 erts/emulator/beam/global.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index cfd8fbe2f5..071a356260 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -117,7 +117,7 @@ new_binary(Process *p, byte *buf, Uint len)
  * When heap binary is not desired...
  */
 
-Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
+Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len)
 {
     ProcBin* pb;
     Binary* bptr;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index a7bc990deb..c8cf7c6191 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -946,7 +946,7 @@ ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q) {
 
 void erts_emasculate_writable_binary(ProcBin* pb);
 Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap);
-Eterm erts_new_mso_binary(Process*, byte*, int);
+Eterm erts_new_mso_binary(Process*, byte*, Uint);
 Eterm new_binary(Process*, byte*, Uint);
 Eterm erts_realloc_binary(Eterm bin, size_t size);
 
-- 
cgit v1.2.3


From d053fed5d3f546b2f1f19e16a60c6330f8526c95 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikael.pettersson@klarna.com>
Date: Tue, 26 Apr 2016 09:13:45 +0200
Subject: do not limit heap fragments to 4 giga-words

- struct erl_heap_fragment: use Uint for sizes
- erl_nif.c: use size_t for sizes
- erts_heap_frag_shrink(): remove now redundant cast
---
 erts/emulator/beam/erl_message.h |  4 ++--
 erts/emulator/beam/erl_nif.c     | 10 +++++-----
 erts/emulator/beam/erl_process.h |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 608cf552a2..60cf9a24e1 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -112,8 +112,8 @@ typedef struct erl_heap_fragment ErlHeapFragment;
 struct erl_heap_fragment {
     ErlHeapFragment* next;	/* Next heap fragment */
     ErlOffHeap off_heap;	/* Offset heap data. */
-    unsigned alloc_size;	/* Size in (half)words of mem */
-    unsigned used_size;         /* With terms to be moved to heap by GC */
+    Uint alloc_size;		/* Size in (half)words of mem */
+    Uint used_size;		/* With terms to be moved to heap by GC */
     Eterm mem[1];		/* Data */
 };
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 941f44b9ec..d52bfbc5d9 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -79,9 +79,9 @@ void dtrace_nifenv_str(ErlNifEnv *, char *);
 #endif
 
 #define MIN_HEAP_FRAG_SZ 200
-static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp);
+static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp);
 
-static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need)
+static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, size_t need)
 {
     Eterm* hp = env->hp;
     env->hp += need;
@@ -91,7 +91,7 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need)
     return alloc_heap_heavy(env, need, hp);
 }
 
-static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
+static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp)
 {
     env->hp = hp;
     if (env->heap_frag == NULL) {
@@ -112,7 +112,7 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
 }
 
 #if SIZEOF_LONG != ERTS_SIZEOF_ETERM
-static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need)
+static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need)
 {
     if (env->hp + may_need > env->hp_end) {
 	alloc_heap_heavy(env, may_need, env->hp);
@@ -1010,7 +1010,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
     Eterm orig;
     Uint offset, bit_offset, bit_size; 
 #ifdef DEBUG
-    unsigned src_size;
+    size_t src_size;
 
     ASSERT(is_binary(bin_term));
     src_size = binary_size(bin_term);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 1c01d705a7..0f69ddeed8 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1291,7 +1291,7 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
 {
     ErlHeapFragment* hf = MBUF(p);
 
-    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));
+    ASSERT(hf!=NULL && (hp - hf->mem < hf->alloc_size));
 
     hf->used_size = hp - hf->mem;
 }	
-- 
cgit v1.2.3


From 325f700a35ecf3e50b8d679f504edbdd5a4ec1ab Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikael.pettersson@klarna.com>
Date: Sun, 24 Apr 2016 11:17:19 +0200
Subject: erl_unicode.c: fix integer truncation problems

- use Sint for 'left' and Uint for 'pos'
- cost_to_proc(): adjust types, remove unused return value
- copy_utf8_bin(): return Uint
- characters_to_utf8_trap(): remove harmful cast
- unicode_characters_to_binary_2(): correct types in debug code
- erts_convert_filename_to_encoding(): remove useless cast
---
 erts/emulator/beam/erl_unicode.c | 58 +++++++++++++++++++++-------------------
 1 file changed, 30 insertions(+), 28 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 3c3536c021..bd5e1482fb 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -55,7 +55,7 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
 					 Uint num_processed_bytes,
 					 Uint num_bytes_to_process, 
 					 Uint num_resulting_chars, 
-					 int state, int left,
+					 int state, Sint left,
 					 Eterm tail);
 static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3);
 static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3);
@@ -173,12 +173,13 @@ static ERTS_INLINE int allowed_iterations(Process *p)
     else
 	return tmp;
 }
-static ERTS_INLINE int cost_to_proc(Process *p, int cost)
+
+static ERTS_INLINE void cost_to_proc(Process *p, Sint cost)
 {
-    int x = (cost / LOOP_FACTOR);
+    Sint x = (cost / LOOP_FACTOR);
     BUMP_REDS(p,x);
-    return x;
 }
+
 static ERTS_INLINE int simple_loops_to_common(int cost)
 {
     int factor = (LOOP_FACTOR_SIMPLE / LOOP_FACTOR);
@@ -243,14 +244,15 @@ static int utf8_len(byte first)
     return -1;
 }
 
-static int copy_utf8_bin(byte *target, byte *source, Uint size, 
-			 byte *leftover, int *num_leftovers, 
-			 byte **err_pos, Uint *characters) {
-    int copied = 0;
+static Uint copy_utf8_bin(byte *target, byte *source, Uint size,
+			  byte *leftover, int *num_leftovers,
+			  byte **err_pos, Uint *characters)
+{
+    Uint copied = 0;
     if (leftover != NULL && *num_leftovers) {
 	int need = utf8_len(leftover[0]);
 	int from_source = need - (*num_leftovers);
-	int c;
+	Uint c;
 	byte *tmp_err_pos = NULL;
 	ASSERT(need > 0);
 	ASSERT(from_source > 0);
@@ -502,8 +504,8 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
 }
     
     
-static Eterm do_build_utf8(Process *p, Eterm ioterm, int *left, int latin1,
-			   byte *target, int *pos, Uint *characters, int *err, 
+static Eterm do_build_utf8(Process *p, Eterm ioterm, Sint *left, int latin1,
+			   byte *target, Uint *pos, Uint *characters, int *err,
 			   byte *leftover, int *num_leftovers)
 {
     int c;
@@ -573,7 +575,7 @@ static Eterm do_build_utf8(Process *p, Eterm ioterm, int *left, int latin1,
 	}
 
 	if (!latin1) {
-	    int num;
+	    Uint num;
 	    byte *err_pos = NULL;
 	    num = copy_utf8_bin(target + (*pos), bytes, 
 				size, leftover, num_leftovers,&err_pos,characters);
@@ -804,7 +806,7 @@ static int check_leftovers(byte *source, int size)
 	
 	 
 
-static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,int pos,
+static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,Uint pos,
 			       Eterm rest_term,int err,
 			       byte *leftover,int num_leftovers,Eterm latin1)
 {
@@ -859,8 +861,8 @@ static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3)
 #endif
     byte* bytes;
     Eterm rest_term;
-    int left, sleft;
-    int pos;
+    Sint left, sleft;
+    Uint pos;
     int err;
     byte leftover[4]; /* used for temp buffer too, 
 			 otherwise 3 bytes would have been enough */
@@ -874,7 +876,7 @@ static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3)
     real_bin = binary_val(BIF_ARG_1);
     ASSERT(*real_bin == HEADER_PROC_BIN);
 #endif
-    pos = (int) binary_size(BIF_ARG_1);
+    pos = binary_size(BIF_ARG_1);
     bytes = binary_bytes(BIF_ARG_1);
     sleft = left = allowed_iterations(BIF_P);
     err = 0;
@@ -934,9 +936,9 @@ BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
     int latin1;
     Eterm bin;
     byte *bytes;
-    int pos;
+    Uint pos;
     int err;
-    int left, sleft;
+    Sint left, sleft;
     Eterm rest_term, subject;
     byte leftover[4]; /* used for temp buffer too, o
 			 therwise 3 bytes would have been enough */
@@ -999,7 +1001,7 @@ BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
 	    byte *t = NULL;
 	    Uint sz = binary_size(bin);
 	    byte *by = erts_get_aligned_binary_bytes(bin,&t);
-	    int i;
+	    Uint i;
 	    erts_printf("<<");
 	    for (i = 0;i < sz; ++i) {
 		erts_printf((i == sz -1) ? "0x%X" : "0x%X, ", (unsigned) by[i]);
@@ -1007,7 +1009,7 @@ BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
 	    erts_printf(">>: ");
 	    erts_free_aligned_binary_bytes(t);
 	}
-	erts_printf("%d - %d = %d\n",sleft,left,sleft - left);
+	erts_printf("%ld - %ld = %ld\n", sleft, left, sleft - left);
     }
 #endif
     cost_to_proc(BIF_P, sleft - left); 
@@ -1015,10 +1017,10 @@ BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
 			     leftover,num_leftovers,BIF_ARG_2);
 }
 
-static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint characters,
+static BIF_RETTYPE build_list_return(Process *p, byte *bytes, Uint pos, Uint characters,
 				     Eterm rest_term, int err,
 				     byte *leftover, int num_leftovers,
-				     Eterm latin1, int left)
+				     Eterm latin1, Sint left)
 {
     Eterm *hp;
     
@@ -1070,11 +1072,11 @@ static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3)
 {
     RestartContext *rc;
     byte* bytes;
-    int pos;
+    Uint pos;
     Uint characters;
     int err;
     Eterm rest_term;
-    int left, sleft;
+    Sint left, sleft;
 
     int latin1 = 0;
     byte leftover[4]; /* used for temp buffer too, 
@@ -1107,9 +1109,9 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2)
     int latin1;
     Uint characters = 0;
     byte *bytes;
-    int pos;
+    Uint pos;
     int err;
-    int left, sleft;
+    Sint left, sleft;
     Eterm rest_term;
     byte leftover[4]; /* used for temp buffer too, o
 			 therwise 3 bytes would have been enough */
@@ -1541,7 +1543,7 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
 					 Uint num_processed_bytes,
 					 Uint num_bytes_to_process, 
 					 Uint num_resulting_chars, 
-					 int state, int left,
+					 int state, Sint left,
 					 Eterm tail) 
 {
     Uint num_built; /* characters */
@@ -2016,7 +2018,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	    ++need;
 	}
 	if (used) 
-	    *used = (Sint) need;
+	    *used = need;
 	if (need+extra > statbuf_size) {
 	    name_buf = (char *) erts_alloc(alloc_type, need+extra);
 	} else {
-- 
cgit v1.2.3


From e146a3eec5a2d384260aa8829777c89eaab09cbd Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 21 Apr 2016 18:36:45 +0200
Subject: erts: Implement max_heap_size process flag

The max_heap_size process flag can be used to limit the
growth of a process heap by killing it before it becomes
too large to handle. It is possible to set the maximum
using the `erl +hmax` option, `system_flag(max_heap_size, ...)`,
`spawn_opt(Fun, [{max_heap_size, ...}])` and
`process_flag(max_heap_size, ...)`.

It is possible to configure the behaviour of the process
when the maximum heap size is reached. The process may be
sent an untrappable exit signal with reason kill and/or
send an error_logger message with details on the process
state. A new trace event called gc_max_heap_size is
also triggered for the garbage_collection trace flag
when the heap grows larger than the configured size.

If kill and error_logger are disabled, it is still
possible to see that the maximum has been reached by
doing garbage collection tracing on the process.

The heap size is defined as the sum of the heap memory
that the process is currently using. This includes
all generational heaps, the stack, any messages that
are considered to be part of the heap and any extra
memory the garbage collector may need during collection.

In the current implementation this means that when a process
is set using on_heap message queue data mode, the messages
that are in the internal message queue are counted towards
this value. For off_heap, only matched messages count towards
the size of the heap. For mixed, it depends on race conditions
within the VM whether a message is part of the heap or not.

Below is an example run of the new behaviour:

Eshell V8.0  (abort with ^G)
1> f(P),P = spawn_opt(fun() -> receive ok -> ok end end, [{max_heap_size, 512}]).
<0.60.0>
2> erlang:trace(P, true, [garbage_collection, procs]).
1
3> [P ! lists:duplicate(M,M) || M <- lists:seq(1,15)],ok.
ok
4>
=ERROR REPORT==== 26-Apr-2016::16:25:10 ===
     Process:          <0.60.0>
     Context:          maximum heap size reached
     Max heap size:    512
     Total heap size:  723
     Kill:             true
     Error Logger:     true
     GC Info:          [{old_heap_block_size,0},
                        {heap_block_size,609},
                        {mbuf_size,145},
                        {recent_size,0},
                        {stack_size,9},
                        {old_heap_size,0},
                        {heap_size,211},
                        {bin_vheap_size,0},
                        {bin_vheap_block_size,46422},
                        {bin_old_vheap_size,0},
                        {bin_old_vheap_block_size,46422}]
flush().
Shell got {trace,<0.60.0>,gc_start,
                 [{old_heap_block_size,0},
                  {heap_block_size,233},
                  {mbuf_size,145},
                  {recent_size,0},
                  {stack_size,9},
                  {old_heap_size,0},
                  {heap_size,211},
                  {bin_vheap_size,0},
                  {bin_vheap_block_size,46422},
                  {bin_old_vheap_size,0},
                  {bin_old_vheap_block_size,46422}]}
Shell got {trace,<0.60.0>,gc_max_heap_size,
                 [{old_heap_block_size,0},
                  {heap_block_size,609},
                  {mbuf_size,145},
                  {recent_size,0},
                  {stack_size,9},
                  {old_heap_size,0},
                  {heap_size,211},
                  {bin_vheap_size,0},
                  {bin_vheap_block_size,46422},
                  {bin_old_vheap_size,0},
                  {bin_old_vheap_block_size,46422}]}
Shell got {trace,<0.60.0>,exit,killed}
---
 erts/emulator/beam/atom.names     |   2 +
 erts/emulator/beam/beam_emu.c     |  43 +++++++
 erts/emulator/beam/bif.c          |  52 ++++++++
 erts/emulator/beam/erl_bif_info.c |  49 ++++++--
 erts/emulator/beam/erl_gc.c       | 246 +++++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_gc.h       |   4 +-
 erts/emulator/beam/erl_init.c     |  54 ++++++++-
 erts/emulator/beam/erl_process.c  |   7 +-
 erts/emulator/beam/erl_process.h  |  12 ++
 erts/emulator/beam/erl_trace.c    |  21 ++--
 erts/emulator/beam/erl_trace.h    |   2 +-
 erts/emulator/beam/erl_vm.h       |   3 +
 12 files changed, 457 insertions(+), 38 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 3022c0a99a..8f65e71531 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -275,6 +275,7 @@ atom garbage_collection_info
 atom gc_end
 atom gc_major_end
 atom gc_major_start
+atom gc_max_heap_size
 atom gc_minor_end
 atom gc_minor_start
 atom gc_start
@@ -366,6 +367,7 @@ atom match_spec
 atom match_spec_result
 atom max
 atom maximum
+atom max_heap_size
 atom max_tables max_processes
 atom mbuf_size
 atom md5
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 99eb26bec5..a88fea3802 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1704,6 +1704,14 @@ void process_main(void)
      BeamInstr *next;
      Eterm result;
 
+     if (!(FCALLS > 0 || FCALLS > neg_o_reds)) {
+         /* If we have run out of reductions, we do a context
+            switch before calling the bif */
+         c_p->arity = 2;
+         c_p->current = NULL;
+         goto context_switch3;
+     }
+
      PRE_BIF_SWAPOUT(c_p);
      c_p->fcalls = FCALLS - 1;
      result = erl_send(c_p, r(0), x(1));
@@ -2810,6 +2818,15 @@ do {						\
 	BeamInstr *next;
 	ErlHeapFragment *live_hf_end;
 
+
+        if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) {
+            /* If we have run out of reductions, we do a context
+               switch before calling the bif */
+            c_p->arity = ((Export *)Arg(0))->code[2];
+            c_p->current = NULL;
+            goto context_switch3;
+        }
+
         if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
             if (GET_BIF_MODULE(Arg(0)) == am_ets) {
                 ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
@@ -3338,10 +3355,19 @@ do {						\
  context_switch2:		/* Entry for fun calls. */
     c_p->current = I-3;		/* Pointer to Mod, Func, Arity */
 
+ context_switch3:
+
  {
      Eterm* argp;
      int i;
 
+     if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
+         c_p->i = beam_exit;
+         c_p->arity = 0;
+         c_p->current = NULL;
+         goto do_schedule;
+     }
+
      /*
       * Make sure that there is enough room for the argument registers to be saved.
       */
@@ -3509,6 +3535,14 @@ do {						\
 	    BifFunction vbf;
 	    ErlHeapFragment *live_hf_end;
 
+            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+                /* If we have run out of reductions, we do a context
+                   switch before calling the nif */
+                c_p->arity = I[-1];
+                c_p->current = NULL;
+                goto context_switch3;
+            }
+
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
 	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
@@ -3551,6 +3585,15 @@ do {						\
 	     * code[3]: &&apply_bif
 	     * code[4]: Function pointer to BIF function
 	     */
+
+            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+                /* If we have run out of reductions, we do a context
+                   switch before calling the bif */
+                c_p->arity = I[-1];
+                c_p->current = NULL;
+                goto context_switch3;
+            }
+
             if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
                 if ((Eterm)I[-3] == am_ets) {
                     ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index eed0702688..483c5320d7 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -44,6 +44,7 @@
 #include "erl_ptab.h"
 #include "erl_bits.h"
 #include "erl_bif_unique.h"
+#include "erl_map.h"
 #include "erl_msacc.h"
 
 Export *erts_await_result;
@@ -880,6 +881,8 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
     so.flags          = erts_default_spo_flags|SPO_USE_ARGS;
     so.min_heap_size  = H_MIN_SIZE;
     so.min_vheap_size = BIN_VH_MIN_SIZE;
+    so.max_heap_size  = H_MAX_SIZE;
+    so.max_heap_flags = H_MAX_FLAGS;
     so.priority       = PRIORITY_NORMAL;
     so.max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
     so.scheduler      = 0;
@@ -937,6 +940,9 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
 		} else {
 		    so.min_heap_size = erts_next_heap_size(min_heap_size, 0);
 		}
+            } else if (arg == am_max_heap_size) {
+                if (!erts_max_heap_size(val, &so.max_heap_size, &so.max_heap_flags))
+                    goto error;
 	    } else if (arg == am_min_bin_vheap_size && is_small(val)) {
 		Sint min_vheap_size = signed_val(val);
 		if (min_vheap_size < 0) {
@@ -970,6 +976,10 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
 	goto error;
     }
 
+    if (so.max_heap_size != 0 && so.max_heap_size < so.min_heap_size) {
+        goto error;
+    }
+
     /*
      * Spawn the process.
      */
@@ -1731,6 +1741,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        }
        BIF_RET(old_value);
    }
+   else if (BIF_ARG_1 == am_max_heap_size) {
+       Eterm *hp;
+       Uint sz = 0, max_heap_size, max_heap_flags;
+
+       if (!erts_max_heap_size(BIF_ARG_2, &max_heap_size, &max_heap_flags))
+           goto error;
+
+       if ((max_heap_size < MIN_HEAP_SIZE(BIF_P) && max_heap_size != 0))
+	   goto error;
+
+       erts_max_heap_size_map(MAX_HEAP_SIZE_GET(BIF_P), MAX_HEAP_SIZE_FLAGS_GET(BIF_P), NULL, &sz);
+       hp = HAlloc(BIF_P, sz);
+       old_value = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(BIF_P), MAX_HEAP_SIZE_FLAGS_GET(BIF_P), &hp, NULL);
+       MAX_HEAP_SIZE_SET(BIF_P, max_heap_size);
+       MAX_HEAP_SIZE_FLAGS_SET(BIF_P, max_heap_flags);
+       BIF_RET(old_value);
+   }
    else if (BIF_ARG_1 == am_message_queue_data) {
        old_value = erts_change_message_queue_management(BIF_P, BIF_ARG_2);
        if (is_non_value(old_value))
@@ -4350,6 +4377,31 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(make_small(oval));
+    } else if (BIF_ARG_1 == am_max_heap_size) {
+
+        Eterm *hp, old_value;
+        Uint sz = 0, max_heap_size, max_heap_flags;
+
+        if (!erts_max_heap_size(BIF_ARG_2, &max_heap_size, &max_heap_flags))
+            goto error;
+
+        if (max_heap_size < H_MIN_SIZE && max_heap_size != 0)
+            goto error;
+
+        erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, NULL, &sz);
+        hp = HAlloc(BIF_P, sz);
+        old_value = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL);
+
+        erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        erts_smp_thr_progress_block();
+
+        H_MAX_SIZE = max_heap_size;
+        H_MAX_FLAGS = max_heap_flags;
+
+        erts_smp_thr_progress_unblock();
+        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+
+        BIF_RET(old_value);
     } else if (BIF_ARG_1 == am_display_items) {
 	int oval = display_items;
 	if (!is_small(BIF_ARG_2) || (n = signed_val(BIF_ARG_2)) < 0) {
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 85cadafebc..da480f8fce 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -45,6 +45,7 @@
 #include "erl_async.h"
 #include "erl_thr_progress.h"
 #include "erl_bif_unique.h"
+#include "erl_map.h"
 #define ERTS_PTAB_WANT_DEBUG_FUNCS__
 #include "erl_ptab.h"
 #ifdef HIPE
@@ -594,6 +595,7 @@ static Eterm pi_args[] = {
     am_suspending,
     am_min_heap_size,
     am_min_bin_vheap_size,
+    am_max_heap_size,
     am_current_location,
     am_current_stacktrace,
     am_message_queue_data,
@@ -643,10 +645,11 @@ pi_arg2ix(Eterm arg)
     case am_suspending:				return 26;
     case am_min_heap_size:			return 27;
     case am_min_bin_vheap_size:			return 28;
-    case am_current_location:			return 29;
-    case am_current_stacktrace:			return 30;
-    case am_message_queue_data:			return 31;
-    case am_garbage_collection_info:		return 32;
+    case am_max_heap_size:			return 29;
+    case am_current_location:			return 30;
+    case am_current_stacktrace:			return 31;
+    case am_message_queue_data:			return 32;
+    case am_garbage_collection_info:		return 33;
     default:					return -1;
     }
 }
@@ -1348,6 +1351,18 @@ process_info_aux(Process *BIF_P,
 	break;
     }
 
+    case am_max_heap_size: {
+	Uint hsz = 3;
+	(void) erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp),
+                                      MAX_HEAP_SIZE_FLAGS_GET(rp),
+                                      NULL, &hsz);
+	hp = HAlloc(BIF_P, hsz);
+	res = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp),
+                                     MAX_HEAP_SIZE_FLAGS_GET(rp),
+                                     &hp, NULL);
+	break;
+    }
+
     case am_total_heap_size: {
 	ErtsMessage *mp;
 	Uint total_heap_size;
@@ -1391,8 +1406,12 @@ process_info_aux(Process *BIF_P,
     case am_garbage_collection: {
         DECL_AM(minor_gcs);
         Eterm t;
+        Uint map_sz = 0;
+
+        erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), NULL, &map_sz);
 
-	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3); /* last "3" is for outside tuple */
+	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + map_sz + 3);
+        /* last "3" is for outside tuple */
 
 	t = TUPLE2(hp, AM_minor_gcs, make_small(GEN_GCS(rp))); hp += 3;
 	res = CONS(hp, t, NIL); hp += 2;
@@ -1403,6 +1422,11 @@ process_info_aux(Process *BIF_P,
 	res = CONS(hp, t, res); hp += 2;
 	t = TUPLE2(hp, am_min_bin_vheap_size, make_small(MIN_VHEAP_SIZE(rp))); hp += 3;
 	res = CONS(hp, t, res); hp += 2;
+
+        t = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), &hp, NULL);
+
+	t = TUPLE2(hp, am_max_heap_size, t); hp += 3;
+	res = CONS(hp, t, res); hp += 2;
 	break;
     }
 
@@ -1412,12 +1436,12 @@ process_info_aux(Process *BIF_P,
         if (rp == BIF_P) {
             sz += ERTS_PROCESS_GC_INFO_MAX_SIZE;
         } else {
-            erts_process_gc_info(rp, &sz, NULL);
+            erts_process_gc_info(rp, &sz, NULL, 0, 0);
             sz += 3;
         }
 
         hp = HAlloc(BIF_P, sz);
-        res = erts_process_gc_info(rp, &actual_sz, &hp);
+        res = erts_process_gc_info(rp, &actual_sz, &hp, 0, 0);
 
         /* We may have some extra space, fill with 0 tuples */
         if (actual_sz <= sz - 3) {
@@ -2173,7 +2197,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     } else if (BIF_ARG_1 == am_garbage_collection){
 	Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
 	Eterm tup;
-	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2);
+	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2);
 
 	tup = TUPLE2(hp, am_fullsweep_after, make_small(val)); hp += 3;
 	res = CONS(hp, tup, NIL); hp += 2;
@@ -2184,6 +2208,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	tup = TUPLE2(hp, am_min_bin_vheap_size, make_small(BIN_VH_MIN_SIZE)); hp += 3;
 	res = CONS(hp, tup, res); hp += 2;
 
+	tup = TUPLE2(hp, am_max_heap_size, make_small(H_MAX_SIZE)); hp += 3;
+	res = CONS(hp, tup, res); hp += 2;
+
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_fullsweep_after){
 	Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
@@ -2194,6 +2221,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, am_min_heap_size,make_small(H_MIN_SIZE));
 	BIF_RET(res);
+    } else if (BIF_ARG_1 == am_max_heap_size) {
+        Uint sz = 0;
+        erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, NULL, &sz);
+	hp = HAlloc(BIF_P, sz);
+	res = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL);
+	BIF_RET(res);
     } else if (BIF_ARG_1 == am_min_bin_vheap_size) {
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, am_min_bin_vheap_size,make_small(BIN_VH_MIN_SIZE));
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index bed7e668d7..fa0782867d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -41,6 +41,7 @@
 #endif
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
+#include "dist.h"
 
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
@@ -146,7 +147,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
 static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
 static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size);
 static void move_msgq_to_heap(Process *p);
-
+static int reached_max_heap_size(Process *p, Uint total_heap_size,
+                                 Uint extra_heap_size, Uint extra_old_heap_size);
 static void init_gc_info(ErtsGCInfo *gcip);
 
 #ifdef HARDDEBUG
@@ -577,9 +579,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 		int need, Eterm* objv, int nobj, int fcalls)
 {
     Uint reclaimed_now = 0;
+    Eterm gc_trace_end_tag;
     int reds;
     ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
     ErtsSchedulerData *esdp;
+    erts_aint32_t state;
     ERTS_MSACC_PUSH_STATE_M();
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
@@ -588,7 +592,9 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
 	   >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
 
-    if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
+    state = erts_smp_atomic32_read_nob(&p->state);
+
+    if (p->flags & (F_DISABLE_GC|F_DELAY_GC) || state & ERTS_PSFLG_EXITING)
 	return delay_garbage_collection(p, live_hf_end, need, fcalls);
 
     if (p->abandoned_heap)
@@ -623,28 +629,28 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) {
         if (IS_TRACED_FL(p, F_TRACE_GC)) {
-            trace_gc(p, am_gc_minor_start, need);
+            trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE);
         }
         DTRACE2(gc_minor_start, pidbuf, need);
         reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
         DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
-        if (IS_TRACED_FL(p, F_TRACE_GC)) {
-            trace_gc(p, am_gc_minor_end, reclaimed_now);
-        }
-        if (reds < 0)
+        if (reds == -1) {
+            if (IS_TRACED_FL(p, F_TRACE_GC)) {
+                trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE);
+            }
             goto do_major_collection;
+        }
+        gc_trace_end_tag = am_gc_minor_end;
     } else {
 do_major_collection:
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL);
         if (IS_TRACED_FL(p, F_TRACE_GC)) {
-            trace_gc(p, am_gc_major_start, need);
+            trace_gc(p, am_gc_major_start, need, THE_NON_VALUE);
         }
         DTRACE2(gc_major_start, pidbuf, need);
         reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
         DTRACE2(gc_major_end, pidbuf, reclaimed_now);
-        if (IS_TRACED_FL(p, F_TRACE_GC)) {
-            trace_gc(p, am_gc_major_end, reclaimed_now);
-        }
+        gc_trace_end_tag = am_gc_major_end;
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC);
     }
 
@@ -660,6 +666,33 @@ do_major_collection:
 
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 
+    /* Max heap size has been reached and the process was configured
+       to be killed, so we kill it and set it in a delayed garbage
+       collecting state. There should be no gc_end trace or
+       long_gc/large_gc triggers when this happens as process was
+       killed before a GC could be done. */
+    if (reds == -2) {
+        ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL;
+
+        erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+        erts_send_exit_signal(p, p->common.id, p, &locks,
+                              am_kill, NIL, NULL, 0);
+        erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+
+        /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so
+           we have to remove it after the signal is sent */
+        erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
+        /* We have to make sure that we have space for need on the heap */
+        return delay_garbage_collection(p, live_hf_end, need, fcalls);
+    }
+
+    erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
+    if (IS_TRACED_FL(p, F_TRACE_GC)) {
+        trace_gc(p, gc_trace_end_tag, reclaimed_now, THE_NON_VALUE);
+    }
+
     if (erts_system_monitor_long_gc != 0) {
 	ErtsMonotonicTime end_time;
 	Uint gc_time;
@@ -761,6 +794,7 @@ erts_garbage_collect_hibernate(Process* p)
 
 
     heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz;
+
     heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP,
 				    sizeof(Eterm)*heap_size);
     htop = heap;
@@ -1030,6 +1064,34 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
     Uint mature_size = p->high_water - mature;
     Uint size_before = young_gen_usage(p);
 
+    /*
+     * Check if we have gone past the max heap size limit
+     */
+
+    if (MAX_HEAP_SIZE_GET(p)) {
+        Uint heap_size = size_before,
+            /* Note that we also count the un-allocated area
+               in between the stack and heap */
+            stack_size = HEAP_END(p) - HEAP_TOP(p),
+            extra_heap_size,
+            extra_old_heap_size = 0;
+
+        /* Add potential old heap size */
+        if (OLD_HEAP(p) == NULL && mature_size != 0) {
+            extra_old_heap_size = erts_next_heap_size(size_before, 1);
+            heap_size += extra_old_heap_size;
+        } else if (OLD_HEAP(p))
+            heap_size += OLD_HEND(p) - OLD_HEAP(p);
+
+        /* Add potential new young heap size */
+        extra_heap_size = next_heap_size(p, stack_size + size_before, 0);
+        heap_size += extra_heap_size;
+
+        if (heap_size > MAX_HEAP_SIZE_GET(p))
+            if (reached_max_heap_size(p, heap_size, extra_heap_size, extra_old_heap_size))
+                return -2;
+    }
+
     /*
      * Allocate an old heap if we don't have one and if we'll need one.
      */
@@ -1139,6 +1201,16 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
 	ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
 	ASSERT(MBUF(p) == NULL);
 
+        /* The heap usage during GC should be larger than what we end up
+           after a GC, even if we grow it. If this assertion is not true
+           we have to check size in grow_new_heap and potentially kill the
+           process from there */
+        ASSERT(!MAX_HEAP_SIZE_GET(p) ||
+               !(MAX_HEAP_SIZE_FLAGS_GET(p) & MAX_HEAP_SIZE_KILL) ||
+               MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p) +
+                                       (OLD_HEND(p) - OLD_HEAP(p)) +
+                                       (HEAP_END(p) - HEAP_TOP(p))));
+
 	return gc_cost(size_after, adjust_size);
     }
 
@@ -1457,6 +1529,25 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     if (new_sz == HEAP_SIZE(p) && FLAGS(p) & F_HEAP_GROW) {
         new_sz = next_heap_size(p, HEAP_SIZE(p), 1);
     }
+
+
+    if (MAX_HEAP_SIZE_GET(p)) {
+        Uint heap_size = size_before;
+
+        /* Add unused space in old heap */
+        heap_size += OLD_HEND(p) - OLD_HTOP(p);
+
+        /* Add stack + unused space in young heap */
+        heap_size += HEAP_END(p) - HEAP_TOP(p);
+
+        /* Add size of new young heap */
+        heap_size += new_sz;
+
+        if (MAX_HEAP_SIZE_GET(p) < heap_size)
+            if (reached_max_heap_size(p, heap_size, new_sz, 0))
+                return -2;
+    }
+
     FLAGS(p) &= ~(F_HEAP_GROW|F_NEED_FULLSWEEP);
     n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 						sizeof(Eterm)*new_sz);
@@ -2986,7 +3077,9 @@ erts_gc_info_request(Process *c_p)
 }
 
 Eterm
-erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp)
+erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp,
+                     Uint extra_heap_block,
+                     Uint extra_old_heap_block_size)
 {
     ERTS_DECL_AM(bin_vheap_size);
     ERTS_DECL_AM(bin_vheap_block_size);
@@ -3009,8 +3102,9 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp)
         AM_bin_old_vheap_block_size
     };
     UWord values[] = {
-        OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0,
-        HEAP_SIZE(p),
+        OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) + extra_old_heap_block_size
+                    : extra_old_heap_block_size,
+        HEAP_SIZE(p) + extra_heap_block,
         MBUF_SIZE(p),
         HIGH_WATER(p) - HEAP_START(p),
         STACK_START(p) - p->stop,
@@ -3056,6 +3150,130 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp)
     return res;
 }
 
+static int
+reached_max_heap_size(Process *p, Uint total_heap_size,
+                      Uint extra_heap_size, Uint extra_old_heap_size)
+{
+    Uint max_heap_flags = MAX_HEAP_SIZE_FLAGS_GET(p);
+    if (IS_TRACED_FL(p, F_TRACE_GC) ||
+        max_heap_flags & MAX_HEAP_SIZE_LOG) {
+        Eterm msg;
+        Uint size = 0;
+        Eterm *o_hp , *hp;
+        erts_process_gc_info(p, &size, NULL, extra_heap_size,
+                             extra_old_heap_size);
+        o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, size * sizeof(Eterm));
+        msg = erts_process_gc_info(p, NULL, &hp, extra_heap_size,
+                                   extra_old_heap_size);
+
+        if (max_heap_flags & MAX_HEAP_SIZE_LOG) {
+            int alive = erts_is_alive;
+            erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+            Eterm *o_hp, *hp, args = NIL;
+
+            /* Build the format message */
+            erts_dsprintf(dsbufp, "     Process:          ~p ");
+            if (alive)
+                erts_dsprintf(dsbufp, "on node ~p");
+            erts_dsprintf(dsbufp, "~n     Context:          maximum heap size reached~n");
+            erts_dsprintf(dsbufp, "     Max heap size:    ~p~n");
+            erts_dsprintf(dsbufp, "     Total heap size:  ~p~n");
+            erts_dsprintf(dsbufp, "     Kill:             ~p~n");
+            erts_dsprintf(dsbufp, "     Error Logger:     ~p~n");
+            erts_dsprintf(dsbufp, "     GC Info:          ~p~n");
+
+            /* Build the args in reverse order */
+            o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 7 : 6) * sizeof(Eterm));
+            args = CONS(hp, msg, args); hp += 2;
+            args = CONS(hp, am_true, args); hp += 2;
+            args = CONS(hp, (max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false), args); hp += 2;
+            args = CONS(hp, make_small(total_heap_size), args); hp += 2;
+            args = CONS(hp, make_small(MAX_HEAP_SIZE_GET(p)), args); hp += 2;
+            if (alive) {
+                args = CONS(hp, erts_this_node->sysname, args); hp += 2;
+            }
+            args = CONS(hp, p->common.id, args); hp += 2;
+
+            erts_send_error_term_to_logger(p->group_leader, dsbufp, args);
+            erts_free(ERTS_ALC_T_TMP, o_hp);
+        }
+
+        if (IS_TRACED_FL(p, F_TRACE_GC))
+            trace_gc(p, am_gc_max_heap_size, 0, msg);
+
+        erts_free(ERTS_ALC_T_TMP, o_hp);
+    }
+    /* returns true if we should kill the process */
+    return max_heap_flags & MAX_HEAP_SIZE_KILL;
+}
+
+Eterm
+erts_max_heap_size_map(Sint max_heap_size, Uint max_heap_flags,
+                       Eterm **hpp, Uint *sz)
+{
+    if (!hpp) {
+        *sz += (2*3 + 1 + MAP_HEADER_FLATMAP_SZ);
+        return THE_NON_VALUE;
+    } else {
+        Eterm *hp = *hpp;
+        Eterm keys = TUPLE3(hp, am_error_logger, am_kill, am_size);
+        flatmap_t *mp;
+        hp += 4;
+        mp = (flatmap_t*) hp;
+        mp->thing_word = MAP_HEADER_FLATMAP;
+        mp->size = 3;
+        mp->keys = keys;
+        hp += MAP_HEADER_FLATMAP_SZ;
+        *hp++ = max_heap_flags & MAX_HEAP_SIZE_LOG ? am_true : am_false;
+        *hp++ = max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false;
+        *hp++ = make_small(max_heap_size);
+        *hpp = hp;
+        return make_flatmap(mp);
+    }
+}
+
+int
+erts_max_heap_size(Eterm arg, Uint *max_heap_size, Uint *max_heap_flags)
+{
+    Sint sz;
+    *max_heap_flags = H_MAX_FLAGS;
+    if (is_small(arg)) {
+        sz = signed_val(arg);
+        *max_heap_flags = H_MAX_FLAGS;
+    } else if (is_map(arg)) {
+        const Eterm *size = erts_maps_get(am_size, arg);
+        const Eterm *kill = erts_maps_get(am_kill, arg);
+        const Eterm *log = erts_maps_get(am_error_logger, arg);
+        if (size && is_small(*size)) {
+            sz = signed_val(*size);
+        } else {
+            /* size is mandatory */
+            return 0;
+        }
+        if (kill) {
+            if (*kill == am_true)
+                *max_heap_flags |= MAX_HEAP_SIZE_KILL;
+            else if (*kill == am_false)
+                *max_heap_flags &= ~MAX_HEAP_SIZE_KILL;
+            else
+                return 0;
+        }
+        if (log) {
+            if (*log == am_true)
+                *max_heap_flags |= MAX_HEAP_SIZE_LOG;
+            else if (*log == am_false)
+                *max_heap_flags &= ~MAX_HEAP_SIZE_LOG;
+            else
+                return 0;
+        }
+    } else
+        return 0;
+    if (sz < 0)
+        return 0;
+    *max_heap_size = sz;
+    return 1;
+}
+
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 
 static int
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 8a6ff99990..54ea9ca3c0 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -135,7 +135,7 @@ typedef struct {
 #define ERTS_PROCESS_GC_INFO_MAX_TERMS (11)  /* number of elements in process_gc_info*/
 #define ERTS_PROCESS_GC_INFO_MAX_SIZE                                   \
     (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE))
-Eterm erts_process_gc_info(struct process*, Uint *, Eterm **);
+Eterm erts_process_gc_info(struct process*, Uint *, Eterm **, Uint, Uint);
 
 void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
@@ -155,5 +155,7 @@ void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*);
 void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
 void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
 void erts_free_heap_frags(struct process* p);
+Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *);
+int erts_max_heap_size(Eterm, Uint *, Uint *);
 
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index fcd2739ac3..0649fb68de 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -164,6 +164,8 @@ int erts_use_sender_punish;
 Uint display_items;	    	/* no of items to display in traces etc */
 int H_MIN_SIZE;			/* The minimum heap grain */
 int BIN_VH_MIN_SIZE;		/* The minimum binary virtual*/
+int H_MAX_SIZE;			/* The maximum heap size */
+int H_MAX_FLAGS;		/* The maximum heap flags */
 
 Uint32 erts_debug_flags;	/* Debug flags. */
 int erts_backtrace_depth;	/* How many functions to show in a backtrace
@@ -576,6 +578,10 @@ void erts_usage(void)
 	       H_DEFAULT_SIZE);
     erts_fprintf(stderr, "-hmbs size     set minimum binary virtual heap size in words (default %d)\n",
 	       VH_DEFAULT_SIZE);
+    erts_fprintf(stderr, "-hmax size     set maximum heap size in words (default %d)\n",
+	       H_DEFAULT_MAX_SIZE);
+    erts_fprintf(stderr, "-hmaxk bool    enable or disable kill at max heap size (default true)\n");
+    erts_fprintf(stderr, "-hmaxel bool   enable or disable error_logger report at max heap size (default true)\n");
     erts_fprintf(stderr, "-hpds size     initial process dictionary size (default %d)\n",
 	       erts_pd_initial_size);
     erts_fprintf(stderr, "-hmqd  val     set default message queue data flag for processes,\n");
@@ -759,6 +765,8 @@ early_init(int *argc, char **argv) /*
     erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE;
     H_MIN_SIZE = H_DEFAULT_SIZE;
     BIN_VH_MIN_SIZE = VH_DEFAULT_SIZE;
+    H_MAX_SIZE = H_DEFAULT_MAX_SIZE;
+    H_MAX_FLAGS = MAX_HEAP_SIZE_KILL|MAX_HEAP_SIZE_LOG;
 
     erts_initialized = 0;
 
@@ -1484,10 +1492,13 @@ erl_start(int argc, char **argv)
 	    char *sub_param = argv[i]+2;
 	    /* set default heap size
 	     *
-	     * h|ms  - min_heap_size
-	     * h|mbs - min_bin_vheap_size
-	     * h|pds - erts_pd_initial_size
-	     * h|mqd - message_queue_data
+	     * h|ms    - min_heap_size
+	     * h|mbs   - min_bin_vheap_size
+	     * h|pds   - erts_pd_initial_size
+	     * h|mqd   - message_queue_data
+             * h|max   - max_heap_size
+             * h|maxk  - max_heap_kill
+             * h|maxel - max_heap_error_logger
 	     *
 	     */
 	    if (has_prefix("mbs", sub_param)) {
@@ -1530,6 +1541,41 @@ erl_start(int argc, char **argv)
 				 "Invalid message_queue_data flag: %s\n", arg);
 		    erts_usage();
 		}
+            } else if (has_prefix("maxk", sub_param)) {
+		arg = get_arg(sub_param+4, argv[i+1], &i);
+		if (strcmp(arg,"true") == 0) {
+                    H_MAX_FLAGS |= MAX_HEAP_SIZE_KILL;
+                } else if (strcmp(arg,"false") == 0) {
+                    H_MAX_FLAGS &= ~MAX_HEAP_SIZE_KILL;
+                } else {
+		    erts_fprintf(stderr, "bad max heap kill %s\n", arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM, ("using max heap kill %d\n", H_MAX_FLAGS));
+            } else if (has_prefix("maxel", sub_param)) {
+		arg = get_arg(sub_param+5, argv[i+1], &i);
+		if (strcmp(arg,"true") == 0) {
+                    H_MAX_FLAGS |= MAX_HEAP_SIZE_LOG;
+                } else if (strcmp(arg,"false") == 0) {
+                    H_MAX_FLAGS &= ~MAX_HEAP_SIZE_LOG;
+                } else {
+		    erts_fprintf(stderr, "bad max heap error logger %s\n", arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM, ("using max heap log %d\n", H_MAX_FLAGS));
+	    } else if (has_prefix("max", sub_param)) {
+		arg = get_arg(sub_param+3, argv[i+1], &i);
+		if ((H_MAX_SIZE = atoi(arg)) < 0) {
+		    erts_fprintf(stderr, "bad max heap size %s\n", arg);
+		    erts_usage();
+		}
+                if (H_MAX_SIZE < H_MIN_SIZE && H_MAX_SIZE) {
+		    erts_fprintf(stderr, "max heap size (%s) is not allowed to be "
+                                 "smaller than min heap size (%d)\n",
+                                 arg, H_MIN_SIZE);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM, ("using max heap size %d\n", H_MAX_SIZE));
 	    } else {
 	        /* backward compatibility */
 		arg = get_arg(argv[i]+2, argv[i+1], &i);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 36b0af0c1a..e2f14c23bf 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11021,9 +11021,13 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->min_heap_size  = so->min_heap_size;
 	p->min_vheap_size = so->min_vheap_size;
 	p->max_gen_gcs    = so->max_gen_gcs;
+        MAX_HEAP_SIZE_SET(p, so->max_heap_size);
+        MAX_HEAP_SIZE_FLAGS_SET(p, so->max_heap_flags);
     } else {
 	p->min_heap_size  = H_MIN_SIZE;
 	p->min_vheap_size = BIN_VH_MIN_SIZE;
+        MAX_HEAP_SIZE_SET(p, H_MAX_SIZE);
+        MAX_HEAP_SIZE_FLAGS_SET(p, H_MAX_FLAGS);
 	p->max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
     }
     p->schedule_count = 0;
@@ -11600,7 +11604,8 @@ set_proc_exiting(Process *p,
     p->i = (BeamInstr *) beam_exit;
 
 #ifndef ERTS_SMP
-    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) {
+    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
+        && !(state & ERTS_PSFLG_GC)) {
 	/*
 	 * I non smp case:
 	 *
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 59da9c1779..12a919bc87 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -918,6 +918,15 @@ struct ErtsPendingSuspend_ {
 #  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
 #  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap
 
+#  define MAX_HEAP_SIZE_GET(p)     ((p)->max_heap_size >> 2)
+#  define MAX_HEAP_SIZE_SET(p, sz) ((p)->max_heap_size = ((sz) << 2) |  \
+                                    MAX_HEAP_SIZE_FLAGS_GET(p))
+#  define MAX_HEAP_SIZE_FLAGS_GET(p)          ((p)->max_heap_size & 0x3)
+#  define MAX_HEAP_SIZE_FLAGS_SET(p, flags)   ((p)->max_heap_size = flags | \
+                                               ((p)->max_heap_size & ~0x3))
+#  define MAX_HEAP_SIZE_KILL 1
+#  define MAX_HEAP_SIZE_LOG  2
+
 struct process {
     ErtsPTabElementCommon common; /* *Need* to be first in struct */
 
@@ -935,6 +944,7 @@ struct process {
     Uint heap_sz;		/* Size of heap in words */
     Uint min_heap_size;         /* Minimum size of heap (in words). */
     Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */
+    Uint max_heap_size;         /* Maximum size of heap (in words). */
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     volatile unsigned long fp_exception;
@@ -1285,6 +1295,8 @@ typedef struct {
     Uint min_vheap_size;	/* Minimum virtual heap size  */
     int priority;		/* Priority for process. */
     Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
+    Uint max_heap_size;         /* Maximum heap size in words */
+    Uint max_heap_flags;        /* Maximum heap flags (kill | log) */
     int scheduler;
 } ErlSpawnOpts;
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 1c0fc0a11f..6d6373b706 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1409,25 +1409,28 @@ void save_calls(Process *p, Export *e)
  * are all small (atomic) integers.
  */
 void
-trace_gc(Process *p, Eterm what, Uint size)
+trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
 {
     ErtsTracerNif *tnif = NULL;
     Eterm* hp;
-    Eterm msg = NIL;
     Uint sz = 0;
     Eterm tup;
 
-    if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, TRACE_FUN_E_GC, what)) {
+    if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif,
+                          TRACE_FUN_E_GC, what)) {
+
+        if (is_non_value(msg)) {
 
-        (void) erts_process_gc_info(p, &sz, NULL);
-        hp = HAlloc(p, sz + 3 + 2);
+            (void) erts_process_gc_info(p, &sz, NULL, 0, 0);
+            hp = HAlloc(p, sz + 3 + 2);
 
-        msg = erts_process_gc_info(p, NULL, &hp);
-	tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3;
-        msg = CONS(hp, tup, msg); hp += 2;
+            msg = erts_process_gc_info(p, NULL, &hp, 0, 0);
+            tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3;
+            msg = CONS(hp, tup, msg); hp += 2;
+        }
 
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC,
-                what, msg, am_undefined, am_true);
+                           what, msg, THE_NON_VALUE, am_true);
     }
 }
 
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index de4beadb7e..fd5879bc9d 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -112,7 +112,7 @@ void trace_sched(Process*, ErtsProcLocks, Eterm);
 void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm);
 void trace_proc_spawn(Process*, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args);
 void save_calls(Process *p, Export *);
-void trace_gc(Process *p, Eterm what, Uint size);
+void trace_gc(Process *p, Eterm what, Uint size, Eterm msg);
 /* port tracing */
 void trace_virtual_sched(Process*, ErtsProcLocks, Eterm);
 void trace_sched_ports(Port *pp, Eterm);
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index f3c54de214..f97716d030 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -50,6 +50,7 @@
 
 #define H_DEFAULT_SIZE  233        /* default (heap + stack) min size */
 #define VH_DEFAULT_SIZE  32768     /* default virtual (bin) heap min size (words) */
+#define H_DEFAULT_MAX_SIZE 0       /* default max heap size is off */
 
 #define CP_SIZE 1
 
@@ -160,6 +161,8 @@ extern int num_instructions;	/* Number of instruction in opc[]. */
 
 extern int H_MIN_SIZE;		/* minimum (heap + stack) */
 extern int BIN_VH_MIN_SIZE;	/* minimum virtual (bin) heap */
+extern int H_MAX_SIZE;          /* maximum (heap + stack) */
+extern int H_MAX_FLAGS;         /* maximum heap flags  */
 
 extern int erts_atom_table_size;/* Atom table size */
 extern int erts_pd_initial_size;/* Initial Process dictionary table size */
-- 
cgit v1.2.3


From dc203f539692a1191240209fb95afa020fa342b5 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 3 May 2016 09:55:15 +0200
Subject: erts: Fix pre-bif yield current_function

---
 erts/emulator/beam/beam_emu.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a88fea3802..d00a563087 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2823,7 +2823,7 @@ do {						\
             /* If we have run out of reductions, we do a context
                switch before calling the bif */
             c_p->arity = ((Export *)Arg(0))->code[2];
-            c_p->current = NULL;
+            c_p->current = ((Export *)Arg(0))->code;
             goto context_switch3;
         }
 
@@ -3538,9 +3538,7 @@ do {						\
             if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
                 /* If we have run out of reductions, we do a context
                    switch before calling the nif */
-                c_p->arity = I[-1];
-                c_p->current = NULL;
-                goto context_switch3;
+                goto context_switch;
             }
 
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
@@ -3589,9 +3587,7 @@ do {						\
             if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
                 /* If we have run out of reductions, we do a context
                    switch before calling the bif */
-                c_p->arity = I[-1];
-                c_p->current = NULL;
-                goto context_switch3;
+                goto context_switch;
             }
 
             if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
-- 
cgit v1.2.3


From 0aa5f873fdfb391a455bc134256b7c464ffa161b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 15 Apr 2016 11:24:14 +0200
Subject: Remove conditional dirty schedulers API

---
 erts/emulator/beam/erl_drv_nif.h       |  9 ++++-----
 erts/emulator/beam/erl_nif.h           | 13 +++----------
 erts/emulator/beam/erl_nif_api_funcs.h | 15 ++-------------
 erts/emulator/beam/io.c                |  8 +++++---
 4 files changed, 14 insertions(+), 31 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 2700b62854..6ec5fbb895 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -43,12 +43,11 @@ typedef struct {
     int suggested_stack_size;
 } ErlDrvThreadOpts;
 
-#if defined(ERL_DRV_DIRTY_SCHEDULER_SUPPORT) || defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT)
+
 typedef enum {
-    ERL_DRV_DIRTY_JOB_CPU_BOUND = 1,
-    ERL_DRV_DIRTY_JOB_IO_BOUND  = 2
-} ErlDrvDirtyJobFlags;
-#endif
+    ERL_DIRTY_JOB_CPU_BOUND = 1,
+    ERL_DIRTY_JOB_IO_BOUND  = 2
+} ErlDirtyJobFlags;
 
 #ifdef SIZEOF_CHAR
 #  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 3964f7f679..02c82415fd 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -28,7 +28,6 @@
 #  include "config.h"
 #endif
 
-#include "erl_native_features_config.h"
 #include "erl_drv_nif.h"
 
 /* Version history:
@@ -167,13 +166,11 @@ typedef int ErlNifTSDKey;
 
 typedef ErlDrvThreadOpts ErlNifThreadOpts;
 
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 typedef enum
 {
-    ERL_NIF_DIRTY_JOB_CPU_BOUND = ERL_DRV_DIRTY_JOB_CPU_BOUND,
-    ERL_NIF_DIRTY_JOB_IO_BOUND  = ERL_DRV_DIRTY_JOB_IO_BOUND
+    ERL_NIF_DIRTY_JOB_CPU_BOUND = ERL_DIRTY_JOB_CPU_BOUND,
+    ERL_NIF_DIRTY_JOB_IO_BOUND  = ERL_DIRTY_JOB_IO_BOUND
 }ErlNifDirtyTaskFlags;
-#endif
 
 typedef struct /* All fields all internal and may change */
 {
@@ -257,11 +254,7 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS)
 #endif
 
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-#  define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION
-#else
-#  define ERL_NIF_ENTRY_OPTIONS 0
-#endif
+#define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION
 
 #ifdef __cplusplus
 }
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index a5acd86551..93e4313791 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -172,20 +172,11 @@ ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, E
 ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin));
 ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsigned char* data, size_t sz, ERL_NIF_TERM *term, unsigned int opts));
 ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg));
+ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
 */
-
-
-/*
- * Conditional EXPERIMENTAL stuff always last.
- * Must be moved up and made unconditional to support binary backward
- * compatibility on Windows.
- */
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
-#endif
 #endif /* ERL_NIF_API_FUNC_DECL */
 
 /*
@@ -336,6 +327,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary)
 #  define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term)
 #  define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command)
+#  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
@@ -346,9 +338,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
  * Must be moved up and made unconditional to support binary backward
  * compatibility on Windows.
  */
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-#  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
-#endif
 #endif  /* ERL_NIF_API_FUNC_MACRO */
 
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b9f6ab04c6..c35d80652f 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7881,11 +7881,13 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
      * (driver version 3.1, NIF version 2.7)
      */
     if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) {
-#if defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) && defined(USE_THREADS)
-	sip->dirty_scheduler_support = 1;
+	sip->dirty_scheduler_support =
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    1
 #else
-	sip->dirty_scheduler_support = 0;
+	    0
 #endif
+	    ;
     }
 
 }
-- 
cgit v1.2.3


From cedf53ed9d29e7a7cf8ebaf77dea971a025205e1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 11 May 2016 15:03:42 +0200
Subject: erts: Fix non-smp max_heap_size assert

---
 erts/emulator/beam/erl_gc.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index fa0782867d..bd238d0f45 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -664,8 +664,6 @@ do_major_collection:
 
     ErtsGcQuickSanityCheck(p);
 
-    erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
-
     /* Max heap size has been reached and the process was configured
        to be killed, so we kill it and set it in a delayed garbage
        collecting state. There should be no gc_end trace or
@@ -3176,8 +3174,8 @@ reached_max_heap_size(Process *p, Uint total_heap_size,
             if (alive)
                 erts_dsprintf(dsbufp, "on node ~p");
             erts_dsprintf(dsbufp, "~n     Context:          maximum heap size reached~n");
-            erts_dsprintf(dsbufp, "     Max heap size:    ~p~n");
-            erts_dsprintf(dsbufp, "     Total heap size:  ~p~n");
+            erts_dsprintf(dsbufp, "     Max Heap Size:    ~p~n");
+            erts_dsprintf(dsbufp, "     Total Heap Size:  ~p~n");
             erts_dsprintf(dsbufp, "     Kill:             ~p~n");
             erts_dsprintf(dsbufp, "     Error Logger:     ~p~n");
             erts_dsprintf(dsbufp, "     GC Info:          ~p~n");
-- 
cgit v1.2.3


From 8d8d8c3817988a50ed1e058e73b247ce0a8c3616 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 11 May 2016 16:17:49 +0200
Subject: erts: Add enif_snprintf

* Add the capability to format erlang terms to a char buffer in nifs.
* Bump NIF version to 2.11
---
 erts/emulator/beam/erl_nif.c           | 10 ++++++++++
 erts/emulator/beam/erl_nif.h           |  3 ++-
 erts/emulator/beam/erl_nif_api_funcs.h |  2 ++
 3 files changed, 14 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index a695a028ba..fa20ce3c86 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1603,6 +1603,16 @@ int enif_fprintf(void* filep, const char* format, ...)
     return ret;
 }    
 
+int enif_snprintf(char *buffer, size_t size, const char* format, ...) 
+{ 
+    int ret;
+    va_list arglist;
+    va_start(arglist, format);
+    ret = erts_vsnprintf(buffer, size, format, arglist);
+    va_end(arglist);
+    return ret;
+}
+
 /***********************************************************
  **       Memory managed (GC'ed) "resource" objects       **
  ***********************************************************/
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 3964f7f679..468f5cf5ec 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -50,9 +50,10 @@
 ** 2.8: 18.0 add enif_has_pending_exception
 ** 2.9: 18.2 enif_getenv
 ** 2.10: Time API
+** 2.11: 19.0 enif_snprintf
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 10
+#define ERL_NIF_MINOR_VERSION 11
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index a5acd86551..c7389b1626 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -172,6 +172,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, E
 ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin));
 ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsigned char* data, size_t sz, ERL_NIF_TERM *term, unsigned int opts));
 ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg));
+ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -336,6 +337,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary)
 #  define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term)
 #  define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command)
+#  define enif_snprintf ERL_NIF_API_FUNC_MACRO(enif_snprintf)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From 8cbf3bd3b9d552423812df4acc7a40d9a23fdeae Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 2 May 2016 16:02:02 +0200
Subject: Add better support for communication with a process executing dirty
 NIF

- Termination of a process...
- Modify trace flags of process...
- Process info on process...
- Register/unregister of name on process...
- Set group leader on process...

...  while it is executing a dirty NIF.
---
 erts/emulator/beam/beam_bp.c           |   8 +-
 erts/emulator/beam/beam_bp.h           |   2 +-
 erts/emulator/beam/beam_emu.c          |  60 ++-
 erts/emulator/beam/bif.c               |  26 +-
 erts/emulator/beam/bif.h               |  42 +-
 erts/emulator/beam/break.c             |   7 +-
 erts/emulator/beam/erl_alloc.types     |   1 +
 erts/emulator/beam/erl_bif_info.c      |  15 +-
 erts/emulator/beam/erl_bif_unique.c    |   4 +-
 erts/emulator/beam/erl_bits.h          |   4 +-
 erts/emulator/beam/erl_db.c            |   2 +-
 erts/emulator/beam/erl_db_util.c       |   2 +-
 erts/emulator/beam/erl_gc.c            |  17 +-
 erts/emulator/beam/erl_hl_timer.c      |  24 +-
 erts/emulator/beam/erl_message.c       |   7 +-
 erts/emulator/beam/erl_msacc.c         |   2 +-
 erts/emulator/beam/erl_nif.c           | 561 +++++++++++++++++------
 erts/emulator/beam/erl_nif_api_funcs.h |   2 +
 erts/emulator/beam/erl_port.h          |  39 ++
 erts/emulator/beam/erl_process.c       | 795 ++++++++++++++++++++++-----------
 erts/emulator/beam/erl_process.h       | 105 +++--
 erts/emulator/beam/erl_process_dump.c  |  34 +-
 erts/emulator/beam/erl_time_sup.c      |  10 +-
 erts/emulator/beam/global.h            |   1 +
 erts/emulator/beam/io.c                |   4 +-
 erts/emulator/beam/sys.h               |   4 +
 26 files changed, 1232 insertions(+), 546 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 1e30e8d8d1..308e5ce205 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -82,7 +82,7 @@ erts_smp_atomic32_t erts_staging_bp_index;
 static ERTS_INLINE ErtsMonotonicTime
 get_mtime(Process *c_p)
 {
-    return erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(c_p));
+    return erts_get_monotonic_time(erts_proc_sched_data(c_p));
 }
 
 /* *************************************************************************
@@ -976,7 +976,8 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
     BpDataTime *pbdt = NULL;
 
     ASSERT(c_p);
-    ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & ERTS_PSFLG_RUNNING);
+    ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING
+						       | ERTS_PSFLG_DIRTY_RUNNING));
 
     /* get previous timestamp and breakpoint
      * from the process psd  */
@@ -1053,7 +1054,8 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
     BpDataTime *pbdt = NULL;
 
     ASSERT(p);
-    ASSERT(erts_smp_atomic32_read_acqb(&p->state) & ERTS_PSFLG_RUNNING);
+    ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING
+						     | ERTS_PSFLG_DIRTY_RUNNING));
 
     /* get previous timestamp and breakpoint
      * from the process psd  */
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 9c2fc007a2..541af77211 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -80,7 +80,7 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
 #ifdef ERTS_SMP
-#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1)
+#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1)
 #else
 #define bp_sched2ix_proc(p) (0)
 #endif
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 99eb26bec5..9402317185 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -64,18 +64,21 @@
 #  ifdef ERTS_SMP
 #    define PROCESS_MAIN_CHK_LOCKS(P)					\
 do {									\
-    if ((P)) {								\
+    if ((P))								\
 	erts_proc_lc_chk_only_proc_main((P));				\
-    }									\
-    else								\
-	erts_lc_check_exact(NULL, 0);					\
-    	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());		\
+    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());		\
+} while (0)
+#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)				\
+do {									\
+    if ((P))								\
+	erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,		\
+				  __FILE__, __LINE__);			\
+} while (0)
+#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)				\
+do {									\
+    if ((P))								\
+	erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN);		\
 } while (0)
-#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
-        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
-					   __FILE__, __LINE__)
-#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
-        if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
 #  else
 #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
 #    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
@@ -1202,12 +1205,12 @@ init_emulator(void)
     do {								\
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) {			\
 	    ASSERT(FC <= 0);						\
-	    ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds		\
+	    ASSERT(erts_proc_sched_data(c_p)->virtual_reds		\
 		   <= 0 - (FC));					\
 	}								\
 	else {								\
 	    ASSERT(FC <= CONTEXT_REDS);					\
-	    ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds		\
+	    ASSERT(erts_proc_sched_data(c_p)->virtual_reds		\
 		   <= CONTEXT_REDS - (FC));				\
 	}								\
 } while (0)
@@ -1321,8 +1324,8 @@ void process_main(void)
     if (start_time != 0) {
         Sint64 diff = erts_timestamp_millis() - start_time;
 	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    && !ERTS_SCHEDULER_IS_DIRTY(c_p->scheduler_data)
+#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
+	    && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))
 #endif
 	    ) {
 	    BeamInstr *inptr = find_function_from_pc(start_time_i);
@@ -1351,8 +1354,8 @@ void process_main(void)
 	start_time_i = c_p->i;
     }
 
-    reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
-    freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
+    reg = erts_proc_sched_data(c_p)->x_reg_array;
+    freg = erts_proc_sched_data(c_p)->f_reg_array;
     ERL_BITS_RELOAD_STATEP(c_p);
     {
 	int reds;
@@ -3524,18 +3527,27 @@ do {						\
 		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (!c_p->scheduler_data)
+		    live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */
+		else
+#endif
+		    live_hf_end = c_p->mbuf;
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
-		live_hf_end = c_p->mbuf;
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
 		    nif_bif_result = THE_NON_VALUE;
 		erts_post_nif(&env);
-	    }
-	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
-	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 
-	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+		PROCESS_MAIN_CHK_LOCKS(c_p);
+		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+		if (env.exiting) {
+		    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+		    goto do_schedule;
+		}
+		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	    }
 
 	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    goto apply_bif_or_nif_epilogue;
@@ -4887,8 +4899,8 @@ do {						\
 #ifdef DEBUG
      pid = c_p->common.id; /* may have switched process... */
 #endif
-     reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
-     freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
+     reg = erts_proc_sched_data(c_p)->x_reg_array;
+     freg = erts_proc_sched_data(c_p)->f_reg_array;
      ERL_BITS_RELOAD_STATEP(c_p);
      /* XXX: this abuse of def_arg_reg[] is horrid! */
      neg_o_reds = -c_p->def_arg_reg[4];
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index eed0702688..4516d53284 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1686,7 +1686,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 						 ERTS_PSFLG_BOUND);
        }
 
-       curr = ERTS_GET_SCHEDULER_DATA_FROM_PROC(BIF_P)->run_queue;
+       curr = erts_proc_sched_data(BIF_P)->run_queue;
        old = (ERTS_PSFLG_BOUND & state) ? curr : NULL;
 
        ASSERT(!old || old == curr);
@@ -4198,8 +4198,28 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 	    else {
 		locks &= ~ERTS_PROC_LOCK_STATUS;
 		erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS);
-		new_member->group_leader = STORE_NC_IN_PROC(new_member,
-							    BIF_ARG_1);
+		if (erts_smp_atomic32_read_nob(&new_member->state)
+		    & !(ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+		    new_member->group_leader = STORE_NC_IN_PROC(new_member,
+								BIF_ARG_1);
+		}
+		else {
+		    ErlHeapFragment *bp;
+		    Eterm *hp;
+		    /*
+		     * Other process executing on a dirty scheduler,
+		     * so we are not allowed to write to its heap.
+		     * Store in heap fragment.
+		     */
+
+		    bp = new_message_buffer(NC_HEAP_SIZE(BIF_ARG_1));
+		    hp = bp->mem;
+		    new_member->group_leader = STORE_NC(&hp,
+							&new_member->off_heap,
+							BIF_ARG_1);
+		    bp->next = new_member->mbuf;
+		    new_member->mbuf = bp;
+		}
 	    }
 	}
 
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 5d751dd67d..2203182a0d 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -59,12 +59,12 @@ extern Export *erts_convert_time_unit_trap;
 do {									\
     if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) {				\
 	if ((fcalls) > 0)						\
-	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (fcalls);	\
+	    erts_proc_sched_data((p))->virtual_reds += (fcalls);	\
 	(fcalls) = 0;							\
     }									\
     else {								\
 	if ((fcalls) > -CONTEXT_REDS)					\
-	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds			\
+	    erts_proc_sched_data((p))->virtual_reds			\
 		+= ((fcalls) - (-CONTEXT_REDS));			\
 	(fcalls) = -CONTEXT_REDS;					\
     }									\
@@ -91,22 +91,22 @@ do {									\
     if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) {				\
 	if ((p)->fcalls >= reds) {					\
 	    (p)->fcalls -= reds;					\
-	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += reds;		\
+	    erts_proc_sched_data((p))->virtual_reds += reds;		\
 	}								\
 	else {								\
 	    if ((p)->fcalls > 0)					\
-		ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (p)->fcalls;\
+		erts_proc_sched_data((p))->virtual_reds += (p)->fcalls;	\
 	    (p)->fcalls = 0;						\
 	}								\
     }									\
     else {								\
 	if ((p)->fcalls >= reds - CONTEXT_REDS) {			\
 	    (p)->fcalls -= reds;					\
-	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += reds;		\
+	    erts_proc_sched_data((p))->virtual_reds += reds;		\
 	}								\
 	else {								\
 	    if ((p)->fcalls > -CONTEXT_REDS)				\
-		ERTS_PROC_GET_SCHDATA((p))->virtual_reds		\
+		erts_proc_sched_data((p))->virtual_reds			\
 		    += (p)->fcalls - (-CONTEXT_REDS);			\
 	    (p)->fcalls = -CONTEXT_REDS;				\
 	}								\
@@ -118,14 +118,14 @@ do {									\
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) {			\
 	    int nreds__ = ((int)(Reds)) - CONTEXT_REDS;			\
 	    if ((FCalls) > nreds__) {					\
-		ERTS_PROC_GET_SCHDATA((P))->virtual_reds		\
+		erts_proc_sched_data((P))->virtual_reds			\
 		    += (FCalls) - nreds__;				\
 		(FCalls) = nreds__;					\
 	    }								\
 	}								\
 	else {								\
 	    if ((FCalls) > (Reds)) {					\
-		ERTS_PROC_GET_SCHDATA((P))->virtual_reds		\
+		erts_proc_sched_data((P))->virtual_reds			\
 		    += (FCalls) - (Reds);				\
 		(FCalls) = (Reds);					\
 	    }								\
@@ -165,7 +165,7 @@ do {								\
 
 #define ERTS_BIF_ERROR_TRAPPED1(Proc, Reason, Bif, A0)		\
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
     (Proc)->current = (Bif)->code;				\
     reg[0] = (Eterm) (A0);					\
@@ -174,7 +174,7 @@ do {								\
 
 #define ERTS_BIF_ERROR_TRAPPED2(Proc, Reason, Bif, A0, A1)	\
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
     (Proc)->current = (Bif)->code;				\
     reg[0] = (Eterm) (A0);					\
@@ -184,7 +184,7 @@ do {								\
 
 #define ERTS_BIF_ERROR_TRAPPED3(Proc, Reason, Bif, A0, A1, A2)	\
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
     (Proc)->current = (Bif)->code;				\
     reg[0] = (Eterm) (A0);					\
@@ -208,7 +208,7 @@ do {								\
 
 #define ERTS_BIF_PREP_ERROR_TRAPPED1(Ret, Proc, Reason, Bif, A0) \
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
     (Proc)->current = (Bif)->code;				\
     reg[0] = (Eterm) (A0);					\
@@ -217,7 +217,7 @@ do {								\
 
 #define ERTS_BIF_PREP_ERROR_TRAPPED2(Ret, Proc, Reason, Bif, A0, A1) \
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
     (Proc)->current = (Bif)->code;				\
     reg[0] = (Eterm) (A0);					\
@@ -227,7 +227,7 @@ do {								\
 
 #define ERTS_BIF_PREP_ERROR_TRAPPED3(Ret, Proc, Reason, Bif, A0, A1, A2) \
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
     (Proc)->current = (Bif)->code;				\
     reg[0] = (Eterm) (A0);					\
@@ -246,7 +246,7 @@ do {						\
 
 #define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0)		\
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->arity = 1;						\
     reg[0] = (Eterm) (A0);					\
     (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
@@ -256,7 +256,7 @@ do {								\
 
 #define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1)		\
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->arity = 2;						\
     reg[0] = (Eterm) (A0);					\
     reg[1] = (Eterm) (A1);					\
@@ -267,7 +267,7 @@ do {								\
 
 #define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2)	\
 do {								\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->arity = 3;						\
     reg[0] = (Eterm) (A0);					\
     reg[1] = (Eterm) (A1);					\
@@ -279,7 +279,7 @@ do {								\
 
 #define ERTS_BIF_PREP_TRAP3_NO_RET(Trap, Proc, A0, A1, A2)\
 do {							\
-    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->arity = 3;					\
     reg[0] = (Eterm) (A0);		\
     reg[1] = (Eterm) (A1);		\
@@ -296,7 +296,7 @@ do {							\
  } while(0)
 
 #define BIF_TRAP1(Trap_, p, A0) do {				\
-      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
+      Eterm* reg = erts_proc_sched_data((p))->x_reg_array;	\
       (p)->arity = 1;						\
       reg[0] = (A0);						\
       (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
@@ -305,7 +305,7 @@ do {							\
  } while(0)
 
 #define BIF_TRAP2(Trap_, p, A0, A1) do {			\
-      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
+      Eterm* reg = erts_proc_sched_data((p))->x_reg_array;	\
       (p)->arity = 2;						\
       reg[0] = (A0);						\
       reg[1] = (A1);						\
@@ -315,7 +315,7 @@ do {							\
  } while(0)
 
 #define BIF_TRAP3(Trap_, p, A0, A1, A2) do {			\
-      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
+      Eterm* reg = erts_proc_sched_data((p))->x_reg_array;	\
       (p)->arity = 3;						\
       reg[0] = (A0);						\
       reg[1] = (A1);						\
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index d02c6828f9..3c19e82b66 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -118,7 +118,9 @@ process_killer(void)
 				 | ERTS_PSFLG_ACTIVE_SYS
 				 | ERTS_PSFLG_IN_RUNQ
 				 | ERTS_PSFLG_RUNNING
-				 | ERTS_PSFLG_RUNNING_SYS)) {
+				 | ERTS_PSFLG_RUNNING_SYS
+				 | ERTS_PSFLG_DIRTY_RUNNING
+				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 			erts_printf("Can only kill WAITING processes this way\n");
 		    }
 		    else {
@@ -214,7 +216,8 @@ print_process_info(int to, void *to_arg, Process *p)
     if (state & ERTS_PSFLG_GC) {
         garbing = 1;
         running = 1;
-    } else if (state & ERTS_PSFLG_RUNNING)
+    } else if (state & (ERTS_PSFLG_RUNNING
+			| ERTS_PSFLG_DIRTY_RUNNING))
         running = 1;
 
     /*
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index ba216c7eb4..227fedfb69 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -278,6 +278,7 @@ type	IOB_REQ		SHORT_LIVED	SYSTEM		io_bytes_request
 type    TRACER_NIF      LONG_LIVED      SYSTEM          tracer_nif
 type    TRACE_MSG_QUEUE SHORT_LIVED     SYSTEM          trace_message_queue
 type    SCHED_ASYNC_JOB SHORT_LIVED     SYSTEM          async_calls
+type    DIRTY_START     STANDARD	PROCESSES       dirty_start
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 85cadafebc..b12301812a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1107,7 +1107,7 @@ process_info_aux(Process *BIF_P,
 	break;
 
     case am_status:
-	res = erts_process_status(BIF_P, ERTS_PROC_LOCK_MAIN, rp, rpid);
+	res = erts_process_status(rp, rpid);
 	ASSERT(res != am_undefined);
 	hp = HAlloc(BIF_P, 3);
 	break;
@@ -2035,12 +2035,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	Uint arity = *tp++;
 	return info_1_tuple(BIF_P, tp, arityval(arity));
     } else if (BIF_ARG_1 == am_scheduler_id) {
-#ifdef ERTS_SMP
-	    ASSERT(BIF_P->scheduler_data);
-	    BIF_RET(make_small(BIF_P->scheduler_data->no));
-#else
-	    BIF_RET(make_small(1));
-#endif
+	ErtsSchedulerData *esdp = erts_proc_sched_data(BIF_P);
+	BIF_RET(make_small(esdp->no));
     } else if (BIF_ARG_1 == am_compat_rel) {
 	ASSERT(erts_compat_rel > 0);
 	BIF_RET(make_small(erts_compat_rel));
@@ -3589,10 +3585,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		/* Used by timer process_SUITE, timer_bif_SUITE, and
 		   node_container_SUITE (emulator) */
 		if (is_internal_pid(tp[2])) {
-		    BIF_RET(erts_process_status(BIF_P,
-						ERTS_PROC_LOCK_MAIN,
-						NULL,
-						tp[2]));
+		    BIF_RET(erts_process_status(NULL, tp[2]));
 		}
 	    }
 	    else if (ERTS_IS_ATOM_STR("link_list", tp[1])) {
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index 1e57e9fa53..7c70217d8d 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -257,7 +257,7 @@ static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive)
     Uint hsz;
     Eterm *hp;
 
-    esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    esdp = erts_proc_sched_data(c_p);
     thr_id = (Uint64) esdp->thr_id;
     unique = esdp->unique++;
     bld_unique_integer_term(NULL, &hsz, thr_id, unique, positive);
@@ -515,7 +515,7 @@ BIF_RETTYPE make_ref_0(BIF_ALIST_0)
 
     hp = HAlloc(BIF_P, REF_THING_SIZE);
 
-    res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp);
+    res = erts_sched_make_ref_in_buffer(erts_proc_sched_data(BIF_P), hp);
 
     BIF_RET(res);
 }
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 1c2a090f07..4bd5b24157 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -83,8 +83,8 @@ typedef struct erl_bin_match_struct{
 #ifdef ERTS_SMP
 /* the state resides in the current process' scheduler data */
 #define ERL_BITS_DECLARE_STATEP			struct erl_bits_state *EBS
-#define ERL_BITS_RELOAD_STATEP(P)		do{EBS = &(P)->scheduler_data->erl_bits_state;}while(0)
-#define ERL_BITS_DEFINE_STATEP(P)		struct erl_bits_state *EBS = &(P)->scheduler_data->erl_bits_state
+#define ERL_BITS_RELOAD_STATEP(P)		do{EBS = &erts_proc_sched_data((P))->erl_bits_state;}while(0)
+#define ERL_BITS_DEFINE_STATEP(P)		struct erl_bits_state *EBS = &erts_proc_sched_data((P))->erl_bits_state
 #else
 /* reentrant API but with a hidden single global state, for testing only */
 extern struct erl_bits_state ErlBitsState_;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index acaca54e9a..bad34211a5 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3462,7 +3462,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
     fix = tb->common.fixations;
     if (fix == NULL) {
 	tb->common.time.monotonic
-	    = erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(p));
+	    = erts_get_monotonic_time(erts_proc_sched_data(p));
 	tb->common.time.offset = erts_get_time_offset();
     }
     else {
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 95b1cd0148..3e75b9fd5f 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1801,7 +1801,7 @@ Eterm db_prog_match(Process *c_p,
     /* We need to lure the scheduler into believing in the pseudo process, 
        because of floating point exceptions. Do *after* mpsp is set!!! */
 
-    esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(psp);
+    esdp = erts_get_scheduler_data();
     if (esdp)
         current_scheduled = esdp->current_process;
     /* SMP: psp->scheduler_data is set by get_match_pseudo_process */
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index bed7e668d7..201a5acaef 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -389,7 +389,7 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 	if (p->freason == TRAP) {
 	  #if HIPE
 	    if (regs == NULL) {
-		regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array;
+		regs = erts_proc_sched_data(p)->x_reg_array;
 	    }
 	  #endif
 	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls);
@@ -404,6 +404,7 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 	result = val[0];
     }
     BUMP_REDS(p, cost);
+
     return result;
 }
 
@@ -507,14 +508,14 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
     /* Make sure that we do a proper GC as soon as possible... */
     p->flags |= F_FORCE_GC;
     reds_left = ERTS_REDS_LEFT(p, fcalls);
-    ASSERT(CONTEXT_REDS - reds_left >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+    ASSERT(CONTEXT_REDS - reds_left >= erts_proc_sched_data(p)->virtual_reds);
 
     if (reds_left > ERTS_ABANDON_HEAP_COST) {
 	int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
-	ERTS_PROC_GET_SCHDATA((p))->virtual_reds += vreds;
+	erts_proc_sched_data((p))->virtual_reds += vreds;
     }
 
-    ASSERT(CONTEXT_REDS >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+    ASSERT(CONTEXT_REDS >= erts_proc_sched_data(p)->virtual_reds);
     return reds_left;
 }
 
@@ -586,7 +587,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 #endif
 
     ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
-	   >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+	   >= erts_proc_sched_data(p)->virtual_reds);
 
     if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
 	return delay_garbage_collection(p, live_hf_end, need, fcalls);
@@ -716,7 +717,7 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca
     int reds_left = ERTS_REDS_LEFT(p, fcalls);
     if (reds > reds_left)
 	reds = reds_left;
-    ASSERT(CONTEXT_REDS - (reds_left - reds) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+    ASSERT(CONTEXT_REDS - (reds_left - reds) >= erts_proc_sched_data(p)->virtual_reds);
     return reds;
 }
 
@@ -726,7 +727,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls);
     BUMP_REDS(p, reds);
     ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
-	   >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+	   >= erts_proc_sched_data(p)->virtual_reds);
 }
 
 /*
@@ -2955,7 +2956,7 @@ reply_gc_info(void *vgcirp)
 Eterm
 erts_gc_info_request(Process *c_p)
 {
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     Eterm ref;
     ErtsGCInfoReq *gcirp;
     Eterm *hp;
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index c418762578..ebeff51aac 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1766,7 +1766,7 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
     if (is_not_internal_pid(rcvr) && is_not_atom(rcvr))
 	goto badarg;
 
-    esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    esdp = erts_proc_sched_data(c_p);
 
     hp = HAlloc(c_p, REF_THING_SIZE);
     ref = erts_sched_make_ref_in_buffer(esdp, hp);
@@ -1871,7 +1871,7 @@ access_sched_local_btm(Process *c_p, Eterm pid,
     if (!c_p)
 	esdp = erts_get_scheduler_data();
     else {
-	esdp = ERTS_PROC_GET_SCHDATA(c_p);
+	esdp = erts_proc_sched_data(c_p);
 	ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data());
     }
 
@@ -2138,7 +2138,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
 	    goto no_timer;
     }
 
-    esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    esdp = erts_proc_sched_data(c_p);
 
     trefn = internal_ref_numbers(tref);
     sid = erts_get_ref_numbers_thr_id(trefn);
@@ -2363,7 +2363,7 @@ typedef struct {
 
 int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
 {
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(p);
     ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}};
     ErtsBifTimerYieldState *ysp;
     int res;
@@ -2409,7 +2409,7 @@ detach_bif_timer(ErtsHLTimer *tmr, void *vesdp)
 
 int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
 {
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(p);
     ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}};
     ErtsBifTimerYieldState *ysp;
     int res;
@@ -2516,7 +2516,7 @@ BIF_RETTYPE send_after_3(BIF_ALIST_3)
     ErtsMonotonicTime timeout_pos;
     int short_time, tres;
 
-    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+    tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
 			     0, &timeout_pos, &short_time);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
@@ -2534,7 +2534,7 @@ BIF_RETTYPE send_after_4(BIF_ALIST_4)
     if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor))
 	BIF_ERROR(BIF_P, BADARG);
     
-    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+    tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
 			     abs, &timeout_pos, &short_time);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
@@ -2548,7 +2548,7 @@ BIF_RETTYPE start_timer_3(BIF_ALIST_3)
     ErtsMonotonicTime timeout_pos;
     int short_time, tres;
 
-    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+    tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
 			     0, &timeout_pos, &short_time);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
@@ -2566,7 +2566,7 @@ BIF_RETTYPE start_timer_4(BIF_ALIST_4)
     if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor))
 	BIF_ERROR(BIF_P, BADARG);
 
-    tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL,
+    tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
 			     abs, &timeout_pos, &short_time);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
@@ -2720,7 +2720,7 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo,
 int
 erts_set_proc_timer_term(Process *c_p, Eterm etmo)
 {
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     ErtsMonotonicTime tmo, timeout_pos;
     int short_time, tres;
 
@@ -2742,7 +2742,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo)
 void
 erts_set_proc_timer_uword(Process *c_p, UWord tmo)
 {
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
 
     ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer)
 		    == ERTS_PTMR_NONE);
@@ -2776,7 +2776,7 @@ erts_cancel_proc_timer(Process *c_p)
 	erts_smp_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE);
 	return;
     }
-    continue_cancel_ptimer(ERTS_PROC_GET_SCHDATA(c_p),
+    continue_cancel_ptimer(erts_proc_sched_data(c_p),
 			   (ErtsTimer *) tval);
 }
 
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 9bb6e40a11..579f6e427d 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -492,7 +492,6 @@ queue_messages(Process* receiver,
     erts_proc_notify_new_message(receiver, receiver_locks);
 #else
     erts_proc_notify_new_message(receiver, 0);
-    ERTS_HOLE_CHECK(receiver);
 #endif
     return res;
 }
@@ -601,7 +600,9 @@ erts_try_alloc_message_on_heap(Process *pp,
 
     ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ));
 
-    if (
+    if ((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
+	goto in_message_fragment;
+    else if (
 #if defined(ERTS_SMP)
 	*plp & ERTS_PROC_LOCK_MAIN
 #else
@@ -611,7 +612,7 @@ erts_try_alloc_message_on_heap(Process *pp,
 #ifdef ERTS_SMP
     try_on_heap:
 #endif
-	if ((*psp & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+	if (((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
 	    || (pp->flags & F_DISABLE_GC)
 	    || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) {
 	    /*
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 0e625f213b..544bc8b983 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -338,7 +338,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
 {
 #ifdef ERTS_ENABLE_MSACC
     ErtsMsAcc *msacc =  ERTS_MSACC_TSD_GET();
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     Eterm ref;
     ErtsMSAccReq *msaccrp;
     Eterm *hp;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index a695a028ba..8a3007d52a 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -20,6 +20,23 @@
 /* Erlang Native InterFace
  */
 
+/*
+ * Environment contains a pointer to currently executing process.
+ * In the dirty case this pointer do however not point to the
+ * actual process structure of the executing process, but instead
+ * a "shadow process structure". This in order to be able to handle
+ * heap allocation without the need to acquire the main lock on
+ * the process.
+ *
+ * The dirty process is allowed to allocate on the heap without
+ * the main lock, i.e., incrementing htop, but is not allowed to
+ * modify mbuf, offheap, etc without the main lock. The dirty
+ * process moves mbuf list and offheap list of the shadow process
+ * structure into the real structure when the dirty nif call
+ * completes.
+ */
+
+
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
@@ -81,6 +98,43 @@ void dtrace_nifenv_str(ErlNifEnv *, char *);
 #define MIN_HEAP_FRAG_SZ 200
 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp);
 
+static ERTS_INLINE int
+is_scheduler(void)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    if (!esdp)
+	return 0;
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+	return -1;
+    return 1;
+}
+
+static ERTS_INLINE void
+execution_state(ErlNifEnv *env, Process **c_pp, int *schedp)
+{
+    if (schedp)
+	*schedp = is_scheduler();
+    if (c_pp) {
+	if (!env || env->proc->common.id == ERTS_INVALID_PID)
+	    *c_pp = NULL;
+	else {
+	    Process *c_p = env->proc;
+
+	    if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC))
+		ASSERT(is_scheduler() > 0);
+	    else {
+		c_p = env->proc->next;
+		ASSERT(is_scheduler() < 0);
+		ASSERT(c_p && env->proc->common.id == c_p->common.id);
+	    }
+
+	    *c_pp = c_p;
+
+	    ASSERT(!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC));
+	}
+    }
+}
+
 static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need)
 {
     Eterm* hp = env->hp;
@@ -124,6 +178,9 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need)
 void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
                   Process* tracee)
 {
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ErtsSchedulerData *esdp;
+#endif
     env->mod_nif = mod_nif;
     env->proc = p;
     env->hp = HEAP_TOP(p);
@@ -133,6 +190,61 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
     env->tmp_obj_list = NULL;
     env->exception_thrown = 0;
     env->tracee = tracee;
+
+    ASSERT(p->common.id != ERTS_INVALID_PID);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    esdp = erts_get_scheduler_data();
+    ASSERT(esdp);
+
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+#ifdef DEBUG
+	erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+
+	ASSERT(p->scheduler_data == esdp);
+	ASSERT((state & (ERTS_PSFLG_RUNNING
+			 | ERTS_PSFLG_RUNNING_SYS))
+	       && !(state & (ERTS_PSFLG_DIRTY_RUNNING
+			     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+#endif
+
+    }
+    else {
+	Process *sproc;
+#ifdef DEBUG
+	erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+
+	ASSERT(!p->scheduler_data);
+	ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
+	       && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+#endif
+
+	sproc = esdp->dirty_shadow_process;
+	ASSERT(sproc);
+	ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+	ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
+	       == (ERTS_PSFLG_ACTIVE
+		   | ERTS_PSFLG_DIRTY_RUNNING
+		   | ERTS_PSFLG_PROXY));
+
+	sproc->next = p;
+	sproc->common.id = p->common.id;
+	sproc->htop = p->htop;
+	sproc->stop = p->stop;
+	sproc->hend = p->hend;
+	sproc->heap = p->heap;
+	sproc->abandoned_heap = p->abandoned_heap;
+	sproc->heap_sz = p->heap_sz;
+	sproc->high_water = p->high_water;
+	sproc->old_hend = p->old_hend;
+	sproc->old_htop = p->old_htop;
+	sproc->old_heap = p->old_heap;
+	sproc->mbuf = NULL;
+	sproc->mbuf_sz = 0;
+	ERTS_INIT_OFF_HEAP(&sproc->off_heap);
+	env->proc = sproc;
+    }
+#endif
 }
 
 /* Temporary object header, auto-deallocated when NIF returns
@@ -157,18 +269,75 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env)
 void erts_post_nif(ErlNifEnv* env)
 {
     erts_unblock_fpe(env->fpe_was_unmasked);
-    if (env->heap_frag == NULL) {
-	ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
-	ASSERT(env->hp >= HEAP_TOP(env->proc));
-	ASSERT(env->hp <= HEAP_LIMIT(env->proc));	
-	HEAP_TOP(env->proc) = env->hp;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC))
+#endif
+    {
+	ASSERT(is_scheduler() > 0);
+	if (env->heap_frag == NULL) {
+	    ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
+	    ASSERT(env->hp >= HEAP_TOP(env->proc));
+	    ASSERT(env->hp <= HEAP_LIMIT(env->proc));	
+	    HEAP_TOP(env->proc) = env->hp;
+	}
+	else {
+	    ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
+	    ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
+	    env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+	    ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
+	}
+	env->exiting = ERTS_PROC_IS_EXITING(env->proc);
     }
-    else {
-	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
-	env->heap_frag->used_size = env->hp - env->heap_frag->mem;
-	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    else { /* Dirty nif call using shadow process struct */
+	Process *c_p = env->proc->next;
+
+	ASSERT(is_scheduler() < 0);
+	ASSERT(env->proc->common.id == c_p->common.id);
+
+	if (!env->heap_frag) {
+	    ASSERT(env->hp_end == HEAP_LIMIT(c_p));
+	    ASSERT(env->hp >= HEAP_TOP(c_p));
+	    ASSERT(env->hp <= HEAP_LIMIT(c_p));
+	    HEAP_TOP(c_p) = env->hp;
+	}
+	else {
+	    ASSERT(env->hp_end != HEAP_LIMIT(c_p));
+	    ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
+
+	    HEAP_TOP(c_p) = HEAP_TOP(env->proc);
+	    env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+
+	    ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
+
+	    if (c_p->mbuf) {
+		ErlHeapFragment *bp;
+		for (bp = env->proc->mbuf; bp->next; bp = bp->next)
+		    ;
+		bp->next = c_p->mbuf;
+	    }
+
+	    c_p->mbuf = env->proc->mbuf;
+	    c_p->mbuf_sz += env->proc->mbuf_sz;
+
+	}
+
+	if (!c_p->off_heap.first)
+	    c_p->off_heap.first = env->proc->off_heap.first;
+	else if (env->proc->off_heap.first) {
+	    struct erl_off_heap_header *ohhp;
+	    for (ohhp = env->proc->off_heap.first; ohhp->next; ohhp = ohhp->next)
+		;
+	    ohhp->next = c_p->off_heap.first;
+	    c_p->off_heap.first = env->proc->off_heap.first;
+	}
+	c_p->off_heap.overhead += env->proc->off_heap.overhead;
+
+	env->exiting = ERTS_PROC_IS_EXITING(c_p);
+	BUMP_ALL_REDS(c_p);
     }
+#endif
     free_tmp_objs(env);
 }
 
@@ -400,9 +569,8 @@ error:
 
 #endif
 
-int
-enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
-          ErlNifEnv* msg_env, ERL_NIF_TERM msg)
+int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
+	      ErlNifEnv* msg_env, ERL_NIF_TERM msg)
 {
     struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
     ErtsProcLocks rp_locks = 0;
@@ -413,34 +581,32 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     Process* c_p;
     ErtsMessage *mp;
     Eterm receiver = to_pid->pid;
-    int flush_me = 0;
-    ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    int scheduler = esdp ? esdp->no : 0;
+    int scheduler;
 
-    if (env != NULL) {
-	c_p = env->proc;
-	if (receiver == c_p->common.id) {
+    execution_state(env, &c_p, &scheduler);
+
+#ifndef ERTS_SMP
+    if (!scheduler) {
+	erts_exit(ERTS_ABORT_EXIT,
+		  "enif_send: called from non-scheduler thread on non-SMP VM");
+	return 0;
+   }
+#endif
+
+    if (scheduler > 0) { /* Normal scheduler */
+	rp = erts_proc_lookup(receiver);
+	if (c_p == rp)
 	    rp_locks = ERTS_PROC_LOCK_MAIN;
-	    flush_me = 1;
-	}
     }
     else {
-#ifdef ERTS_SMP
-	c_p = NULL;
-#else
-	erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
-#endif
+	if (c_p && ERTS_PROC_IS_EXITING(c_p))
+	    return 0;
+	rp = erts_pid2proc_opt(c_p, 0, receiver, rp_locks,
+			       ERTS_P2P_FLG_INC_REFC);
     }
-
-    rp = (scheduler
-	  ? erts_proc_lookup(receiver)
-	  : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
-			      receiver, rp_locks, ERTS_P2P_FLG_INC_REFC));
-
-    if (rp == NULL) {
-	ASSERT(env == NULL || receiver != c_p->common.id);
+    if (rp == NULL)
 	return 0;
-    }
+
     if (menv) {
         flush_env(msg_env);
         mp = erts_alloc_message(0, NULL);
@@ -465,10 +631,6 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 
     ERL_MESSAGE_TERM(mp) = msg;
 
-    if (flush_me) {
-	flush_env(env); /* Needed for ERTS_HOLE_CHECK */
-    }
-
     if (!env || !env->tracee) {
 
         if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND))
@@ -546,11 +708,9 @@ done:
     if (rp_locks & ~lc_locks)
 	erts_smp_proc_unlock(rp, rp_locks & ~lc_locks);
 #endif
-    if (!scheduler)
+    if (scheduler <= 0)
 	erts_proc_dec_refc(rp);
-    if (flush_me) {
-	cache_env(env);
-    }
+
     return 1;
 }
 
@@ -558,26 +718,52 @@ int
 enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port,
                   ErlNifEnv *msg_env, ERL_NIF_TERM msg)
 {
-
-    ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    int scheduler = esdp ? esdp->no : 0;
+    int iflags = (erts_port_synchronous_ops
+		  ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+		  : ERTS_PORT_SFLGS_INVALID_LOOKUP);
+    int scheduler;
+    Process *c_p;
     Port *prt;
+    int res;
 
-    if (scheduler == 0 || !env)
-        return 0;
+    if (!env)
+	erts_exit(ERTS_ABORT_EXIT, "enif_port_command: env == NULL");
+
+    execution_state(env, &c_p, &scheduler);
+
+    if (!c_p)
+	c_p = env->proc;
 
-    prt = erts_port_lookup(to_port->port_id,
-                           (erts_port_synchronous_ops
-                            ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
-                            : ERTS_PORT_SFLGS_INVALID_LOOKUP));
+    if (scheduler > 0)
+	prt = erts_port_lookup(to_port->port_id, iflags);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    else if (scheduler < 0) {
+	if (ERTS_PROC_IS_EXITING(c_p))
+	    return 0;
+	prt = erts_thr_port_lookup(to_port->port_id, iflags);
+    }
+#endif
+    else {
+	erts_exit(ERTS_ABORT_EXIT, "enif_port_command: "
+		  "called from non-scheduler thread");
+    }
 
     if (!prt)
-        return 0;
+	res = 0;
+    else {
+
+	if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
+	    trace_port_receive(prt, c_p->common.id, am_command, msg);
 
-    if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
-        trace_port_receive(prt, env->proc->common.id, am_command, msg);
+	res = erts_port_output_async(prt, c_p->common.id, msg);
+    }
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (scheduler < 0)
+	erts_port_dec_refc(prt);
+#endif
 
-    return erts_port_output_async(prt, env->proc->common.id, msg);
+    return res;
 }
 
 ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
@@ -1039,15 +1225,21 @@ Eterm enif_make_badarg(ErlNifEnv* env)
 
 Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason)
 {
+    Process *c_p;
+
+    execution_state(env, &c_p, NULL);
+
     env->exception_thrown = 1;
-    env->proc->fvalue = reason;
-    BIF_ERROR(env->proc, EXC_ERROR);
+    c_p->fvalue = reason;
+    BIF_ERROR(c_p, EXC_ERROR);
 }
 
 int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason)
 {
     if (env->exception_thrown && reason != NULL) {
-	*reason = env->proc->fvalue;
+	Process *c_p;
+	execution_state(env, &c_p, NULL);
+	*reason = c_p->fvalue;
     }
     return env->exception_thrown;
 }
@@ -1441,56 +1633,71 @@ int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list
     return 1;
 }
 
+int enif_is_current_process_alive(ErlNifEnv* env)
+{
+    Process *c_p;
+    int scheduler;
+
+    execution_state(env, &c_p, &scheduler);
+
+    if (!c_p)
+	erts_exit(ERTS_ABORT_EXIT,
+		  "enif_is_current_process_alive: "
+                  "Invalid environment");
+
+    if (!scheduler)
+	erts_exit(ERTS_ABORT_EXIT, "enif_is_current_process_alive: "
+		  "called from non-scheduler thread");
+
+    return !ERTS_PROC_IS_EXITING(c_p);
+}
+
 int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc)
 {
-    ErtsProcLocks rp_locks = 0; /* We don't need any locks,
-                                   just to check if it is alive */
-    Eterm target = proc->pid;
-    Process* rp;
-    Process* c_p;
-    int scheduler = erts_get_scheduler_id() != 0;
+    int scheduler;
 
-    if (env != NULL) {
-	c_p = env->proc;
-	if (target == c_p->common.id) {
-            /* We are alive! */
-	    return 1;
-	}
-    }
+    execution_state(env, NULL, &scheduler);
+
+    if (scheduler > 0)
+	return !!erts_proc_lookup(proc->pid);
     else {
 #ifdef ERTS_SMP
-	c_p = NULL;
+	Process* rp = erts_pid2proc_opt(NULL, 0, proc->pid, 0,
+					ERTS_P2P_FLG_INC_REFC);
+	if (rp)
+	    erts_proc_dec_refc(rp);
+	return !!rp;
 #else
-	erts_exit(ERTS_ABORT_EXIT,"enif_is_process_alive: "
-                  "env==NULL on non-SMP VM");
-#endif
-    }
-
-    rp = (scheduler
-	  ? erts_proc_lookup(target)
-	  : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
-			      target, rp_locks, ERTS_P2P_FLG_INC_REFC));
-    if (rp == NULL) {
-        ASSERT(env == NULL || target != c_p->common.id);
+	erts_exit(ERTS_ABORT_EXIT, "enif_is_process_alive: "
+		  "called from non-scheduler thread");
 	return 0;
-    } else {
-        if (!scheduler)
-            erts_proc_dec_refc(rp);
-        return 1;
+#endif
     }
 }
 
 int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port)
 {
-    /* only allowed if called from scheduler */
-    if (erts_get_scheduler_id() == 0)
-        erts_exit(ERTS_ABORT_EXIT,"enif_is_port_alive: called from non-scheduler");
+    int scheduler;
+    Uint32 iflags = (erts_port_synchronous_ops
+		     ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+		     : ERTS_PORT_SFLGS_INVALID_LOOKUP);
 
-    return erts_port_lookup(
-        port->port_id,
-        (erts_port_synchronous_ops
-         ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
-         : ERTS_PORT_SFLGS_INVALID_LOOKUP)) != NULL;
+    execution_state(env, NULL, &scheduler);
+
+    if (scheduler > 0)
+	return !!erts_port_lookup(port->port_id, iflags);
+    else {
+#ifdef ERTS_SMP
+	Port *prt = erts_thr_port_lookup(port->port_id, iflags);
+	if (prt)
+	    erts_port_dec_refc(prt);
+	return !!prt;
+#else
+	erts_exit(ERTS_ABORT_EXIT, "enif_is_port_alive: "
+		  "called from non-scheduler thread");
+	return 0;
+#endif
+    }
 }
 
 ERL_NIF_TERM
@@ -1957,16 +2164,19 @@ void* enif_dlsym(void* handle, const char* symbol,
 
 int enif_consume_timeslice(ErlNifEnv* env, int percent)
 {
+    Process *proc;
     Sint reds;
 
+    execution_state(env, &proc, NULL);
+
     ASSERT(is_proc_bound(env) && percent >= 1 && percent <= 100);
     if (percent < 1) percent = 1;
     else if (percent > 100) percent = 100;
 
     reds = ((CONTEXT_REDS+99) / 100) * percent;
     ASSERT(reds > 0 && reds <= CONTEXT_REDS);
-    BUMP_REDS(env->proc, reds);
-    return ERTS_BIF_REDS_LEFT(env->proc) == 0;
+    BUMP_REDS(proc, reds);
+    return ERTS_BIF_REDS_LEFT(proc) == 0;
 }
 
 /*
@@ -2061,10 +2271,19 @@ static ERL_NIF_TERM
 init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
 		    int need_save, int argc, const ERL_NIF_TERM argv[])
 {
-    Process* proc = env->proc;
-    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
+    Process* proc;
+    Eterm* reg;
     NifExport* ep;
-    int i;
+    int i, scheduler;
+
+    execution_state(env, &proc, &scheduler);
+
+    ASSERT(scheduler);
+
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    reg = erts_proc_sched_data(proc)->x_reg_array;
 
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     if (!ep)
@@ -2076,12 +2295,13 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     }
     if (env->exception_thrown) {
 	ep->exception_thrown = 1;
-	ep->rootset[0] = env->proc->fvalue;
+	ep->rootset[0] = proc->fvalue;
     } else {
 	ep->exception_thrown = 0;
 	ep->rootset[0] = NIL;
     }
-    ERTS_VBUMP_ALL_REDS(proc);
+    if (scheduler > 0)
+	ERTS_VBUMP_ALL_REDS(proc);
     for (i = 0; i < argc; i++) {
 	if (need_save)
 	    ep->rootset[i+1] = reg[i];
@@ -2113,7 +2333,12 @@ static void
 restore_nif_mfa(Process* proc, NifExport* ep, int exception)
 {
     int i;
-    Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array;
+    Eterm* reg = erts_proc_sched_data(proc)->x_reg_array;
+
+    ERTS_SMP_LC_ASSERT(!(proc->static_flags
+			 & ERTS_STC_FLG_SHADOW_PROC));
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc)
+		       & ERTS_PROC_LOCK_MAIN);
 
     proc->current[0] = ep->saved_mfa[0];
     proc->current[1] = ep->saved_mfa[1];
@@ -2138,11 +2363,13 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception)
 static ERL_NIF_TERM
 dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    Process* proc = env->proc;
+    Process* proc;
     NifExport* ep;
 
+    execution_state(env, &proc, NULL);
+
     ASSERT(argc == 1);
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
     ASSERT(!ep->exception_thrown);
@@ -2157,10 +2384,12 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 static ERL_NIF_TERM
 dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    Process* proc = env->proc;
+    Process* proc;
     NifExport* ep;
 
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
+    execution_state(env, &proc, NULL);
+
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
     ASSERT(ep->exception_thrown);
@@ -2177,23 +2406,32 @@ dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 static ERL_NIF_TERM
 execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    Process* proc = env->proc;
-    NativeFunPtr fp = (NativeFunPtr) proc->current[6];
+    Process* proc;
+    NativeFunPtr fp;
     NifExport* ep;
     ERL_NIF_TERM result;
 
-    ASSERT(ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data));
+    execution_state(env, &proc, NULL);
+
+    fp = (NativeFunPtr) proc->current[6];
+
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
 
     /*
      * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution
      */
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep);
+    ASSERT(ep && fp);
     ep->fp = NULL;
     erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
 						   | ERTS_PSFLG_DIRTY_IO_PROC));
+
+    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+
     result = (*fp)(env, argc, argv);
 
+    erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+
     if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL)
 	close_lib(env->mod_nif);
     /*
@@ -2230,29 +2468,49 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 static ERTS_INLINE ERL_NIF_TERM
 schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[])
 {
-    erts_aint32_t state, n, a;
-    Process* proc = env->proc;
-    NativeFunPtr fp = (NativeFunPtr) proc->current[6];
+    ERL_NIF_TERM result;
+    erts_aint32_t act, dirty_flag;
+    Process* proc;
+    NativeFunPtr fp;
     NifExport* ep;
-    int need_save;
+    int need_save, scheduler;
+
+    execution_state(env, &proc, &scheduler);
+    if (scheduler <= 0) {
+	ASSERT(scheduler < 0);
+	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+    }
+
+    fp = (NativeFunPtr) proc->current[6];
+
+    ASSERT(fp);
 
     ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND);
 
-    a = erts_smp_atomic32_read_acqb(&proc->state);
-    while (1) {
-	n = state = a;
+    if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
+	dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC;
+    else
+	dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC;
+
+    act = erts_smp_atomic32_read_bor_nob(&proc->state, dirty_flag);
+    if (!(act & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)))
+	erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
+    else if ((act & (ERTS_PSFLG_DIRTY_CPU_PROC
+		     | ERTS_PSFLG_DIRTY_IO_PROC)) & ~dirty_flag) {
+	/* clear other flag... */
 	if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	    n |= ERTS_PSFLG_DIRTY_CPU_PROC;
+	    dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC;
 	else
-	    n |= ERTS_PSFLG_DIRTY_IO_PROC;
-	a = erts_smp_atomic32_cmpxchg_mb(&proc->state, n, state);
-	if (a == state)
-	    break;
+	    dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC;
+	erts_smp_atomic32_read_band_nob(&proc->state, ~dirty_flag);
     }
-    erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
+
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     need_save = (ep == NULL || is_non_value(ep->saved_mfa[0]));
-    return init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv);
+    result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv);
+    if (scheduler <= 0)
+	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+    return result;
 }
 
 static ERL_NIF_TERM
@@ -2277,11 +2535,14 @@ schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 static ERL_NIF_TERM
 execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    Process* proc = env->proc;
-    NativeFunPtr fp = (NativeFunPtr) proc->current[6];
+    Process* proc;
+    NativeFunPtr fp;
     NifExport* ep;
     ERL_NIF_TERM result;
 
+    execution_state(env, &proc, NULL);
+    fp = (NativeFunPtr) proc->current[6];
+
     ASSERT(!env->exception_thrown);
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
@@ -2304,10 +2565,10 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 		  ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]),
 		  int argc, const ERL_NIF_TERM argv[])
 {
-    Process* proc = env->proc;
+    Process* proc;
     NifExport* ep;
     ERL_NIF_TERM fun_name_atom, result;
-    int need_save;
+    int need_save, scheduler;
 
     if (argc > MAX_ARG)
 	return enif_make_badarg(env);
@@ -2315,6 +2576,13 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
     if (enif_is_exception(env, fun_name_atom))
 	return fun_name_atom;
 
+    execution_state(env, &proc, &scheduler);
+    if (scheduler <= 0) {
+	if (scheduler == 0)
+	    enif_make_badarg(env);
+	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+    }
+
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     need_save = (ep == NULL || is_non_value(ep->saved_mfa[0]));
 
@@ -2326,12 +2594,15 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 	    sched_fun = schedule_dirty_io_nif;
 	else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND)
 	    sched_fun = schedule_dirty_cpu_nif;
-	else
-	    return enif_make_badarg(env);
+	else {
+	    result = enif_make_badarg(env);
+	    goto done;
+	}
 	result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv);
 #else
-	return enif_make_badarg(env);
+	result = enif_make_badarg(env);
 #endif
+	goto done;
     }
     else
 	result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv);
@@ -2339,18 +2610,28 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
     ep->exp.code[1] = (BeamInstr) fun_name_atom;
+
+done:
+    if (scheduler < 0)
+	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+
     return result;
 }
 
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
-
 int
 enif_is_on_dirty_scheduler(ErlNifEnv* env)
 {
-    return ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data);
-}
+    int scheduler;
+    Process *c_p;
+
+    execution_state(env, &c_p, &scheduler);
 
-#endif /* ERL_NIF_DIRTY_SCHEDULER_SUPPORT */
+    if (!c_p || !scheduler)
+	erts_exit(ERTS_ABORT_EXIT, "enif_is_on_dirty_scheduler: "
+		  "Invalid env");
+
+    return scheduler < 0;
+}
 
 /* Maps */
 
@@ -3051,16 +3332,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		       (BeamInstr) BeamOp(op_i_generic_breakpoint));
 		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
 	    }
+#ifdef ERTS_DIRTY_SCHEDULERS
 	    if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
 		&& (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) {
-#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
 		code_ptr[5+3] = (BeamInstr) f->fptr;
 		code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
 		    (BeamInstr) schedule_dirty_io_nif :
 		    (BeamInstr) schedule_dirty_cpu_nif;
-#endif
 	    }
 	    else
+#endif
 		code_ptr[5+1] = (BeamInstr) f->fptr;
 	    code_ptr[5+2] = (BeamInstr) lib;
 	    f = next_func(entry, &incr, f);
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 93e4313791..1bdac51d1f 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -166,6 +166,7 @@ ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTim
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_now_time, (ErlNifEnv *env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_cpu_time, (ErlNifEnv *env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_unique_integer, (ErlNifEnv *env, ErlNifUniqueInteger properties));
+ERL_NIF_API_FUNC_DECL(int, enif_is_current_process_alive, (ErlNifEnv *env));
 ERL_NIF_API_FUNC_DECL(int, enif_is_process_alive, (ErlNifEnv *env, ErlNifPid *pid));
 ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port_id));
 ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id));
@@ -321,6 +322,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_now_time ERL_NIF_API_FUNC_MACRO(enif_now_time)
 #  define enif_cpu_time ERL_NIF_API_FUNC_MACRO(enif_cpu_time)
 #  define enif_make_unique_integer ERL_NIF_API_FUNC_MACRO(enif_make_unique_integer)
+#  define enif_is_current_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_current_process_alive)
 #  define enif_is_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_process_alive)
 #  define enif_is_port_alive ERL_NIF_API_FUNC_MACRO(enif_is_port_alive)
 #  define enif_get_local_port ERL_NIF_API_FUNC_MACRO(enif_get_local_port)
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index c2588e718d..f0075ca2b9 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -487,6 +487,7 @@ ERTS_GLB_INLINE Port*erts_id2port(Eterm id);
 ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
 ERTS_GLB_INLINE void erts_port_release(Port *);
 #ifdef ERTS_SMP
+ERTS_GLB_INLINE Port *erts_thr_port_lookup(Eterm id, Uint32 invalid_sflgs);
 ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs);
 ERTS_GLB_INLINE void erts_thr_port_release(Port *prt);
 #endif
@@ -626,6 +627,44 @@ erts_port_release(Port *prt)
 }
 
 #ifdef ERTS_SMP
+/*
+ * erts_thr_id2port_sflgs() and erts_port_dec_refc(prt) can
+ * be used by unmanaged threads in the SMP case.
+ */
+ERTS_GLB_INLINE Port *
+erts_thr_port_lookup(Eterm id, Uint32 invalid_sflgs)
+{
+    Port *prt;
+    ErtsThrPrgrDelayHandle dhndl;
+
+    if (is_not_internal_port(id))
+	return NULL;
+
+    dhndl = erts_thr_progress_unmanaged_delay();
+
+    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
+					     internal_port_index(id));
+
+    if (!prt || prt->common.id != id) {
+	erts_thr_progress_unmanaged_continue(dhndl);
+	return NULL;
+    }
+    else {
+	erts_aint32_t state;
+	erts_port_inc_refc(prt);
+
+	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
+	    erts_thr_progress_unmanaged_continue(dhndl);
+
+	state = erts_atomic32_read_acqb(&prt->state);
+	if (state & invalid_sflgs) {
+	    erts_port_dec_refc(prt);
+	    return NULL;
+	}
+
+	return prt;
+    }
+}
 
 /*
  * erts_thr_id2port_sflgs() and erts_thr_port_release() can
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 36b0af0c1a..b50ca6d009 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -409,6 +409,10 @@ ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
 #ifdef ERTS_DIRTY_SCHEDULERS
 ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
 ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
+typedef union {
+    Process dsp;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))];
+} ErtsAlignedDirtyShadowProcess;
 #endif
 
 typedef union {
@@ -589,6 +593,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
     valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
+    valid |= ERTS_SSI_AUX_WORK_PENDING_EXITERS;
 #endif
 #if HAVE_ERTS_MSEG
     valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
@@ -611,7 +616,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 #endif
 
 #ifdef ERTS_SMP
-static void handle_pending_exiters(ErtsProcList *);
+static void do_handle_pending_exiters(ErtsProcList *);
 static void wake_scheduler(ErtsRunQueue *rq);
 #endif
 
@@ -679,6 +684,8 @@ erts_pre_init_process(void)
 	= "MISC_THR_PRGR";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX]
 	= "MISC";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX]
+	= "PENDING_EXITERS";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX]
 	= "SET_TMO";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX]
@@ -1156,7 +1163,7 @@ reply_sched_wall_time(void *vswtrp)
 Eterm
 erts_sched_wall_time_request(Process *c_p, int set, int enable)
 {
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     Eterm ref;
     ErtsSchedWallTimeReq *swtrp;
     Eterm *hp;
@@ -1234,7 +1241,7 @@ reply_system_check(void *vscrp)
 
 
 Eterm erts_system_check_request(Process *c_p) {
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     Eterm ref;
     ErtsSystemCheckReq *scrp;
     Eterm *hp;
@@ -2336,6 +2343,30 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiti
 
 #endif
 
+#ifdef ERTS_SMP
+
+static ERTS_INLINE erts_aint32_t
+handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsProcList *pnd_xtrs;
+    ErtsRunQueue *rq;
+
+    rq = awdp->esdp->run_queue;
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS);
+
+    erts_smp_runq_lock(rq);
+    pnd_xtrs = rq->procs.pending_exiters;
+    rq->procs.pending_exiters = NULL;
+    erts_smp_runq_unlock(rq);
+
+    if (erts_proclist_fetch(&pnd_xtrs, NULL))
+	do_handle_pending_exiters(pnd_xtrs);
+
+    return aux_work & ~ERTS_SSI_AUX_WORK_PENDING_EXITERS;
+}
+
+#endif
+
 static ERTS_INLINE erts_aint32_t
 handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 {
@@ -2427,6 +2458,10 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
 		    handle_misc_aux_work);
 
+#ifdef ERTS_SMP
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_PENDING_EXITERS,
+		    handle_pending_exiters);
+#endif
 
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
 		    handle_setup_aux_work_timer);
@@ -3979,6 +4014,33 @@ schedule_bound_processes(ErtsRunQueue *rq,
     }
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static ERTS_INLINE void
+clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit)
+{
+#ifdef DEBUG
+    erts_aint32_t old;
+#endif
+    erts_aint32_t qb = prio_bit;
+    if (rq == ERTS_DIRTY_CPU_RUNQ)
+	qb <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET;
+    else {
+	ASSERT(rq == ERTS_DIRTY_IO_RUNQ);
+	qb <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET;
+    }
+#ifdef DEBUG
+    old = (int)
+#else
+	(void)
+#endif
+	erts_smp_atomic32_read_band_mb(&p->dirty_state, ~qb);
+    ASSERT(old & qb);
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+
 static void
 evacuate_run_queue(ErtsRunQueue *rq,
 		   ErtsStuckBoundProcesses *sbpp)
@@ -4141,29 +4203,8 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	    }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-
-	    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
-		erts_aint32_t dqbit = qbit;
-#ifdef DEBUG
-		erts_aint32_t old_dqbit;
-#endif
-
-		if (rq == ERTS_DIRTY_CPU_RUNQ)
-		    dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET;
-		else {
-		    ASSERT(rq == ERTS_DIRTY_IO_RUNQ);
-		    dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET;
-		}
-
-#ifdef DEBUG
-		old_dqbit = (int)
-#else
-		    (void)
-#endif
-		    erts_smp_atomic32_read_band_mb(&real_proc->dirty_state,
-						   ~dqbit);
-		ASSERT(old_dqbit & dqbit);
-	    }
+	    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
+		clear_proc_dirty_queue_bit(real_proc, rq, qbit);
 #endif
 
 	    if (ERTS_PSFLG_BOUND & real_state) {
@@ -5653,7 +5694,8 @@ static void
 init_scheduler_data(ErtsSchedulerData* esdp, int num,
 		    ErtsSchedulerSleepInfo* ssi,
 		    ErtsRunQueue* runq,
-		    char** daww_ptr, size_t daww_sz)
+		    char** daww_ptr, size_t daww_sz,
+		    Process *shadow_proc)
 {
     esdp->timer_wheel = NULL;
 #ifdef ERTS_SMP
@@ -5677,6 +5719,15 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 	esdp->no = (Uint) num;
 	ERTS_DIRTY_SCHEDULER_NO(esdp) = 0;
     }
+    esdp->dirty_shadow_process = shadow_proc;
+    if (shadow_proc) {
+	erts_init_empty_process(shadow_proc);
+	erts_smp_atomic32_init_nob(&shadow_proc->state,
+				   (ERTS_PSFLG_ACTIVE
+				    | ERTS_PSFLG_DIRTY_RUNNING
+				    | ERTS_PSFLG_PROXY));
+	shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC;
+    }
 #else
     esdp->no = (Uint) num;
 #endif
@@ -5928,30 +5979,40 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     for (ix = 0; ix < n; ix++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 	init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix),
-			    ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz);
+			    ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz,
+			    NULL);
     }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef ERTS_SMP
-    erts_aligned_dirty_cpu_scheduler_data =
-	erts_alloc_permanent_cache_aligned(
-					   ERTS_ALC_T_SCHDLR_DATA,
-					   no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerData));
-    for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) {
-	ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
-	init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix),
-			    ERTS_DIRTY_CPU_RUNQ, NULL, 0);
-    }
-    erts_aligned_dirty_io_scheduler_data =
-	erts_alloc_permanent_cache_aligned(
-					   ERTS_ALC_T_SCHDLR_DATA,
-					   no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerData));
-    for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
-	ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
-	init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
-			    ERTS_DIRTY_IO_RUNQ, NULL, 0);
+    {
+	int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers;
+	int adspix = 0;
+	ErtsAlignedDirtyShadowProcess *adsp =
+	    erts_alloc_permanent_cache_aligned(
+		ERTS_ALC_T_SCHDLR_DATA,
+		dirty_scheds * sizeof(ErtsAlignedDirtyShadowProcess));
+						   
+	erts_aligned_dirty_cpu_scheduler_data =
+	    erts_alloc_permanent_cache_aligned(
+		ERTS_ALC_T_SCHDLR_DATA,
+		dirty_scheds * sizeof(ErtsAlignedSchedulerData));
+
+	erts_aligned_dirty_io_scheduler_data =
+	    &erts_aligned_dirty_cpu_scheduler_data[no_dirty_cpu_schedulers];
+
+	for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) {
+	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix),
+				ERTS_DIRTY_CPU_RUNQ, NULL, 0,
+				&adsp[adspix++].dsp);
+	}
+	for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
+	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
+	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
+				ERTS_DIRTY_IO_RUNQ, NULL, 0,
+				&adsp[adspix++].dsp);
+	}
     }
-#endif
 #endif
 
     init_misc_aux_work();
@@ -6167,7 +6228,7 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
     erts_aint32_t dact, max_qbit;
 
     /* Termination should be done on an ordinary scheduler */
-    if (actual & ERTS_PSFLG_EXITING) {
+    if ((*newp) & ERTS_PSFLG_EXITING) {
 	*newp &= ~ERTS_PSFLGS_DIRTY_WORK;
 	return ERTS_ENQUEUE_NORMAL_QUEUE;
     }
@@ -6176,7 +6237,7 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
      * If we have system tasks, we enqueue on ordinary run-queue
      * and take care of those system tasks first.
      */
-    if (actual & ERTS_PSFLG_ACTIVE_SYS)
+    if ((*newp) & ERTS_PSFLG_ACTIVE_SYS)
 	return ERTS_ENQUEUE_NORMAL_QUEUE;
 
     dact = erts_smp_atomic32_read_mb(&c_p->dirty_state);
@@ -6356,23 +6417,29 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
  * schedule_out_process() return with c_rq locked.
  */
 static ERTS_INLINE int
-schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Process *proxy)
+schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
+		     Process *proxy, int is_normal_sched)
 {
-    erts_aint32_t a, e, n, enq_prio = -1;
+    erts_aint32_t a, e, n, enq_prio = -1, running_flgs;
     int enqueue; /* < 0 -> use proxy */
     ErtsRunQueue* runq;
 
+    if (is_normal_sched)
+	running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS;
+    else
+	running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS;
+
     a = state;
 
     while (1) {
 	n = e = a;
 
-	ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
+	ASSERT(a & running_flgs);
 
 	enqueue = ERTS_ENQUEUE_NOT;
 
-	n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS);
-	if (a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)
+	n &= ~running_flgs;
+	if ((a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS))
 	    || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
 	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
 	}
@@ -6485,8 +6552,9 @@ change_proc_schedule_state(Process *p,
 			   ErtsProcLocks locks)
 {
     /*
-     * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS and
-     *       ERTS_PSFLG_ACTIVE_SYS are not allowed to be
+     * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS,
+     *       ERTS_PSFLG_DIRTY_RUNNING, ERTS_PSFLG_DIRTY_RUNNING_SYS
+     *       and ERTS_PSFLG_ACTIVE_SYS are not allowed to be
      *       altered by this function!
      */
     erts_aint32_t a = *statep, n;
@@ -6500,9 +6568,13 @@ change_proc_schedule_state(Process *p,
     ASSERT(!(a & ERTS_PSFLG_PROXY));
     ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING
 				 | ERTS_PSFLG_RUNNING_SYS
+				 | ERTS_PSFLG_DIRTY_RUNNING
+				 | ERTS_PSFLG_DIRTY_RUNNING_SYS
 				 | ERTS_PSFLG_ACTIVE_SYS)) == 0);
     ASSERT((set_state_flags & (ERTS_PSFLG_RUNNING
 			       | ERTS_PSFLG_RUNNING_SYS
+			       | ERTS_PSFLG_DIRTY_RUNNING
+			       | ERTS_PSFLG_DIRTY_RUNNING_SYS
 			       | ERTS_PSFLG_ACTIVE_SYS)) == 0);
 
     if (lock_status)
@@ -6526,8 +6598,16 @@ change_proc_schedule_state(Process *p,
 	if ((n & (ERTS_PSFLG_SUSPENDED
 		  | ERTS_PSFLG_RUNNING
 		  | ERTS_PSFLG_RUNNING_SYS
+		  | ERTS_PSFLG_DIRTY_RUNNING
+		  | ERTS_PSFLG_DIRTY_RUNNING_SYS
 		  | ERTS_PSFLG_IN_RUNQ
-		  | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) {
+		  | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    || (n & (ERTS_PSFLG_RUNNING
+		     | ERTS_PSFLG_RUNNING_SYS
+		     | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING
+#endif
+	    ) {
 	    /*
 	     * Active and seemingly need to be enqueued, but
 	     * process may be in a run queue via proxy, need
@@ -6551,7 +6631,9 @@ change_proc_schedule_state(Process *p,
 		   | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE)
 	    && (!(a & (ERTS_PSFLG_ACTIVE_SYS
 		       | ERTS_PSFLG_RUNNING
-		       | ERTS_PSFLG_RUNNING_SYS)
+		       | ERTS_PSFLG_RUNNING_SYS
+		       | ERTS_PSFLG_DIRTY_RUNNING
+		       | ERTS_PSFLG_DIRTY_RUNNING_SYS)
 		  && (!(a & ERTS_PSFLG_ACTIVE)
 		      || (a & ERTS_PSFLG_SUSPENDED))))) {
 	    /* We activated a prevously inactive process */
@@ -6693,7 +6775,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st)
 
 	enqueue = ERTS_ENQUEUE_NOT;
 	n |= ERTS_PSFLG_ACTIVE_SYS;
-	if (!(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)))
+	if (!(a & (ERTS_PSFLG_RUNNING
+		   | ERTS_PSFLG_RUNNING_SYS
+		   | ERTS_PSFLG_DIRTY_RUNNING
+		   | ERTS_PSFLG_DIRTY_RUNNING_SYS)))
 	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
@@ -6706,7 +6791,9 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st)
 
 	if (!(a & (ERTS_PSFLG_ACTIVE_SYS
 		   | ERTS_PSFLG_RUNNING
-		   | ERTS_PSFLG_RUNNING_SYS))
+		   | ERTS_PSFLG_RUNNING_SYS
+		   | ERTS_PSFLG_DIRTY_RUNNING
+		   | ERTS_PSFLG_DIRTY_RUNNING_SYS))
 	    && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) {
 	    /* We activated a prevously inactive process */
 	    profile_runnable_proc(p, am_active);
@@ -6746,11 +6833,16 @@ suspend_process(Process *c_p, Process *p)
 	if (c_p == p) {
 	    state = erts_smp_atomic32_read_bor_relb(&p->state,
 						    ERTS_PSFLG_SUSPENDED);
-	    ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
+	    ASSERT(state & (ERTS_PSFLG_RUNNING
+			    | ERTS_PSFLG_RUNNING_SYS
+			    | ERTS_PSFLG_DIRTY_RUNNING
+			    | ERTS_PSFLG_DIRTY_RUNNING_SYS));
 	    suspended = (state & ERTS_PSFLG_SUSPENDED) ? -1: 1;
 	}
 	else {
-	    while (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_EXITING))) {
+	    while (!(state & (ERTS_PSFLG_RUNNING
+			      | ERTS_PSFLG_DIRTY_RUNNING
+			      | ERTS_PSFLG_EXITING))) {
 		erts_aint32_t n, e;
 
 		n = e = state;
@@ -6776,8 +6868,11 @@ suspend_process(Process *c_p, Process *p)
 
 	    if ((state & (ERTS_PSFLG_ACTIVE
 			  | ERTS_PSFLG_ACTIVE_SYS
+			  | ERTS_PSFLG_DIRTY_ACTIVE_SYS
 			  | ERTS_PSFLG_RUNNING
 			  | ERTS_PSFLG_RUNNING_SYS
+			  | ERTS_PSFLG_DIRTY_RUNNING
+			  | ERTS_PSFLG_DIRTY_RUNNING_SYS
 			  | ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
 		/* We made process inactive */
 		profile_runnable_proc(p, am_inactive);
@@ -7759,8 +7854,10 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    plp = proclist_create(p);
 	    erts_proclist_store_last(&msbp->blckrs, plp);
 	    p->flags |= have_blckd_flg;
-	    ASSERT(schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0));
-	    ASSERT(p->scheduler_data->no == 1);
+	    ASSERT(normal
+		   ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL)
+		   : schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0));
+	    ASSERT(erts_proc_sched_data(p)->no == 1);
 	    if (schdlr_sspnd.msb.ongoing)
 		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 	    else
@@ -7780,7 +7877,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)
 		|| (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
 						       ERTS_SCHED_NORMAL) == 1)) {
-		ASSERT(p->scheduler_data->no == 1);
+		ASSERT(erts_proc_sched_data(p)->no == 1);
 		plp = proclist_create(p);
 		erts_proclist_store_last(&msbp->blckrs, plp);
 		if (schdlr_sspnd.msb.ongoing)
@@ -7830,7 +7927,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 		else
 		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED;
 	    }
-	    ASSERT(p->scheduler_data);
+	    ASSERT(erts_proc_sched_data(p));
 	}
     }
     else if (!msbp->ongoing) {
@@ -8420,9 +8517,23 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	if (!suspend_process(c_p, rp)) {
 	    /* Other process running */
 
-	    ASSERT(ERTS_PSFLG_RUNNING
+	    ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING)
 		   & erts_smp_atomic32_read_nob(&rp->state));
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    if (!suspend
+		&& (erts_smp_atomic32_read_nob(&rp->state)
+		    & ERTS_PSFLG_DIRTY_RUNNING)) {
+		ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
+		if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
+		    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+		    rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
+				       pid, pid_locks|ERTS_PROC_LOCK_STATUS);
+		}
+		goto done;
+	    }
+#endif
+
 	running:
 
 	    /*
@@ -8447,7 +8558,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	else {
 	    ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
 	    if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
-		if (ERTS_PSFLG_RUNNING_SYS
+		if ((ERTS_PSFLG_RUNNING_SYS|ERTS_PSFLG_DIRTY_RUNNING_SYS)
 		    & erts_smp_atomic32_read_nob(&rp->state)) {
 		    /* Executing system task... */
 		    resume_process(rp, ERTS_PROC_LOCK_STATUS);
@@ -8474,7 +8585,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	     * from being selected for normal execution regardless
 	     * of locks held or not held on it...
 	     */
-	    ASSERT(!(ERTS_PSFLG_RUNNING
+	    ASSERT(!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS)
 		     & erts_smp_atomic32_read_nob(&rp->state)));
 
 	    if (!suspend)
@@ -9015,28 +9126,43 @@ erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched)
 }
 
 Eterm
-erts_process_status(Process *c_p, ErtsProcLocks c_p_locks,
-		    Process *rp, Eterm rpid)
+erts_process_state2status(erts_aint32_t state)
+{
+    if (state & ERTS_PSFLG_FREE)
+	return am_free;
+
+    if (state & ERTS_PSFLG_EXITING)
+	return am_exiting;
+
+    if (state & ERTS_PSFLG_GC)
+	return am_garbage_collecting;
+
+    if (state & ERTS_PSFLG_SUSPENDED)
+	return am_suspended;
+
+    if (state & (ERTS_PSFLG_RUNNING
+		 | ERTS_PSFLG_RUNNING_SYS
+		 | ERTS_PSFLG_DIRTY_RUNNING
+		 | ERTS_PSFLG_DIRTY_RUNNING_SYS))
+	return am_running;
+
+    if (state & (ERTS_PSFLG_ACTIVE
+		 | ERTS_PSFLG_ACTIVE_SYS
+		 | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
+	return am_runnable;
+
+    return am_waiting;
+}
+
+Eterm
+erts_process_status(Process *rp, Eterm rpid)
 {
     Eterm res = am_undefined;
     Process *p = rp ? rp : erts_proc_lookup_raw(rpid);
 
     if (p) {
 	erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
-	if (state & ERTS_PSFLG_FREE)
-	    res = am_free;
-	else if (state & ERTS_PSFLG_EXITING)
-	    res = am_exiting;
-	else if (state & ERTS_PSFLG_GC)
-	    res = am_garbage_collecting;
-	else if (state & ERTS_PSFLG_SUSPENDED)
-	    res = am_suspended;
-	else if (state & ERTS_PSFLG_RUNNING)
-	    res = am_running;
-	else if (state & ERTS_PSFLG_ACTIVE)
-	    res = am_runnable;
-	else
-	    res = am_waiting;
+	res = erts_process_state2status(state);
     }
 #ifdef ERTS_SMP
     else {
@@ -9251,7 +9377,76 @@ scheduler_gc_proc(Process *c_p, int reds_left)
     return reds;
 }
 
+static ERTS_INLINE void
+clean_dirty_start(Process *p)
+{
+#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64)
+    void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL);
+    if (ptr)
+	erts_free(ERTS_ALC_T_DIRTY_START, ptr);
+#endif
+}
+
+static ERTS_INLINE void
+save_dirty_start(ErtsSchedulerData *esdp, Process *c_p)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
+	ErtsMonotonicTime time = erts_get_monotonic_time(esdp);
+#ifdef ARCH_64
+	ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time);
+#else
+	ErtsMonotonicTime *stimep;
 
+	stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
+	if (!stimep) {
+	    stimep = erts_alloc(ERTS_ALC_T_DIRTY_START,
+				sizeof(ErtsMonotonicTime));
+	    ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep);
+	}
+	*stimep = time;
+#endif
+    }
+#endif
+}
+
+static ERTS_INLINE int
+get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p)
+{
+
+#ifndef ERTS_DIRTY_SCHEDULERS
+    return -1;
+#else
+    ErtsMonotonicTime stime, time;
+
+    if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue))
+	return 1;
+
+#ifdef ARCH_64
+    stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
+#else
+    {
+	ErtsMonotonicTime *stimep;
+	stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
+	ASSERT(stimep);
+	stime = *stimep;
+    }
+#endif
+
+    time = erts_get_monotonic_time(esdp);
+
+    ASSERT(stime && stime < time);
+
+    time -= stime;
+    time = ERTS_MONOTONIC_TO_USEC(time);
+    time *= 2;
+
+    if (time > INT_MAX)
+	return INT_MAX;
+    return (int) time;
+#endif
+
+}
 
 /*
  * schedule() is called from BEAM (process_main()) or HiPE
@@ -9283,6 +9478,7 @@ Process *schedule(Process *p, int calls)
     int reds;
     Uint32 flags;
     erts_aint32_t state = 0; /* Supress warning... */
+    int is_normal_sched;
 
     ERTS_MSACC_DECLARE_CACHE();
 
@@ -9312,25 +9508,44 @@ Process *schedule(Process *p, int calls)
      */
     if (!p) {	/* NULL in the very first schedule() call */
 	esdp = erts_get_scheduler_data();
+	is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp);
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
 	fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
 	actual_reds = reds = 0;
 	erts_smp_runq_lock(rq);
     } else {
-    sched_out_proc:
-
 #ifdef ERTS_SMP
-	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+#ifdef ERTS_DIRTY_SCHEDULERS
 	esdp = p->scheduler_data;
+	is_normal_sched = esdp != NULL;
+	if (is_normal_sched)
+	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+	else {
+	    esdp = erts_get_scheduler_data();
+	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
+	}
+#else
+	esdp = p->scheduler_data;
+	is_normal_sched = 1;
+#endif
 	ASSERT(esdp->current_process == p
 	       || esdp->free_process == p);
 #else
 	esdp = erts_scheduler_data;
 	ASSERT(esdp->current_process == p);
+	is_normal_sched = 1;
 #endif
 
-	reds = actual_reds = calls - esdp->virtual_reds;
+    sched_out_proc:
+
+	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+
+	if (is_normal_sched)
+	    reds = actual_reds = calls - esdp->virtual_reds;
+	else
+	    reds = actual_reds = get_dirty_reds(esdp, p);
+
 	ASSERT(actual_reds >= 0);
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
@@ -9377,7 +9592,7 @@ Process *schedule(Process *p, int calls)
         state = erts_smp_atomic32_read_nob(&p->state);
 
 #ifdef ERTS_SMP
-	if (state & ERTS_PSFLG_PENDING_EXIT)
+	if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT))
 	    erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN
 					 | ERTS_PROC_LOCK_STATUS));
 	if (p->pending_suspenders) 
@@ -9387,7 +9602,8 @@ Process *schedule(Process *p, int calls)
 
 	esdp->reductions += reds;
 
-	schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */
+	/* schedule_out_process() returns with rq locked! */
+	schedule_out_process(rq, state, p, proxy_p, is_normal_sched);
 	proxy_p = NULL;
 
 	ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
@@ -9405,13 +9621,20 @@ Process *schedule(Process *p, int calls)
         ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
 
 	if (state & ERTS_PSFLG_FREE) {
+	    if (!is_normal_sched) {
+		ASSERT(p->flags & F_DELAYED_DEL_PROC);
+		erts_proc_dec_refc(p);
+	    }
+	    else {
 #ifdef ERTS_SMP
-	    ASSERT(esdp->free_process == p);
-	    esdp->free_process = NULL;
+		ASSERT(esdp->free_process == p);
+		esdp->free_process = NULL;
 #else
-	    erts_proc_dec_refc(p);
+		erts_proc_dec_refc(p);
 #endif
+	    }
 	}
+
 #ifdef ERTS_SMP
 	ASSERT(!esdp->free_process);
 #endif
@@ -9419,7 +9642,7 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	if (is_normal_sched) {
 	    if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
 		(void) erts_get_monotonic_time(esdp);
 
@@ -9433,23 +9656,15 @@ Process *schedule(Process *p, int calls)
 
     }
 
-    ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)
-		       || !erts_thr_progress_is_blocking());
+    ERTS_SMP_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking());
 
  check_activities_to_run: {
+	erts_aint32_t psflg_running, psflg_running_sys;
 #ifdef ERTS_SMP
 	ErtsMigrationPaths *mps;
 	ErtsMigrationPath *mp;
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	    ErtsProcList *pnd_xtrs = rq->procs.pending_exiters;
-	    if (erts_proclist_fetch(&pnd_xtrs, NULL)) {
-		rq->procs.pending_exiters = NULL;
-		erts_smp_runq_unlock(rq);
-		handle_pending_exiters(pnd_xtrs);
-		erts_smp_runq_lock(rq);
-	    }
-
+	if (is_normal_sched) {
 	    if (rq->check_balance_reds <= 0)
 		check_balance(rq);
 
@@ -9466,32 +9681,35 @@ Process *schedule(Process *p, int calls)
     continue_check_activities_to_run:
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
     continue_check_activities_to_run_known_flags:
-	ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)
-	       || flags & ERTS_RUNQ_FLG_NONEMPTY);
+	ASSERT(!is_normal_sched || (flags & ERTS_RUNQ_FLG_NONEMPTY));
 
-	if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
-	    if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
-		(void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
+	if (!is_normal_sched) {
+	    if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+		& ERTS_SSI_FLG_SUSPENDED) {
 		suspend_scheduler(esdp);
-		flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
-		flags |= ERTS_RUNQ_FLG_EXEC;
-	    }
-	    if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) {
-		flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND);
-		flags &= ~ ERTS_RUNQ_FLG_CHK_CPU_BIND;
-		erts_sched_check_cpu_bind(esdp);
 	    }
 	}
-#ifdef ERTS_DIRTY_SCHEDULERS
-	else if (ERTS_SCHEDULER_IS_DIRTY(esdp)
-		 && (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
-		     & ERTS_SSI_FLG_SUSPENDED))
-	    suspend_scheduler(esdp);
-#endif
-
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	else {
 	    erts_aint32_t aux_work;
-	    int leader_update = erts_thr_progress_update(esdp);
+	    int leader_update;
+
+	    ASSERT(is_normal_sched);
+
+	    if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
+		if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
+		    (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
+		    suspend_scheduler(esdp);
+		    flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
+		    flags |= ERTS_RUNQ_FLG_EXEC;
+		}
+		if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) {
+		    flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND);
+		    flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND;
+		    erts_sched_check_cpu_bind(esdp);
+		}
+	    }
+
+	    leader_update = erts_thr_progress_update(esdp);
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
 	    if (aux_work | leader_update) {
 		erts_smp_runq_unlock(rq);
@@ -9517,19 +9735,13 @@ Process *schedule(Process *p, int calls)
 
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->halt_in_progress) {
-	    /*
-	     * TODO: if halt in progress, need to put the dirty scheduler
-	     * to sleep somewhere around here to prevent it from picking up
-	     * new work
-	     */
+	if (!is_normal_sched && rq->halt_in_progress) {
+	    /* Wait for emulator to terminate... */
+	    while (1)
+		erts_milli_sleep(1000*1000);
 	}
-	else
-#endif
-
-	if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start)
-	    || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) {
+	else if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start)
+		 || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) {
 	    /* Prepare for scheduler wait */
 #ifdef ERTS_SMP
 	    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -9543,7 +9755,7 @@ Process *schedule(Process *p, int calls)
 	    if (flags & ERTS_RUNQ_FLG_INACTIVE)
 		empty_runq(rq);
 	    else {
-		if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && try_steal_task(rq))
+		if (is_normal_sched && try_steal_task(rq))
 		    goto continue_check_activities_to_run;
 
 		empty_runq(rq);
@@ -9572,9 +9784,9 @@ Process *schedule(Process *p, int calls)
 
 	    goto check_activities_to_run;
 	}
-	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) &&
-		 (fcalls > input_reductions &&
-		  prepare_for_sys_schedule(!0))) {
+	else if (is_normal_sched
+		 && (fcalls > input_reductions
+		     && prepare_for_sys_schedule(!0))) {
 	    ErtsMonotonicTime current_time;
 	    /*
 	     * Schedule system-level activities.
@@ -9688,11 +9900,17 @@ Process *schedule(Process *p, int calls)
 
 	    ASSERT(p); /* Wrong qmask in rq->flags? */
 
-	    if (ERTS_SCHEDULER_IS_DIRTY(esdp))
-		psflg_band_mask = ~((erts_aint32_t) 0);
-	    else
+	    if (is_normal_sched) {
+		psflg_running = ERTS_PSFLG_RUNNING;
+		psflg_running_sys = ERTS_PSFLG_RUNNING_SYS;
 		psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state)
 							    + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET));
+	    }
+	    else {
+		psflg_running = ERTS_PSFLG_DIRTY_RUNNING;
+		psflg_running_sys = ERTS_PSFLG_DIRTY_RUNNING_SYS;
+		psflg_band_mask = ~((erts_aint32_t) 0);
+	    }
 
 	    if (!(state & ERTS_PSFLG_PROXY))
 		psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ;
@@ -9707,34 +9925,53 @@ Process *schedule(Process *p, int calls)
 		state = erts_smp_atomic32_read_nob(&p->state);
 	    }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    if (!is_normal_sched)
+		clear_proc_dirty_queue_bit(p, rq, qbit);
+#endif
+
 	    while (1) {
-		erts_aint32_t exp, new, tmp;
-		tmp = new = exp = state;
+		erts_aint32_t exp, new;
+		int run_process;
+		new = exp = state;
 		new &= psflg_band_mask;
-		if (!(state & (ERTS_PSFLG_RUNNING
-			       | ERTS_PSFLG_RUNNING_SYS))) {
-		    tmp = state & (ERTS_PSFLG_SUSPENDED
-				   | ERTS_PSFLG_PENDING_EXIT
-				   | ERTS_PSFLG_ACTIVE_SYS
-				   | ERTS_PSFLG_DIRTY_ACTIVE_SYS);
-		    if (tmp != ERTS_PSFLG_SUSPENDED) {
-			if (state & (ERTS_PSFLG_ACTIVE_SYS
-				     | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
-			    new |= ERTS_PSFLG_RUNNING_SYS;
-			else
-			    new |= ERTS_PSFLG_RUNNING;
-		    }
+		/*
+		 * Run process if not already running (or free)
+		 * or exiting and not running on a normal
+		 * scheduler, and not suspended (and not in a
+		 * state where suspend should be ignored).
+		 */
+		run_process = (((!(state & (ERTS_PSFLG_RUNNING
+					    | ERTS_PSFLG_RUNNING_SYS
+					    | ERTS_PSFLG_DIRTY_RUNNING
+					    | ERTS_PSFLG_DIRTY_RUNNING_SYS
+					    | ERTS_PSFLG_FREE)))
+#ifdef ERTS_DIRTY_SCHEDULERS
+				| (((state & (ERTS_PSFLG_RUNNING
+					      | ERTS_PSFLG_FREE
+					      | ERTS_PSFLG_RUNNING_SYS
+					      | ERTS_PSFLG_EXITING))
+				    == ERTS_PSFLG_EXITING)
+				   & (!!is_normal_sched))
+#endif
+				   )
+			       & ((state & (ERTS_PSFLG_SUSPENDED
+					    | ERTS_PSFLG_EXITING
+					    | ERTS_PSFLG_FREE
+					    | ERTS_PSFLG_PENDING_EXIT
+					    | ERTS_PSFLG_ACTIVE_SYS
+					    | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
+				  != ERTS_PSFLG_SUSPENDED));
+		if (run_process) {
+		    if (state & (ERTS_PSFLG_ACTIVE_SYS
+				 | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
+			new |= psflg_running_sys;
+		    else
+			new |= psflg_running;
 		}
 		state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp);
 		if (state == exp) {
-		    if ((state & (ERTS_PSFLG_RUNNING
-				  | ERTS_PSFLG_RUNNING_SYS
-				  | ERTS_PSFLG_FREE))
-			|| ((state & (ERTS_PSFLG_SUSPENDED
-				      | ERTS_PSFLG_PENDING_EXIT
-				      | ERTS_PSFLG_ACTIVE_SYS
-				      | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
-			    == ERTS_PSFLG_SUSPENDED)) {
+		    if (!run_process) {
 			if (proxy_p) {
 			    free_proxy_proc(proxy_p);
 			    proxy_p = NULL;
@@ -9761,34 +9998,13 @@ Process *schedule(Process *p, int calls)
 
 	    erts_smp_runq_unlock(rq);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-#ifdef DEBUG
-		int old_dqbit;
-#endif
-		int dqbit = qbit;
-
-		if (rq == ERTS_DIRTY_CPU_RUNQ)
-		    dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET;
-		else {
-		    ASSERT(rq == ERTS_DIRTY_IO_RUNQ);
-		    dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET;
-		}
-
-#ifdef DEBUG
-		old_dqbit = (int)
-#else
-		    (void)
-#endif
-		    erts_smp_atomic32_read_band_mb(&p->dirty_state, ~dqbit);
-		ASSERT(old_dqbit & dqbit);
-	    }
-#endif /* ERTS_DIRTY_SCHEDULERS */
-
 #endif /* ERTS_SMP */
 
 	}
 
+	if (!is_normal_sched)
+	    save_dirty_start(esdp, p);
+
 #ifdef ERTS_SMP
 
 	if (flags & ERTS_RUNQ_FLG_PROTECTED)
@@ -9805,9 +10021,7 @@ Process *schedule(Process *p, int calls)
 	    UWord old = ERTS_PROC_SCHED_ID(p, (UWord) esdp->no);
 	    int migrated = old && old != esdp->no;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
+	    ASSERT(is_normal_sched);
 
 	    prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state);
 
@@ -9821,20 +10035,21 @@ Process *schedule(Process *p, int calls)
 	    erts_smp_spin_unlock(&erts_sched_stat.lock);
 	}
 
-	ASSERT(!p->scheduler_data);
-	p->scheduler_data = esdp;
-
 	state = erts_smp_atomic32_read_nob(&p->state);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	    if (!!(state & ERTS_PSFLGS_DIRTY_WORK)
-		& !(state & ERTS_PSFLG_ACTIVE_SYS)) {
+	ASSERT(!p->scheduler_data);
+#ifndef ERTS_DIRTY_SCHEDULERS
+	p->scheduler_data = esdp;
+#else /* ERTS_DIRTY_SCHEDULERS */
+	if (is_normal_sched) {
+	    if ((!!(state & ERTS_PSFLGS_DIRTY_WORK))
+		& (!(state & ERTS_PSFLG_ACTIVE_SYS))) {
 		/* Migrate to dirty scheduler... */
 	    sunlock_sched_out_proc:
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 		goto sched_out_proc;
 	    }
+	    p->scheduler_data = esdp;
 	}
 	else {
 	    if (state & (ERTS_PSFLG_ACTIVE_SYS
@@ -9891,7 +10106,8 @@ Process *schedule(Process *p, int calls)
         (void)ERTS_TRACER_PROC_IS_ENABLED(p);
 #endif
 
-	if (state & ERTS_PSFLG_RUNNING_SYS) {
+	if (state & (ERTS_PSFLG_RUNNING_SYS
+		     | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    /*
 	     * GC is normally never delayed when a process
 	     * is scheduled out, but might be when executing
@@ -9905,7 +10121,7 @@ Process *schedule(Process *p, int calls)
 		reds -= cost;
 		if (reds <= 0
 #ifdef ERTS_DIRTY_SCHEDULERS
-		    || ERTS_SCHEDULER_IS_DIRTY(esdp)
+		    || !is_normal_sched
 		    || (state & ERTS_PSFLGS_DIRTY_WORK)
 #endif
 		    ) {
@@ -9913,8 +10129,8 @@ Process *schedule(Process *p, int calls)
 		}
 	    }
 
-	    ASSERT(state & ERTS_PSFLG_RUNNING_SYS);
-	    ASSERT(!(state & ERTS_PSFLG_RUNNING));
+	    ASSERT(state & psflg_running_sys);
+	    ASSERT(!(state & psflg_running));
 
 	    while (1) {
 		erts_aint32_t n, e;
@@ -9926,8 +10142,8 @@ Process *schedule(Process *p, int calls)
 		}
 
 		n = e = state;
-		n &= ~ERTS_PSFLG_RUNNING_SYS;
-		n |= ERTS_PSFLG_RUNNING;
+		n &= ~psflg_running_sys;
+		n |= psflg_running;
 
 		state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 		if (state == e) {
@@ -9935,8 +10151,8 @@ Process *schedule(Process *p, int calls)
 		    break;
 		}
 
-		ASSERT(state & ERTS_PSFLG_RUNNING_SYS);
-		ASSERT(!(state & ERTS_PSFLG_RUNNING));
+		ASSERT(state & psflg_running_sys);
+		ASSERT(!(state & psflg_running));
 	    }
 	}
 
@@ -10525,7 +10741,10 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
     }
 
     state = erts_smp_atomic32_read_nob(&c_p->state);
-    ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) & state);
+    ASSERT((ERTS_PSFLG_RUNNING
+	    | ERTS_PSFLG_RUNNING_SYS
+	    | ERTS_PSFLG_DIRTY_RUNNING
+	    | ERTS_PSFLG_DIRTY_RUNNING_SYS) & state);
 
     while (!(state & ERTS_PSFLG_DELAYED_SYS)
 	   || prio < ERTS_PSFLGS_GET_ACT_PRIO(state)) {
@@ -10850,6 +11069,8 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 }
 
+static void delete_process(Process* p);
+
 void
 erts_free_proc(Process *p)
 {
@@ -10858,6 +11079,8 @@ erts_free_proc(Process *p)
 #endif
     ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE);
     ASSERT(0 == erts_proc_read_refc(p));
+    if (p->flags & F_DELAYED_DEL_PROC)
+	delete_process(p);
     erts_free(ERTS_ALC_T_PROC, (void *) p);
 }
 
@@ -11486,18 +11709,36 @@ erts_cleanup_empty_process(Process* p)
 #endif
 }
 
-/*
- * p must be the currently executing process.
- */
 static void
 delete_process(Process* p)
 {
     Eterm *heap;
     ErtsPSD *psd;
+    struct saved_calls *scb;
+    process_breakpoint_time_t *pbt;
+    void *nif_export;
+
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
                            HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
 
+    scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
+
+    if (scb) {
+	p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */
+        erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
+    }
+
+    pbt = ERTS_PROC_SET_CALL_TIME(p, NULL);
+    if (pbt)
+        erts_free(ERTS_ALC_T_BPD, (void *) pbt);
+
+    nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
+    if (nif_export)
+	erts_destroy_nif_export(nif_export);
+
+    clean_dirty_start(p);
+
     /* Cleanup psd */
 
     psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
@@ -11629,7 +11870,10 @@ set_proc_self_exiting(Process *c_p)
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL);
 
     state = erts_smp_atomic32_read_nob(&c_p->state);
-    ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
+    ASSERT(state & (ERTS_PSFLG_RUNNING
+		    |ERTS_PSFLG_RUNNING_SYS
+		    | ERTS_PSFLG_DIRTY_RUNNING
+		    | ERTS_PSFLG_DIRTY_RUNNING_SYS));
 
 #ifdef DEBUG
     enqueue =
@@ -11679,51 +11923,73 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
 	erts_smp_proc_unlock(c_p, xlocks);
 }
 
+static void save_pending_exiter(Process *p, ErtsProcList *plp);
+
 static void
-handle_pending_exiters(ErtsProcList *pnd_xtrs)
+do_handle_pending_exiters(ErtsProcList *pnd_xtrs)
 {
     /* 'list' is expected to have been fetched (i.e. not a ring anymore) */
     ErtsProcList *plp = pnd_xtrs;
 
     while (plp) {
-	ErtsProcList *free_plp;
-	Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL);
+	ErtsProcList *next_plp = plp->next;
+	Process *p = erts_proc_lookup(plp->pid);
 	if (p) {
-	    if (erts_proclist_same(plp, p)) {
-		erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
-		if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
-		    ASSERT(state & ERTS_PSFLG_PENDING_EXIT);
-		    erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL);
+	    erts_aint32_t state;
+	    /*
+	     * If the process is running on a normal scheduler, the
+	     * pending exit will soon be detected and handled by the
+	     * scheduler running the process (at schedule in/out).
+	     */
+	    if (erts_smp_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) {
+		if (erts_proclist_same(plp, p)) {
+		    state = erts_smp_atomic32_read_acqb(&p->state);
+		    if (!(state & (ERTS_PSFLG_RUNNING
+				   | ERTS_PSFLG_RUNNING_SYS
+				   | ERTS_PSFLG_EXITING))) {
+			ASSERT(state & ERTS_PSFLG_PENDING_EXIT);
+			erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL);
+		    }
 		}
+		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+	    }
+	    else {
+		erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+		if (erts_proclist_same(plp, p)) {
+		    state = erts_smp_atomic32_read_acqb(&p->state);
+		    if (!(state & (ERTS_PSFLG_RUNNING
+				   | ERTS_PSFLG_RUNNING_SYS
+				   | ERTS_PSFLG_EXITING))) {
+			/*
+			 * Save process and try to acquire all
+			 * locks at a later time...
+			 */
+			save_pending_exiter(p, plp);
+			plp = NULL;
+		    }
+		}
+		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	    }
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 	}
-	free_plp = plp;
-	plp = plp->next;
-	proclist_destroy(free_plp);
+	if (plp)
+	    proclist_destroy(plp);
+	plp = next_plp;
     }
 }
 
 static void
-save_pending_exiter(Process *p)
+save_pending_exiter(Process *p, ErtsProcList *plp)
 {
-    ErtsProcList *plp;
+    ErtsSchedulerSleepInfo *ssi;
     ErtsRunQueue *rq;
-    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    if (!esdp)
-        rq = RUNQ_READ_RQ(&p->run_queue);
-    else
-        rq = esdp->run_queue;
+    rq = RUNQ_READ_RQ(&p->run_queue);
+    ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-	rq = ERTS_RUNQ_IX(0); /* Handle on ordinary scheduler */
-#endif
-
-    plp = proclist_create(p);
+    if (!plp)
+	plp = proclist_create(p);
 
     erts_smp_runq_lock(rq);
 
@@ -11731,9 +11997,11 @@ save_pending_exiter(Process *p)
 
     non_empty_runq(rq);
 
+    ssi = rq->scheduler->ssi;
+
     erts_smp_runq_unlock(rq);
 
-    wake_scheduler(rq);
+    set_aux_work_flags_wakeup_nob(ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS);
 }
 
 #endif
@@ -11933,7 +12201,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		if (need_locks
 		    && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
 		    /* ... but we havn't got all locks on it ... */
-		    save_pending_exiter(rp); 
+		    save_pending_exiter(rp, NULL); 
 		    /*
 		     * The pending exit will be discovered when next
 		     * process is scheduled in
@@ -12401,10 +12669,8 @@ erts_continue_exit_process(Process *p)
     ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN;
     Eterm reason = p->fvalue;
     DistEntry *dep;
-    struct saved_calls *scb;
-    process_breakpoint_time_t *pbt;
     erts_aint32_t state;
-    void *nif_export;
+    int delay_del_proc = 0;
 
 #ifdef DEBUG
     int yield_allowed = 1;
@@ -12547,7 +12813,7 @@ erts_continue_exit_process(Process *p)
     {
 	/* Do *not* use erts_get_runq_proc() */
 	ErtsRunQueue *rq;
-	rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p));
+	rq = erts_get_runq_current(erts_proc_sched_data(p));
 
 	erts_smp_runq_lock(rq);
 
@@ -12593,16 +12859,24 @@ erts_continue_exit_process(Process *p)
 		break;
 	}
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (a & (ERTS_PSFLG_DIRTY_RUNNING
+		 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+	    p->flags |= F_DELAYED_DEL_PROC;
+	    delay_del_proc = 1;
+	    /*
+	     * The dirty scheduler will also decrease
+	     * refc when done...
+	     */
+	    erts_proc_inc_refc(p);
+	}
+#endif
+
 	if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ))
 	    erts_proc_dec_refc(p);
     }
     
     dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
-    scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
-    if (scb)
-	p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */
-    pbt = ERTS_PROC_SET_CALL_TIME(p, NULL);
-    nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 #ifdef BM_COUNTERS
@@ -12642,22 +12916,14 @@ erts_continue_exit_process(Process *p)
 								 have none here */
     }
 
-    if (scb)
-        erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
-
-    if (pbt)
-        erts_free(ERTS_ALC_T_BPD, (void *) pbt);
-
-    if (nif_export)
-	erts_destroy_nif_export(nif_export);
-
 #ifdef ERTS_SMP
     erts_flush_trace_messages(p, 0);
 #endif
 
     ERTS_TRACER_CLEAR(&ERTS_TRACER(p));
 
-    delete_process(p);
+    if (!delay_del_proc)
+	delete_process(p);
 
 #ifdef ERTS_SMP
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
@@ -12687,6 +12953,7 @@ erts_continue_exit_process(Process *p)
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
 
+    BUMP_ALL_REDS(p);
 }
 
 /*
@@ -13006,11 +13273,13 @@ void erts_halt(int code)
 int
 erts_dbg_check_halloc_lock(Process *p)
 {
+    ErtsSchedulerData *esdp;
     if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p))
 	return 1;
     if (p->common.id == ERTS_INVALID_PID)
 	return 1;
-    if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process)
+    esdp = erts_proc_sched_data(p);
+    if (esdp && p == esdp->match_pseudo_process)
 	return 1;
     if (erts_thr_progress_is_blocking())
 	return 1;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 59da9c1779..6de6b4f499 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -304,6 +304,7 @@ typedef enum {
     ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
     ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
     ERTS_SSI_AUX_WORK_MISC_IX,
+    ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX,
     ERTS_SSI_AUX_WORK_SET_TMO_IX,
     ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
     ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
@@ -336,6 +337,8 @@ typedef enum {
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
 #define ERTS_SSI_AUX_WORK_MISC \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
+#define ERTS_SSI_AUX_WORK_PENDING_EXITERS \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX)
 #define ERTS_SSI_AUX_WORK_SET_TMO \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
 #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
@@ -645,6 +648,7 @@ struct ErtsSchedulerData_ {
     Uint no;			/* Scheduler number for normal schedulers */
 #ifdef ERTS_DIRTY_SCHEDULERS
     ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
+    Process *dirty_shadow_process;
 #endif
     Port *current_port;
     ErtsRunQueue *run_queue;
@@ -805,14 +809,26 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_CALL_TIME_BP			3
 #define ERTS_PSD_DELAYED_GC_TASK_QS		4
 #define ERTS_PSD_NIF_TRAP_EXPORT		5
-#ifdef HIPE
 #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	6
-#endif
-
-#ifdef HIPE
-#define ERTS_PSD_SIZE				7
-#else
-#define ERTS_PSD_SIZE				6
+#define ERTS_PSD_DIRTY_CPU_START		7
+
+#define ERTS_PSD_SIZE				8
+
+#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS)
+#  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
+#  undef ERTS_PSD_DIRTY_CPU_START
+#  undef ERTS_PSD_SIZE
+#  define ERTS_PSD_SIZE 6
+#elif !defined(HIPE)
+#  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
+#  undef ERTS_PSD_DIRTY_CPU_START
+#  undef ERTS_PSD_SIZE
+#  define ERTS_PSD_DIRTY_CPU_START		6
+#  define ERTS_PSD_SIZE 7
+#elif !defined(ERTS_DIRTY_SCHEDULERS)
+#  undef ERTS_PSD_DIRTY_CPU_START
+#  undef ERTS_PSD_SIZE
+#  define ERTS_PSD_SIZE 7
 #endif
 
 typedef struct {
@@ -1183,7 +1199,10 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(20)
 #define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(21)
 #define ERTS_PSFLG_DIRTY_ACTIVE_SYS	ERTS_PSFLG_BIT(22)
-#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
+#define ERTS_PSFLG_DIRTY_RUNNING	ERTS_PSFLG_BIT(23)
+#define ERTS_PSFLG_DIRTY_RUNNING_SYS	ERTS_PSFLG_BIT(24)
+
+#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 24)
 
 #define ERTS_PSFLGS_DIRTY_WORK		(ERTS_PSFLG_DIRTY_CPU_PROC	\
 					 | ERTS_PSFLG_DIRTY_IO_PROC	\
@@ -1194,6 +1213,11 @@ void erts_check_for_holes(Process* p);
 					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
 					 | ERTS_PSFLG_IN_PRQ_LOW)
 
+#define ERTS_PSFLGS_VOLATILE_HEAP	(ERTS_PSFLG_EXITING		\
+					 | ERTS_PSFLG_PENDING_EXIT	\
+					 | ERTS_PSFLG_DIRTY_RUNNING	\
+					 | ERTS_PSFLG_DIRTY_RUNNING_SYS)
+
 #define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
     (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
 #define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
@@ -1235,6 +1259,7 @@ void erts_check_for_holes(Process* p);
  * Static flags that do not change after process creation.
  */
 #define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)
+#define ERTS_STC_FLG_SHADOW_PROC	(((Uint32) 1) << 1)
 
 /* The sequential tracing token is a tuple of size 5:
  *
@@ -1363,6 +1388,7 @@ extern int erts_system_profile_ts_type;
 #define F_SCHDLR_ONLN_WAITQ  (1 << 17) /* Process enqueued waiting to change schedulers online */
 #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
 #define F_HIPE_MODE          (1 << 19)
+#define F_DELAYED_DEL_PROC   (1 << 20) /* Delay delete process (dirty proc exit case) */
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1783,7 +1809,8 @@ erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
 void erts_sched_notify_check_cpu_bind(void);
 Uint erts_active_schedulers(void);
 void erts_init_process(int, int, int);
-Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
+Eterm erts_process_state2status(erts_aint32_t);
+Eterm erts_process_status(Process *, Eterm);
 Uint erts_run_queues_len(Uint *, int, int);
 void erts_add_to_runq(Process *);
 Eterm erts_bound_schedulers_term(Process *c_p);
@@ -1860,19 +1887,11 @@ int erts_debug_wait_completed(Process *c_p, int flags);
 
 Uint erts_process_memory(Process *c_p, int incl_msg_inq);
 
-#ifdef ERTS_SMP
-#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
-#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
-#else
-#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
-#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
-#endif
-
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
 #  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
 do {										\
     ErtsSchedulerData *esdp__ = ((P)						\
-				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
+				 ? erts_proc_sched_data((Process *) (P))	\
 				 : erts_get_scheduler_data());			\
     if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
 	esdp__->verify_unused_temp_alloc(					\
@@ -1965,12 +1984,15 @@ erts_psd_set(Process *p, int ix, void *data)
     ErtsPSD *psd;
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
-    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
-	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
-    else {
-	locks &= erts_psd_required_locks[ix].set_locks;
-	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
-			   || erts_thr_progress_is_blocking());
+    erts_aint32_t state = state = erts_smp_atomic32_read_nob(&p->state);
+    if (!(state & ERTS_PSFLG_FREE)) {
+	if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
+	    ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
+	else {
+	    locks &= erts_psd_required_locks[ix].set_locks;
+	    ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
+			       || erts_thr_progress_is_blocking());
+	}
     }
 #endif
     psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
@@ -2027,6 +2049,13 @@ erts_psd_set(Process *p, int ix, void *data)
   ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
 #endif
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#define ERTS_PROC_GET_DIRTY_CPU_START(P) \
+  ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START))
+#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \
+  ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS)))
+#endif
+
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
 
@@ -2169,6 +2198,7 @@ erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
 
 #endif
 
+ERTS_GLB_INLINE ErtsSchedulerData *erts_proc_sched_data(Process *c_p);
 ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
 ERTS_GLB_INLINE Process *erts_get_current_process(void);
 ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
@@ -2201,6 +2231,31 @@ ERTS_GLB_INLINE void erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp,
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
+ERTS_GLB_INLINE
+ErtsSchedulerData *erts_proc_sched_data(Process *c_p)
+{
+    ErtsSchedulerData *esdp;
+    ASSERT(c_p);
+#if !defined(ERTS_SMP)
+    esdp = erts_get_scheduler_data();
+#else
+    esdp = c_p->scheduler_data;
+#  if defined(ERTS_DIRTY_SCHEDULERS)
+    if (esdp) {
+	ASSERT(esdp == erts_get_scheduler_data());
+	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+    }
+    else {
+	esdp = erts_get_scheduler_data();
+	ASSERT(esdp);
+	ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
+    }
+#  endif
+#endif
+    ASSERT(esdp);
+    return esdp;
+}
+
 ERTS_GLB_INLINE
 int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
 {
@@ -2416,7 +2471,7 @@ ERTS_GLB_INLINE ErtsAtomCacheMap *
 erts_get_atom_cache_map(Process *c_p)
 {
     ErtsSchedulerData *esdp = (c_p
-			       ? ERTS_PROC_GET_SCHDATA(c_p)
+			       ? erts_proc_sched_data(c_p)
 			       : erts_get_scheduler_data());
     ASSERT(esdp);
     return &esdp->atom_cache_map;
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index fa76773cac..eeaa9a569c 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -568,23 +568,21 @@ dump_externally(int to, void *to_arg, Eterm term)
     }
 }
 
-void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) {
-    if (psflg & ERTS_PSFLG_FREE)
-	erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */
-    else if (psflg & ERTS_PSFLG_EXITING)
-	erts_print(to, to_arg, "Exiting\n");
-    else if (psflg & ERTS_PSFLG_GC) {
-	erts_print(to, to_arg, "Garbing\n");
-    }
-    else if (psflg & ERTS_PSFLG_SUSPENDED)
-	erts_print(to, to_arg, "Suspended\n");
-    else if (psflg & ERTS_PSFLG_RUNNING) {
-	erts_print(to, to_arg, "Running\n");
+void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg)
+{
+    char *s;
+    switch (erts_process_state2status(psflg)) {
+    case am_free: s = "Non Existing"; break; /* Should never happen */
+    case am_exiting: s = "Exiting"; break;
+    case am_garbage_collecting: s = "Garbing"; break;
+    case am_suspended: s = "Suspended"; break;
+    case am_running: s = "Running"; break;
+    case am_runnable: s = "Scheduled"; break;
+    case am_waiting: s = "Waiting"; break;
+    default: s = "Undefined"; break; /* Should never happen */
     }
-    else if (psflg & ERTS_PSFLG_ACTIVE)
-	erts_print(to, to_arg, "Scheduled\n");
-    else
-	erts_print(to, to_arg, "Waiting\n");
+
+    erts_print(to, to_arg, "%s\n", s);
 }
 
 void
@@ -668,6 +666,10 @@ erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) {
                 erts_print(to, to_arg, "DIRTY_IO_PROC"); break;
             case ERTS_PSFLG_DIRTY_ACTIVE_SYS:
                 erts_print(to, to_arg, "DIRTY_ACTIVE_SYS"); break;
+            case ERTS_PSFLG_DIRTY_RUNNING:
+                erts_print(to, to_arg, "DIRTY_RUNNING"); break;
+            case ERTS_PSFLG_DIRTY_RUNNING_SYS:
+                erts_print(to, to_arg, "DIRTY_RUNNING_SYS"); break;
             default:
                 erts_print(to, to_arg, "UNKNOWN(%d)", chk); break;
             }
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index fadbd704bd..9e37106b88 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -2348,7 +2348,7 @@ erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to)
 BIF_RETTYPE monotonic_time_0(BIF_ALIST_0)
 {
     ErtsMonotonicTime mtime = time_sup.r.o.get_time();
-    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
+    update_last_mtime(erts_proc_sched_data(BIF_P), mtime);
     mtime += ERTS_MONOTONIC_OFFSET_NATIVE;
     BIF_RET(make_time_val(BIF_P, mtime));
 }
@@ -2356,7 +2356,7 @@ BIF_RETTYPE monotonic_time_0(BIF_ALIST_0)
 BIF_RETTYPE monotonic_time_1(BIF_ALIST_1)
 {
     ErtsMonotonicTime mtime = time_sup.r.o.get_time();
-    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
+    update_last_mtime(erts_proc_sched_data(BIF_P), mtime);
     BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime, 1));
 }
 
@@ -2365,7 +2365,7 @@ BIF_RETTYPE system_time_0(BIF_ALIST_0)
     ErtsMonotonicTime mtime, offset;
     mtime = time_sup.r.o.get_time();
     offset = get_time_offset();
-    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
+    update_last_mtime(erts_proc_sched_data(BIF_P), mtime);
     BIF_RET(make_time_val(BIF_P, mtime + offset));
 }
 
@@ -2374,7 +2374,7 @@ BIF_RETTYPE system_time_1(BIF_ALIST_0)
     ErtsMonotonicTime mtime, offset;
     mtime = time_sup.r.o.get_time();
     offset = get_time_offset();
-    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
+    update_last_mtime(erts_proc_sched_data(BIF_P), mtime);
     BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime + offset, 0));
 }
 
@@ -2404,7 +2404,7 @@ BIF_RETTYPE timestamp_0(BIF_ALIST_0)
 
     mtime = time_sup.r.o.get_time();
     offset = get_time_offset();
-    update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime);
+    update_last_mtime(erts_proc_sched_data(BIF_P), mtime);
 
     make_timestamp_value(&mega_sec, &sec, &micro_sec, mtime, offset);
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 550db95ba8..1abcc6cbf4 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -57,6 +57,7 @@ struct enif_environment_t /* ErlNifEnv */
     struct enif_tmp_obj_t* tmp_obj_list;
     int exception_thrown; /* boolean */
     Process *tracee;
+    int exiting; /* boolean (dirty nifs might return in exiting state) */
 };
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c35d80652f..ef326fafec 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2512,7 +2512,7 @@ erts_port_output(Process *c_p,
 	    sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND;
 	else if (async_nosuspend) {
 	    ErtsSchedulerData *esdp = (c_p
-				       ? ERTS_PROC_GET_SCHDATA(c_p)
+				       ? erts_proc_sched_data(c_p)
 				       : erts_get_scheduler_data());
 	    ASSERT(esdp);
 	    ns_pthp = &esdp->nosuspend_port_task_handle;
@@ -5140,7 +5140,7 @@ erts_request_io_bytes(Process *c_p)
     Uint *hp;
     Eterm ref;
     Uint32 *refn;
-    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ,
 				     sizeof(ErtsIOBytesReq));
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index dda3ba565c..f303d4f167 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -34,6 +34,10 @@
   (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
 #endif
 
+#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ERTS_SMP)
+# error "Dirty schedulers not supported without smp support"
+#endif
+
 #ifdef ERTS_INLINE
 #  ifndef ERTS_CAN_INLINE
 #    define ERTS_CAN_INLINE 1
-- 
cgit v1.2.3


From 3471d44a6a5ed5ab038c4cdc76b350119fe745e2 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 11 May 2016 11:16:16 +0200
Subject: erts: Only allow remove from trace_status callback

Make it so that it is only possible to remove a tracer via
returning remove from an erl_tracer. This limition is put in
place in order to avoid a lot of lock checking and taking
in various places, especially in regards to trace events
happening on dirty schedulers.
---
 erts/emulator/beam/beam_bp.c       |   6 +--
 erts/emulator/beam/erl_bif_trace.c |  19 +++----
 erts/emulator/beam/erl_db_util.c   |   3 +-
 erts/emulator/beam/erl_process.c   |  11 ++--
 erts/emulator/beam/erl_trace.c     | 100 +++++++++++++++++++++++--------------
 erts/emulator/beam/erl_trace.h     |  11 ++--
 erts/emulator/beam/io.c            |   2 +-
 7 files changed, 85 insertions(+), 67 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 308e5ce205..8489897d3a 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -655,8 +655,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 	erts_smp_atomic_inc_nob(&bp->count->acount);
     }
 
-    if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE
-        && ERTS_TRACER_PROC_IS_ENABLED(c_p)) {
+    if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) {
 	Eterm w;
 	erts_trace_time_call(c_p, I, bp->time);
 	w = (BeamInstr) *c_p->cp;
@@ -753,8 +752,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	}
     }
     if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE &&
-	IS_TRACED_FL(p, F_TRACE_CALLS) &&
-	ERTS_TRACER_PROC_IS_ENABLED(p)) {
+	IS_TRACED_FL(p, F_TRACE_CALLS)) {
 	BeamInstr *pc = (BeamInstr *)ep->code+3;
 	erts_trace_time_call(p, pc, bp->time);
     }
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index b65c0e303f..66e5146da0 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -512,8 +512,7 @@ start_trace(Process *c_p, ErtsTracer tracer,
             && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) {
             /* This tracee is already being traced, and not by the
              * tracer to be */
-            if (erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCKS_ALL,
-                                            common, am_trace_status)) {
+            if (erts_is_tracer_enabled(tracer, common)) {
                 /* The tracer is still in use */
                 return 1;
             }
@@ -856,7 +855,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
             return am_undefined;
 
         if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee)))
-            erts_is_tracer_proc_enabled(NULL, 0, &tracee->common, am_trace_status);
+            erts_is_tracer_proc_enabled(NULL, 0, &tracee->common);
 
         tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee));
         trace_flags = ERTS_TRACE_FLAGS(tracee);
@@ -864,22 +863,24 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
         erts_port_release(tracee);
 
     } else if (is_internal_pid(pid_spec)) {
-	Process *tracee;
-	tracee = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
-			       pid_spec, ERTS_PROC_LOCK_MAIN);
+	Process *tracee = erts_pid2proc_not_running(p, ERTS_PROC_LOCK_MAIN,
+                                                    pid_spec, ERTS_PROC_LOCK_MAIN);
+
+        if (tracee == ERTS_PROC_LOCK_BUSY)
+            ERTS_BIF_YIELD2(bif_export[BIF_trace_info_2], p, pid_spec, key);
 
 	if (!tracee)
 	    return am_undefined;
 
         if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee)))
             erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN,
-                                        &tracee->common, am_trace_status);
+                                        &tracee->common);
 
         tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee));
         trace_flags = ERTS_TRACE_FLAGS(tracee);
 
-        if (tracee != p)
-            erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN);
+	if (tracee != p)
+	    erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN);
     } else if (is_external_pid(pid_spec)
 	       && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) {
 	    return am_undefined;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 3e75b9fd5f..6732b708a8 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -174,7 +174,8 @@ set_match_trace(Process *tracee_p, Eterm fail_term, ErtsTracer tracer,
         ERTS_PROC_LOCKS_ALL == erts_proc_lc_my_proc_locks(tracee_p)
         || erts_thr_progress_is_blocking());
 
-    if (ERTS_TRACER_IS_NIL(tracer) || erts_is_tracer_enabled(tracee_p, tracer))
+    if (ERTS_TRACER_IS_NIL(tracer)
+        || erts_is_tracer_enabled(tracer, &tracee_p->common))
         return set_tracee_flags(tracee_p, tracer, d_flags, e_flags);
     return fail_term;
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b50ca6d009..9d895f1867 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10085,7 +10085,10 @@ Process *schedule(Process *p, int calls)
 
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
-	if (IS_TRACED(p)) {
+        /* Clear tracer if it has been removed */
+	if (IS_TRACED(p) && erts_is_tracer_proc_enabled(
+                p, ERTS_PROC_LOCK_MAIN, &p->common)) {
+
 	    if (state & ERTS_PSFLG_EXITING) {
 		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
 		    trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in_exiting);
@@ -10100,12 +10103,6 @@ Process *schedule(Process *p, int calls)
 	    }
 	}
 
-
-#ifdef ERTS_SMP
-        /* Clears tracer if it has been removed */
-        (void)ERTS_TRACER_PROC_IS_ENABLED(p);
-#endif
-
 	if (state & (ERTS_PSFLG_RUNNING_SYS
 		     | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    /*
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 1c0fc0a11f..447b46d239 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -396,7 +396,7 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
                    Eterm tag, Eterm msg, Eterm extra,
                    Eterm pam_result);
 static ERTS_INLINE Eterm
-call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
+call_enabled_tracer(const ErtsTracer tracer,
                     ErtsTracerNif **tnif_ref,
                     enum ErtsTracerOpt topt,
                     Eterm tag, Eterm t_p_id);
@@ -459,8 +459,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new
 
     if (!ERTS_TRACER_IS_NIL(new)) {
         Eterm nif_result = call_enabled_tracer(
-            NULL, new, NULL,
-            TRACE_FUN_ENABLED, am_trace_status, am_undefined);
+            new, NULL, TRACE_FUN_ENABLED, am_trace_status, am_undefined);
         switch (nif_result) {
         case am_trace: break;
         default:
@@ -492,7 +491,7 @@ erts_get_system_seq_tracer(void)
     erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
 
     if (st != erts_tracer_nil &&
-        call_enabled_tracer(NULL, st, NULL, TRACE_FUN_ENABLED,
+        call_enabled_tracer(st, NULL, TRACE_FUN_ENABLED,
                             am_trace_status, am_undefined) == am_remove) {
         erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil);
         st = erts_tracer_nil;
@@ -513,7 +512,7 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp,
 	*default_trace_flags &= ~TRACEE_FLAGS;
     } else {
         Eterm nif_res;
-        nif_res = call_enabled_tracer(NULL, *default_tracer,
+        nif_res = call_enabled_tracer(*default_tracer,
                                       NULL, TRACE_FUN_ENABLED,
                                       am_trace_status, am_undefined);
         switch (nif_res) {
@@ -915,8 +914,8 @@ seq_trace_update_send(Process *p)
     ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p))));
     if (have_no_seqtrace(SEQ_TRACE_TOKEN(p)) ||
         (seq_tracer != NIL &&
-         call_enabled_tracer(NULL, seq_tracer, NULL,
-                             TRACE_FUN_ENABLED, am_trace_status,
+         call_enabled_tracer(seq_tracer, NULL,
+                             TRACE_FUN_ENABLED, am_seq_trace,
                              p ? p->common.id : am_undefined) != am_trace)
 #ifdef USE_VM_PROBES
 	 || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag)
@@ -963,9 +962,9 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
     ASSERT(is_tuple(token) || is_nil(token));
     if (token == NIL || (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE) ||
         ERTS_TRACER_IS_NIL(seq_tracer) ||
-        call_enabled_tracer(NULL, seq_tracer,
+        call_enabled_tracer(seq_tracer,
                             NULL, TRACE_FUN_ENABLED,
-                            am_trace_status,
+                            am_seq_trace,
                             process ? process->common.id : am_undefined) != am_trace) {
 	return;
     }
@@ -1186,7 +1185,11 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 	 *     use process flags
 	 */
 	tracee_flags = &ERTS_TRACE_FLAGS(p);
+        /* Is is not ideal at all to call this check twice,
+           it should be optimized so that only one call is made. */
         if (!is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif,
+                               TRACE_FUN_ENABLED, am_trace_status)
+            || !is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif,
                     TRACE_FUN_E_CALL, am_call)) {
             return 0;
         }
@@ -1202,13 +1205,21 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
         }
 	meta_flags = F_TRACE_CALLS | F_NOW_TS;
 	tracee_flags = &meta_flags;
-        switch (call_enabled_tracer(p, *tracer,
-                                    &tnif, TRACE_FUN_T_CALL,
-                                    am_call, p->common.id)) {
+        switch (call_enabled_tracer(*tracer,
+                                    &tnif, TRACE_FUN_ENABLED,
+                                    am_trace_status, p->common.id)) {
         default:
         case am_remove: *tracer = erts_tracer_nil;
         case am_discard: return 0;
-        case am_trace: break;
+        case am_trace:
+            switch (call_enabled_tracer(*tracer,
+                                        &tnif, TRACE_FUN_T_CALL,
+                                        am_call, p->common.id)) {
+            default:
+            case am_discard: return 0;
+            case am_trace: break;
+            }
+            break;
         }
     }
 
@@ -1346,9 +1357,9 @@ trace_proc(Process *c_p, ErtsProcLocks c_p_locks,
            Process *t_p, Eterm what, Eterm data)
 {
     ErtsTracerNif *tnif = NULL;
-    if (is_tracer_enabled(c_p, c_p_locks, &t_p->common, &tnif,
+    if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif,
                 TRACE_FUN_E_PROCS, what))
-        send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS,
+        send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS,
                            what, data, THE_NON_VALUE, am_true);
 }
 
@@ -1365,16 +1376,15 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
 		 Eterm mod, Eterm func, Eterm args)
 {
     ErtsTracerNif *tnif = NULL;
-    if (is_tracer_enabled(p, ERTS_PROC_LOCKS_ALL &
-                               ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE),
-                               &p->common, &tnif, TRACE_FUN_E_PROCS, what)) {
+    if (is_tracer_enabled(NULL, 0,
+			  &p->common, &tnif, TRACE_FUN_E_PROCS, what)) {
         Eterm mfa;
         Eterm* hp;
 
         hp = HAlloc(p, 4);
         mfa = TUPLE3(hp, mod, func, args);
         hp += 4;
-        send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS,
+        send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS,
                            what, pid, mfa, am_true);
     }
 }
@@ -2896,23 +2906,28 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
 }
 
 static ERTS_INLINE Eterm
-call_enabled_tracer(Process *c_p, const ErtsTracer tracer,
+call_enabled_tracer(const ErtsTracer tracer,
                     ErtsTracerNif **tnif_ret,
                     enum ErtsTracerOpt topt,
                     Eterm tag, Eterm t_p_id) {
     ErtsTracerNif *tnif = lookup_tracer_nif(tracer);
     if (tnif) {
-        Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id};
+        Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id},
+            ret;
         topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_ENABLED;
         ASSERT(topt < NIF_TRACER_TYPES);
         ASSERT(tnif->tracers[topt].cb != NULL);
         if (tnif_ret) *tnif_ret = tnif;
-        return erts_nif_call_function(c_p, NULL, tnif->nif_mod,
-                                      tnif->tracers[topt].cb,
-                                      tnif->tracers[topt].arity,
-                                      argv);
+        ret = erts_nif_call_function(NULL, NULL, tnif->nif_mod,
+                                     tnif->tracers[topt].cb,
+                                     tnif->tracers[topt].arity,
+                                     argv);
+        if (tag == am_trace_status && ret != am_remove)
+            return am_trace;
+        ASSERT(tag == am_trace_status || ret != am_remove);
+        return ret;
     }
-    return am_remove;
+    return tag == am_trace_status ? am_remove : am_discard;
 }
 
 static int
@@ -2937,12 +2952,12 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
     }
 #endif
 
-    nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, topt, tag, t_p->id);
+    nif_result = call_enabled_tracer(t_p->tracer, tnif_ret, topt, tag, t_p->id);
     switch (nif_result) {
     case am_discard: return 0;
     case am_trace: return 1;
     case THE_NON_VALUE:
-    case am_remove: break;
+    case am_remove: ASSERT(tag == am_trace_status); break;
     default:
         /* only am_remove should be returned, but if
            something else is returned we fall-through
@@ -2973,19 +2988,14 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
     return 0;
 }
 
-int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
-                                ErtsPTabElementCommon *t_p, Eterm type)
-{
-    return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_ENABLED, am_trace_status);
-}
-
-int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer)
+int erts_is_tracer_enabled(const ErtsTracer tracer, ErtsPTabElementCommon *t_p)
 {
     ErtsTracerNif *tnif = lookup_tracer_nif(tracer);
     if (tnif) {
-        Eterm nif_result = call_enabled_tracer(c_p, tracer, &tnif,
-                                               TRACE_FUN_ENABLED, am_trace_status,
-                                               c_p->common.id);
+        Eterm nif_result = call_enabled_tracer(tracer, &tnif,
+                                               TRACE_FUN_ENABLED,
+                                               am_trace_status,
+                                               t_p->id);
         switch (nif_result) {
         case am_discard:
         case am_trace: return 1;
@@ -2996,6 +3006,20 @@ int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer)
     return 0;
 }
 
+int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks,
+                                ErtsPTabElementCommon *t_p)
+{
+    return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_ENABLED,
+                             am_trace_status);
+}
+
+int erts_is_tracer_proc_enabled_send(Process* c_p, ErtsProcLocks c_p_locks,
+                                     ErtsPTabElementCommon *t_p)
+{
+    return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_T_SEND, am_send);
+}
+
+
 void erts_tracer_replace(ErtsPTabElementCommon *t_p, const ErtsTracer tracer)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index de4beadb7e..0a0c5fc913 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -193,8 +193,10 @@ int erts_finish_breakpointing(void);
 
 /* Nif tracer functions */
 int erts_is_tracer_proc_enabled(Process *c_p, ErtsProcLocks c_p_locks,
-                                ErtsPTabElementCommon *t_p, Eterm type);
-int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer);
+                                ErtsPTabElementCommon *t_p);
+int erts_is_tracer_proc_enabled_send(Process* c_p, ErtsProcLocks c_p_locks,
+                                     ErtsPTabElementCommon *t_p);
+int erts_is_tracer_enabled(const ErtsTracer tracer, ErtsPTabElementCommon *t_p);
 Eterm erts_tracer_to_term(Process *p, ErtsTracer tracer);
 ErtsTracer erts_term_to_tracer(Eterm prefix, Eterm term);
 void erts_tracer_replace(ErtsPTabElementCommon *t_p,
@@ -224,9 +226,4 @@ ERTS_DECLARE_DUMMY(erts_tracer_nil) = NIL;
 #define ERTS_TRACER_FROM_ETERM(termp) \
     ((ErtsTracer*)(termp))
 
-#define ERTS_TRACER_PROC_IS_ENABLED(PROC)                               \
-    (!ERTS_TRACER_IS_NIL(ERTS_TRACER(PROC))                             \
-     && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN,          \
-                                    &(PROC)->common, am_trace_status))
-
 #endif /* ERL_TRACE_H__ */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index ef326fafec..0377f6cb5e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5941,7 +5941,7 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len)
     if (!rp) {
         if (!prt || !IS_TRACED_FL(prt, F_TRACE_SEND))
             goto done;
-        if (!erts_is_tracer_proc_enabled(NULL, 0, &prt->common, am_send))
+        if (!erts_is_tracer_proc_enabled_send(NULL, 0, &prt->common))
             goto done;
 
 	res = -2;
-- 
cgit v1.2.3


From cbee76c388d08100856a92f14850bb3302a51207 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 May 2016 19:20:41 +0200
Subject: erts: Remove compiler warning

'hx' may be used uninitialized
---
 erts/emulator/beam/beam_load.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2e21a553ed..e6b50fbd32 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4368,6 +4368,7 @@ gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
     int good_hash;
 #endif
 
+    ERTS_UNDEF(hx, 0);
     ASSERT(Size.type == TAG_u);
 
     NEW_GENOP(stp, op);
-- 
cgit v1.2.3


From 65c94fe4976cc4589e42ace1873640a0e7c73bf8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 12 May 2016 13:57:23 +0200
Subject: erts: Fix bug when tracing in non-smp non-scheduler thread

---
 erts/emulator/beam/erl_nif.c | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 159dc66ad5..bd9367a3d5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -667,6 +667,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
             erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 
             if (!msgq) {
+#ifdef ERTS_SMP
+                ErtsThrPrgrDelayHandle dhndl;
+#endif
 
                 msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE,
                                   sizeof(ErlTraceMessageQueue));
@@ -681,8 +684,15 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 
                 erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
 
+#ifdef ERTS_SMP
+                if (!scheduler)
+                    dhndl = erts_thr_progress_unmanaged_delay();
+#endif
                 erts_schedule_flush_trace_messages(t_p->common.id);
-
+#ifdef ERTS_SMP
+                if (!scheduler)
+                    erts_thr_progress_unmanaged_continue(dhndl);
+#endif
             } else {
                 msgq->len++;
                 *msgq->last = mp;
@@ -1669,7 +1679,8 @@ int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc)
 	return !!rp;
 #else
 	erts_exit(ERTS_ABORT_EXIT, "enif_is_process_alive: "
-		  "called from non-scheduler thread");
+		  "called from non-scheduler thread "
+                  "in non-smp emulator");
 	return 0;
 #endif
     }
@@ -1694,7 +1705,8 @@ int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port)
 	return !!prt;
 #else
 	erts_exit(ERTS_ABORT_EXIT, "enif_is_port_alive: "
-		  "called from non-scheduler thread");
+		  "called from non-scheduler thread "
+                  "in non-smp emulator");
 	return 0;
 #endif
     }
-- 
cgit v1.2.3


From 32a8787eb7651d41371afdf3f55fd3a77486861d Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 12 May 2016 17:37:40 +0200
Subject: erts: Fix OpenBSD gcc compiler bug in re code

The OpenBSD 5.8 gcc compiler emits erroneous code
which results in the re:run function behaving badly,
though strangely enough it does not segfault the vm.

This fix moves code around a bit in order to make gcc
emit correct code.
---
 erts/emulator/beam/erl_bif_re.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index f4daecd2a4..ff7746ce1d 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -630,9 +630,15 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete
 	}
     } else {
 	ReturnInfo *ri;
-	ReturnInfo defri = {RetIndex,0,{0}};
+	ReturnInfo defri;
 
 	if (restartp->ret_info == NULL) {
+            /* OpenBSD 5.8 gcc compiler for some reason creates
+               bad code if the above initialization is done
+               inline with the struct. So don't do that. */
+            defri.type = RetIndex;
+            defri.num_spec = 0;
+            defri.v[0] = 0;
 	    ri = &defri;
 	} else {
 	    ri = restartp->ret_info;
-- 
cgit v1.2.3


From 0f04b36c15acd592a650936b039997feeb37a639 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 12 May 2016 18:11:22 +0200
Subject: erts: Move max_heap_size field so that ErLLVM works again

---
 erts/emulator/beam/erl_process.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 2801947613..2fc6dfe194 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -960,7 +960,6 @@ struct process {
     Uint heap_sz;		/* Size of heap in words */
     Uint min_heap_size;         /* Minimum size of heap (in words). */
     Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */
-    Uint max_heap_size;         /* Maximum size of heap (in words). */
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     volatile unsigned long fp_exception;
@@ -1061,6 +1060,7 @@ struct process {
     Eterm *old_hend;            /* Heap pointers for generational GC. */
     Eterm *old_htop;
     Eterm *old_heap;
+    Uint max_heap_size;         /* Maximum size of heap (in words). */
     Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
     Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
     ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
-- 
cgit v1.2.3


From 9effc6a6c52d9c492925c4271743d06d68d04bcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 18 May 2016 19:38:47 +0200
Subject: erts: Copy literals in messages on module purge

During check process code, explicitly copy all referenced literals
in a message (in the private queue) to a heap fragment and attach
it to the message reference.

Not all types of message communication does an explicit copy of a
literal and this needs to be taken care of before a module is purged.
---
 erts/emulator/beam/beam_bif_load.c | 136 ++++++++++++++++++++++++++++++++-----
 1 file changed, 118 insertions(+), 18 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 40d44dda4c..8b5d1a0391 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -762,6 +762,11 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size)
     return 0;
 }
 
+static Uint hfrag_literal_size(Eterm* start, Eterm* end,
+                               char* lit_start, Uint lit_size);
+static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp,
+                               Eterm *start, Eterm *end,
+                               char *lit_start, Uint lit_size);
 
 static Eterm
 check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls)
@@ -842,9 +847,14 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
     }
 
     /*
-     * Message queue can contains funs, but (at least currently) no
+     * Message queue can contains funs, and may contain
      * literals. If we got references to this module from the message
-     * queue, a GC cannot remove these...
+     * queue.
+     *
+     * If a literal is in the message queue we maka an explicit copy of
+     * and attach it to the heap fragment. Each message needs to be
+     * self contained, we cannot save the literal in the old_heap or
+     * any other heap than the message it self.
      */
 
     erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
@@ -861,15 +871,31 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 	    hfrag = msgp->data.heap_frag;
 	else
 	    continue;
-	for (; hfrag; hfrag = hfrag->next) {
-	    if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
-		return am_true;
-	    /* Should not contain any literals... */
-	    ASSERT(!any_heap_refs(&hfrag->mem[0],
-				  &hfrag->mem[hfrag->used_size],
-                                  literals,
-				  lit_bsize));
-	}
+        {
+            ErlHeapFragment *hf;
+            Uint lit_sz;
+            for (hf=hfrag; hf; hf = hf->next) {
+                if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
+                    return am_true;
+                lit_sz = hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size],
+                                            literals, lit_bsize);
+            }
+            if (lit_sz > 0) {
+                ErlHeapFragment *bp = new_message_buffer(lit_sz);
+                Eterm *hp = bp->mem;
+
+                for (hf=hfrag; hf; hf = hf->next) {
+                    hfrag_literal_copy(&hp, &bp->off_heap,
+                                       &hf->mem[0], &hf->mem[hf->used_size],
+                                       literals, lit_bsize);
+                    hfrag=hf;
+                }
+                /* link new hfrag last */
+                ASSERT(hfrag->next == NULL);
+                hfrag->next = bp;
+                bp->next = NULL;
+            }
+        }
     }
 
     while (1) {
@@ -916,11 +942,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 		goto try_literal_gc;
 	}
 
-#ifdef DEBUG
 	/*
-	 * Message buffer fragments should not have any references
-	 * to literals, and off heap lists should already have
-	 * been moved into process off heap structure.
+	 * Message buffer fragments (matched messages) 
+         *  - off heap lists should already have been moved into
+         *    process off heap structure.
+         *  - Check for literals
 	 */
 	for (msgp = rp->msg_frag; msgp; msgp = msgp->next) {
 	    if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
@@ -933,12 +959,12 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 
 		hp = &hfrag->mem[0];
 		hp_end = &hfrag->mem[hfrag->used_size];
-		ASSERT(!any_heap_refs(hp, hp_end, literals, lit_bsize));
+
+                if (any_heap_refs(hp, hp_end, literals, lit_bsize))
+                    goto try_literal_gc;
 	    }
 	}
 
-#endif
-
 	return am_false;
 
     try_literal_gc:
@@ -1038,6 +1064,80 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
     return 0;
 }
 
+static Uint
+hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size)
+{
+    Eterm* p;
+    Eterm val;
+    Uint sz = 0;
+
+    for (p = start; p < end; p++) {
+        val = *p;
+        switch (primary_tag(val)) {
+        case TAG_PRIMARY_BOXED:
+        case TAG_PRIMARY_LIST:
+            if (ErtsInArea(val, lit_start, lit_size)) {
+                sz += size_object(val);
+            }
+            break;
+        case TAG_PRIMARY_HEADER:
+            if (!header_is_transparent(val)) {
+                Eterm* new_p;
+                if (header_is_bin_matchstate(val)) {
+                    ErlBinMatchState *ms = (ErlBinMatchState*) p;
+                    ErlBinMatchBuffer *mb = &(ms->mb);
+                    if (ErtsInArea(mb->orig, lit_start, lit_size)) {
+                        sz += size_object(mb->orig);
+                    }
+                }
+                new_p = p + thing_arityval(val);
+                ASSERT(start <= new_p && new_p < end);
+                p = new_p;
+            }
+        }
+    }
+    return sz;
+}
+
+static void
+hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp,
+                   Eterm *start, Eterm *end,
+                   char *lit_start, Uint lit_size) {
+    Eterm* p;
+    Eterm val;
+    Uint sz;
+
+    for (p = start; p < end; p++) {
+        val = *p;
+        switch (primary_tag(val)) {
+        case TAG_PRIMARY_BOXED:
+        case TAG_PRIMARY_LIST:
+            if (ErtsInArea(val, lit_start, lit_size)) {
+                sz = size_object(val);
+                val = copy_struct(val, sz, hpp, ohp);
+                *p = val; 
+            }
+            break;
+        case TAG_PRIMARY_HEADER:
+            if (!header_is_transparent(val)) {
+                Eterm* new_p;
+                /* matchstate in message, not possible. */
+                if (header_is_bin_matchstate(val)) {
+                    ErlBinMatchState *ms = (ErlBinMatchState*) p;
+                    ErlBinMatchBuffer *mb = &(ms->mb);
+                    if (ErtsInArea(mb->orig, lit_start, lit_size)) {
+                        sz = size_object(mb->orig);
+                        mb->orig = copy_struct(mb->orig, sz, hpp, ohp);
+                    }
+                }
+                new_p = p + thing_arityval(val);
+                ASSERT(start <= new_p && new_p < end);
+                p = new_p;
+            }
+        }
+    }
+}
+
 #undef in_area
 
 #ifdef ERTS_SMP
-- 
cgit v1.2.3


From 703eea866831dea0753c5cd151d6d8d55debd470 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 18 May 2016 19:45:42 +0200
Subject: erts: Refactor ERTS_MSG_COMBINED_HFRAG to heap fragment

---
 erts/emulator/beam/beam_bif_load.c |  5 +----
 erts/emulator/beam/erl_gc.c        | 11 ++---------
 erts/emulator/beam/erl_message.h   |  9 +++++----
 3 files changed, 8 insertions(+), 17 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 8b5d1a0391..15e878ba65 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -949,10 +949,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
          *  - Check for literals
 	 */
 	for (msgp = rp->msg_frag; msgp; msgp = msgp->next) {
-	    if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
-		hfrag = &msgp->hfrag;
-	    else
-		hfrag = msgp->data.heap_frag;
+            hfrag = erts_message_to_heap_frag(msgp);
 	    for (; hfrag; hfrag = hfrag->next) {
 		Eterm *hp, *hp_end;
 		ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size));
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d740b2baec..374da9407c 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2279,10 +2279,7 @@ move_msgq_to_heap(Process *p)
 	    }
 	    else {
 
-		if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG)
-		    bp = &mp->hfrag;
-		else
-		    bp = mp->data.heap_frag;
+                bp = erts_message_to_heap_frag(mp);
 
 		if (bp->next)
 		    erts_move_multi_frags(&factory.hp, factory.off_heap, bp,
@@ -3304,11 +3301,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
 
     while (mp) {
 
-	if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG)
-	    bp = &mp->hfrag;
-	else
-	    bp = mp->data.heap_frag;
-
+        bp = erts_message_to_heap_frag(mp);
 	mp = mp->next;
 
     search_heap_frags:
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 851ac37fda..ecf65ca324 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -369,6 +369,10 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg);
 
 #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1)
 
+#define erts_message_to_heap_frag(MP)                   \
+    (((MP)->data.attached == ERTS_MSG_COMBINED_HFRAG) ? \
+        &(MP)->hfrag : (MP)->data.heap_frag)
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp)
@@ -449,10 +453,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg)
     ASSERT(msg->data.attached);
     if (is_value(ERL_MESSAGE_TERM(msg))) {
 	ErlHeapFragment *bp;
-	if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG)
-	    bp = &msg->hfrag;
-	else
-	    bp = msg->data.heap_frag;
+        bp = erts_message_to_heap_frag(msg);
 	return erts_used_frag_sz(bp);
     }
     else if (msg->data.dist_ext->heap_size < 0)
-- 
cgit v1.2.3


From 115f0ba77ad7d01ab95fd9f9bbeca53f04f12284 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Thu, 19 May 2016 10:54:43 +0200
Subject: erts: Move tracer SecondTraceTerm to Opts map

The extra trace data has been moved to the opts map in order
for the tracer to be able to distinguish inbetween extra
trace data 'undefined' and no extra trace data. In the same
commit all opts associations have been changed so that if
the tracer should not use them, the key is left unassicated
instead of being sent to undefined. This should be give a
small performance gain and also makes the API easier to work
with.
---
 erts/emulator/beam/atom.names  |  7 +++--
 erts/emulator/beam/erl_trace.c | 65 ++++++++++++++++++++++++------------------
 2 files changed, 42 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 8f65e71531..7fdce62bd1 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -228,9 +228,6 @@ atom endian
 atom env
 atom eof
 atom eol
-atom exception_from
-atom exception_trace
-atom extended
 atom Eq='=:='
 atom Eqeq='=='
 atom erl_tracer
@@ -243,6 +240,8 @@ atom ets
 atom ETS_TRANSFER='ETS-TRANSFER'
 atom event
 atom exact_reductions
+atom exception_from
+atom exception_trace
 atom exclusive
 atom exit_status
 atom existing
@@ -251,7 +250,9 @@ atom existing_ports
 atom existing
 atom exiting
 atom exports
+atom extended
 atom external
+atom extra
 atom false
 atom fcgi
 atom fd
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index ca001fc156..3dca58d60b 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2625,7 +2625,7 @@ static void init_tracer_template(ErtsTracerNif *tnif) {
 
     /* default tracer functions */
     tnif->tracers[TRACE_FUN_DEFAULT].name  = "trace";
-    tnif->tracers[TRACE_FUN_DEFAULT].arity = 6;
+    tnif->tracers[TRACE_FUN_DEFAULT].arity = 5;
     tnif->tracers[TRACE_FUN_DEFAULT].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_ENABLED].name  = "enabled";
@@ -2634,35 +2634,35 @@ static void init_tracer_template(ErtsTracerNif *tnif) {
 
     /* specific tracer functions */
     tnif->tracers[TRACE_FUN_T_SEND].name  = "trace_send";
-    tnif->tracers[TRACE_FUN_T_SEND].arity = 6;
+    tnif->tracers[TRACE_FUN_T_SEND].arity = 5;
     tnif->tracers[TRACE_FUN_T_SEND].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_RECEIVE].name  = "trace_receive";
-    tnif->tracers[TRACE_FUN_T_RECEIVE].arity = 6;
+    tnif->tracers[TRACE_FUN_T_RECEIVE].arity = 5;
     tnif->tracers[TRACE_FUN_T_RECEIVE].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_CALL].name  = "trace_call";
-    tnif->tracers[TRACE_FUN_T_CALL].arity = 6;
+    tnif->tracers[TRACE_FUN_T_CALL].arity = 5;
     tnif->tracers[TRACE_FUN_T_CALL].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_SCHED_PROC].name  = "trace_running_procs";
-    tnif->tracers[TRACE_FUN_T_SCHED_PROC].arity = 6;
+    tnif->tracers[TRACE_FUN_T_SCHED_PROC].arity = 5;
     tnif->tracers[TRACE_FUN_T_SCHED_PROC].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_SCHED_PORT].name  = "trace_running_ports";
-    tnif->tracers[TRACE_FUN_T_SCHED_PORT].arity = 6;
+    tnif->tracers[TRACE_FUN_T_SCHED_PORT].arity = 5;
     tnif->tracers[TRACE_FUN_T_SCHED_PORT].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_GC].name  = "trace_garbage_collection";
-    tnif->tracers[TRACE_FUN_T_GC].arity = 6;
+    tnif->tracers[TRACE_FUN_T_GC].arity = 5;
     tnif->tracers[TRACE_FUN_T_GC].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_PROCS].name  = "trace_procs";
-    tnif->tracers[TRACE_FUN_T_PROCS].arity = 6;
+    tnif->tracers[TRACE_FUN_T_PROCS].arity = 5;
     tnif->tracers[TRACE_FUN_T_PROCS].cb    = NULL;
 
     tnif->tracers[TRACE_FUN_T_PORTS].name  = "trace_ports";
-    tnif->tracers[TRACE_FUN_T_PORTS].arity = 6;
+    tnif->tracers[TRACE_FUN_T_PORTS].arity = 5;
     tnif->tracers[TRACE_FUN_T_PORTS].cb    = NULL;
 
     /* specific enabled functions */
@@ -2834,10 +2834,12 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee,
                        Eterm tag, Eterm msg, Eterm extra, Eterm pam_result)
 {
     if (tnif || (tnif = lookup_tracer_nif(tracer)) != NULL) {
-#define MAP_SIZE 3
-        Eterm argv[6], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)];
+#define MAP_SIZE 4
+        Eterm argv[5], local_heap[3+MAP_SIZE /* values */ + (MAP_SIZE+1 /* keys */)];
         flatmap_t *map = (flatmap_t*)(local_heap+(MAP_SIZE+1));
         Eterm *map_values = flatmap_get_values(map);
+        Eterm *map_keys = local_heap + 1;
+        Uint map_elem_count = 0;
 
         topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_DEFAULT;
         ASSERT(topt < NIF_TRACER_TYPES);
@@ -2846,31 +2848,40 @@ send_to_tracer_nif_raw(Process *c_p, Process *tracee,
         argv[1] = ERTS_TRACER_STATE(tracer);
         argv[2] = t_p_id;
         argv[3] = msg;
-        argv[4] = extra == THE_NON_VALUE ? am_undefined : extra;
-        argv[5] = make_flatmap(map);
+        argv[4] = make_flatmap(map);
 
         map->thing_word = MAP_HEADER_FLATMAP;
-        map->size = MAP_SIZE;
-        map->keys = TUPLE3(local_heap, am_match_spec_result, am_scheduler_id, am_timestamp);
-
-        *map_values++ = pam_result;
-        if (tracee_flags & F_TRACE_SCHED_NO)
-            *map_values++ = make_small(erts_get_scheduler_id());
-        else
-            *map_values++ = am_undefined;
+
+        if (extra != THE_NON_VALUE) {
+            map_keys[map_elem_count] = am_extra;
+            map_values[map_elem_count++] = extra;
+        }
+
+        if (pam_result != am_true) {
+            map_keys[map_elem_count] = am_match_spec_result;
+            map_values[map_elem_count++] = pam_result;
+        }
+
+        if (tracee_flags & F_TRACE_SCHED_NO) {
+            map_keys[map_elem_count] = am_scheduler_id;
+            map_values[map_elem_count++] = make_small(erts_get_scheduler_id());
+        }
+        map_keys[map_elem_count] = am_timestamp;
         if (tracee_flags & F_NOW_TS)
 #ifdef HAVE_ERTS_NOW_CPU
             if (erts_cpu_timestamp)
-                *map_values++ = am_cpu_timestamp;
+                map_values[map_elem_count++] = am_cpu_timestamp;
             else
 #endif
-                *map_values++ = am_timestamp;
+                map_values[map_elem_count++] = am_timestamp;
         else if (tracee_flags & F_STRICT_MON_TS)
-            *map_values++ = am_strict_monotonic;
+            map_values[map_elem_count++] = am_strict_monotonic;
         else if (tracee_flags & F_MON_TS)
-            *map_values++ = am_monotonic;
-        else
-            *map_values++ = am_undefined;
+            map_values[map_elem_count++] = am_monotonic;
+
+        map->size = map_elem_count;
+        map->keys = make_tuple(local_heap);
+        local_heap[0] = make_arityval(map_elem_count);
 
 #undef MAP_SIZE
         erts_nif_call_function(c_p, tracee ? tracee : c_p,
-- 
cgit v1.2.3


From f9cb80861f169743a96099a06d68149a91f18dfa Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 20 May 2016 15:58:04 +0200
Subject: erts: Implement halt/0 and halt/1 in Erlang

just to make things simpler.
---
 erts/emulator/beam/bif.c   | 50 +---------------------------------------------
 erts/emulator/beam/bif.tab |  2 --
 2 files changed, 1 insertion(+), 51 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2a3bd4afe5..81477422ff 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3836,59 +3836,11 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0)
 
 /**********************************************************************/
 
-/* stop the system */
-/* ARGSUSED */
-BIF_RETTYPE halt_0(BIF_ALIST_0)
-{
-    VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n"));
-    erts_halt(0);
-    ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
-}
-
-/**********************************************************************/
 
 #define HALT_MSG_SIZE	200
 static char halt_msg[HALT_MSG_SIZE];
 
-/* stop the system with exit code */
-/* ARGSUSED */
-BIF_RETTYPE halt_1(BIF_ALIST_1)
-{
-    Uint code;
-    
-    if (term_to_Uint_mask(BIF_ARG_1, &code)) {
-	int pos_int_code = (int) (code & INT_MAX);
-	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
-	erts_halt(pos_int_code);
-	ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
-    }
-    else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
-	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_exit(ERTS_ABORT_EXIT, "");
-    }
-    else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
-	Sint i;
-
-	if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) {
-	    goto error;
-	}
-	halt_msg[i] = '\0';
-	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
-    }
-    else
-	goto error;
-    return NIL;  /* Pedantic (lint does not know about erts_exit) */
- error:
-	BIF_ERROR(BIF_P, BADARG);
-}
-
-/**********************************************************************/
-
 /* stop the system with exit code and flags */
-/* ARGSUSED */
 BIF_RETTYPE halt_2(BIF_ALIST_2)
 {
     Uint code;
@@ -3924,7 +3876,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
 	if (flush) {
 	    erts_halt(pos_int_code);
-	    ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
+	    ERTS_BIF_YIELD2(bif_export[BIF_halt_2], BIF_P, am_undefined, am_undefined);
 	}
 	else {
 	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 872f0f9b2a..065018514a 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -72,8 +72,6 @@ bif erlang:get/1
 bif erlang:get_keys/1
 bif erlang:group_leader/0
 bif erlang:group_leader/2
-bif erlang:halt/0
-bif erlang:halt/1
 bif erlang:halt/2
 bif erlang:phash/2
 bif erlang:phash2/1
-- 
cgit v1.2.3


From dcaa52d75e3bcbc808696597a34f2fca5677fff9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 20 May 2016 15:56:56 +0200
Subject: erts: Make erlang:halt/2 truncate string arg

if too long.
---
 erts/emulator/beam/bif.c   | 11 +++++++----
 erts/emulator/beam/utils.c |  8 +++++---
 2 files changed, 12 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 81477422ff..21763edc5b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3838,7 +3838,7 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0)
 
 
 #define HALT_MSG_SIZE	200
-static char halt_msg[HALT_MSG_SIZE];
+static char halt_msg[HALT_MSG_SIZE+1];
 
 /* stop the system with exit code and flags */
 BIF_RETTYPE halt_2(BIF_ALIST_2)
@@ -3892,9 +3892,12 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
     else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
 	Sint i;
 
-	if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) {
-	    goto error;
-	}
+        if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE)) == -1) {
+            goto error;
+        }
+        if (i == -2) /* truncated string */
+            i = HALT_MSG_SIZE;
+        ASSERT(i >= 0 && i <= HALT_MSG_SIZE);
 	halt_msg[i] = '\0';
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index cedc88e5fe..f0418446a8 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3893,8 +3893,10 @@ void bin_write(int to, void *to_arg, byte* buf, size_t sz)
 }
 
 /* Fill buf with the contents of bytelist list 
-   return number of chars in list or -1 for error */
-
+ * return number of chars in list
+ * or -1 for type error
+ * or -2 for not enough buffer space (buffer contains truncated result)
+ */
 Sint
 intlist_to_buf(Eterm list, char *buf, Sint len)
 {
@@ -3917,7 +3919,7 @@ intlist_to_buf(Eterm list, char *buf, Sint len)
 	    return -1;
 	listptr = list_val(*(listptr + 1));
     }
-    return -1;			/* not enough space */
+    return -2;			/* not enough space */
 }
 
 /*
-- 
cgit v1.2.3


From 36f98375d57daaba3fec42bb91482cdac9ef4cc9 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 25 May 2016 16:15:36 +0200
Subject: Remove the 'message_queue_data' option 'mixed'

---
 erts/emulator/beam/atom.names     |  1 -
 erts/emulator/beam/bif.c          |  3 ---
 erts/emulator/beam/erl_bif_info.c |  5 -----
 erts/emulator/beam/erl_init.c     |  6 ++----
 erts/emulator/beam/erl_message.c  | 28 +---------------------------
 erts/emulator/beam/erl_process.c  |  9 ++++++++-
 6 files changed, 11 insertions(+), 41 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 8f65e71531..464058535a 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -389,7 +389,6 @@ atom min_heap_size
 atom min_bin_vheap_size
 atom minor_version
 atom Minus='-'
-atom mixed
 atom module
 atom module_info
 atom monitored_by
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2a3bd4afe5..e77da526fd 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -917,9 +917,6 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
 		    goto error;
 	    } else if (arg == am_message_queue_data) {
 		switch (val) {
-		case am_mixed:
-		    so.flags &= ~(SPO_OFF_HEAP_MSGQ|SPO_ON_HEAP_MSGQ);
-		    break;
 		case am_on_heap:
 		    so.flags &= ~SPO_OFF_HEAP_MSGQ;
 		    so.flags |= SPO_ON_HEAP_MSGQ;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 2e195db0ee..b410578d37 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1565,9 +1565,6 @@ process_info_aux(Process *BIF_P,
 	case F_ON_HEAP_MSGQ:
 	    res = am_on_heap;
 	    break;
-	case 0:
-	    res = am_mixed;
-	    break;
 	default:
 	    res = am_error;
 	    ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
@@ -2809,8 +2806,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	    BIF_RET(am_off_heap);
 	case SPO_ON_HEAP_MSGQ:
 	    BIF_RET(am_on_heap);
-	case 0:
-	    BIF_RET(am_mixed);
 	default:
 	    ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
 	    BIF_RET(am_error);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 0649fb68de..fbdafec4ef 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -585,7 +585,7 @@ void erts_usage(void)
     erts_fprintf(stderr, "-hpds size     initial process dictionary size (default %d)\n",
 	       erts_pd_initial_size);
     erts_fprintf(stderr, "-hmqd  val     set default message queue data flag for processes,\n");
-    erts_fprintf(stderr, "               valid values are: off_heap | on_heap | mixed\n");
+    erts_fprintf(stderr, "               valid values are: off_heap | on_heap\n");
 
     /*    erts_fprintf(stderr, "-i module  set the boot module (default init)\n"); */
 
@@ -1526,9 +1526,7 @@ erl_start(int argc, char **argv)
 			    erts_pd_initial_size));
             } else if (has_prefix("mqd", sub_param)) {
 		arg = get_arg(sub_param+3, argv[i+1], &i);
-		if (sys_strcmp(arg, "mixed") == 0)
-		    erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ);
-		else if (sys_strcmp(arg, "on_heap") == 0) {
+		if (sys_strcmp(arg, "on_heap") == 0) {
 		    erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ;
 		    erts_default_spo_flags |= SPO_ON_HEAP_MSGQ;
 		}
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 579f6e427d..d82532b354 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1123,11 +1123,9 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	    break;
 	case am_on_heap:
 	    c_p->flags |= F_ON_HEAP_MSGQ;
+	    c_p->flags &= ~F_OFF_HEAP_MSGQ;
 	    erts_smp_atomic32_read_bor_nob(&c_p->state,
 					   ERTS_PSFLG_ON_HEAP_MSGQ);
-	    /* fall through */
-	case am_mixed:
-	    c_p->flags &= ~F_OFF_HEAP_MSGQ;
 	    /*
 	     * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ
 	     * if a off heap change is ongoing. It will be adjusted
@@ -1151,11 +1149,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	switch (new_state) {
 	case am_on_heap:
 	    break;
-	case am_mixed:
-	    c_p->flags &= ~F_ON_HEAP_MSGQ;
-	    erts_smp_atomic32_read_band_nob(&c_p->state,
-					    ~ERTS_PSFLG_ON_HEAP_MSGQ);
-	    break;
 	case am_off_heap:
 	    c_p->flags &= ~F_ON_HEAP_MSGQ;
 	    erts_smp_atomic32_read_band_nob(&c_p->state,
@@ -1167,25 +1160,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	}
 	break;
 
-    case 0:
-	res = am_mixed;
-
-	switch (new_state) {
-	case am_mixed:
-	    break;
-	case am_on_heap:
-	    c_p->flags |= F_ON_HEAP_MSGQ;
-	    erts_smp_atomic32_read_bor_nob(&c_p->state,
-					   ERTS_PSFLG_ON_HEAP_MSGQ);
-	    break;
-	case am_off_heap:
-	    goto change_to_off_heap;
-	default:
-	    res = THE_NON_VALUE; /* badarg */
-	    break;
-	}
-	break;
-
     default:
 	res = am_error;
 	ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a853ec585b..f8cbe60e76 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -149,7 +149,7 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
-int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = 0;
+int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = SPO_ON_HEAP_MSGQ;
 int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
 int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
 int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
@@ -11206,6 +11206,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	flags |= F_ON_HEAP_MSGQ;
     }
 
+    ASSERT((flags & F_ON_HEAP_MSGQ) || (flags & F_OFF_HEAP_MSGQ));
+
     if (!rq)
 	rq = erts_get_runq_proc(parent);
 
@@ -11218,6 +11220,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	goto error;
     }
 
+    ASSERT((erts_smp_atomic32_read_nob(&p->state)
+	    & ERTS_PSFLG_ON_HEAP_MSGQ)
+	   || (erts_smp_atomic32_read_nob(&p->state)
+	       & ERTS_PSFLG_OFF_HEAP_MSGQ));
+
 #ifdef BM_COUNTERS
     processes_busy++;
 #endif
-- 
cgit v1.2.3


From 50447f0bfa84046961cdc7693f7c374f5b81d461 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 26 May 2016 12:13:19 +0200
Subject: erts: Improve some bad hash functions

Multiplying two atoms will always yield the same low 6 bits.

The most important tabke 'export' was saved by the
prime number table size which seemed to yield a decent uniform
distribution anyway.
---
 erts/emulator/beam/export.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 02c24557c1..cfe7c06823 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -31,7 +31,7 @@
 #define EXPORT_INITIAL_SIZE   4000
 #define EXPORT_LIMIT  (512*1024)
 
-#define EXPORT_HASH(m,f,a) ((m)*(f)+(a))
+#define EXPORT_HASH(m,f,a) ((atom_val(m) * atom_val(f)) ^ (a))
 
 #ifdef DEBUG
 #  define IF_DEBUG(x) x
-- 
cgit v1.2.3


From 17cca30753584021dc1c518fffa42f4629a73775 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 26 May 2016 11:34:43 +0200
Subject: erts: Tweak ErtsContainerStruct macro

with surrounding parenthesis
---
 erts/emulator/beam/sys.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index f303d4f167..9a205d50d3 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -97,7 +97,7 @@
     ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 #define ErtsContainerStruct(ptr, type, member) \
-  (type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))
+    ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member)))
 
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
-- 
cgit v1.2.3


From 770442cd4775a9bc56b3d0476754d8ea91ca2e2f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 25 May 2016 20:38:03 +0200
Subject: erts: Add some more use of ErtsContainerStruct

---
 erts/emulator/beam/erl_ao_firstfit_alloc.c | 2 +-
 erts/emulator/beam/export.c                | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index fbe4724047..7e239d1f5d 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -123,7 +123,7 @@ struct AOFF_Carrier_t_ {
     AOFF_RBTree_t rbt_node;     /* My node in the carrier tree */
     AOFF_RBTree_t* root;        /* Root of my block tree */
 };
-#define RBT_NODE_TO_MBC(PTR) ((AOFF_Carrier_t*)((char*)(PTR) - offsetof(AOFF_Carrier_t, rbt_node)))
+#define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node)
 
 /* 
    To support carrier migration we keep two kinds of rb-trees:
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index cfe7c06823..2a19211987 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -79,8 +79,7 @@ struct export_templ
 
 static struct export_blob* entry_to_blob(struct export_entry* ee)
 {
-    return (struct export_blob*)
-        ((char*)ee->ep - offsetof(struct export_blob,exp));
+    return ErtsContainerStruct(ee->ep, struct export_blob, exp);
 }
 
 void
-- 
cgit v1.2.3


From 8241eb36a733d6fee7c4a6f12c8bdfc206889795 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 26 May 2016 16:17:45 +0200
Subject: Fix message queue update on replacement and removal of message

---
 erts/emulator/beam/erl_gc.c      | 15 +++++----------
 erts/emulator/beam/erl_message.c | 15 +++++----------
 erts/emulator/beam/erl_message.h | 29 +++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 20 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 374da9407c..c7bbbd5ca0 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2293,18 +2293,13 @@ move_msgq_to_heap(Process *p)
 		    free_message_buffer(bp);
 		}
 		else {
-		    ErtsMessage *tmp = erts_alloc_message(0, NULL);
-		    sys_memcpy((void *) tmp->m, (void *) mp->m,
-			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); 
-		    tmp->next = mp->next;
-		    if (p->msg.save == &mp->next)
-			p->msg.save = &tmp->next;
-		    if (p->msg.last == &mp->next)
-			p->msg.last = &tmp->next;
-		    *mpp = tmp;
+		    ErtsMessage *new_mp = erts_alloc_message(0, NULL);
+		    sys_memcpy((void *) new_mp->m, (void *) mp->m,
+			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ);
+		    erts_msgq_replace_msg_ref(&p->msg, new_mp, mpp);
 		    mp->next = NULL;
 		    erts_cleanup_messages(mp);
-		    mp = tmp;
+		    mp = new_mp;
 		}
 	    }
 
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 579f6e427d..b761522d91 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1371,10 +1371,10 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp,
 
 		mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
 
-		if (rp->msg.save == &bad_mp->next)
-		    rp->msg.save = mpp;
-		if (rp->msg.last == &bad_mp->next)
-		    rp->msg.last = mpp;
+		ASSERT((*mpp)->next == bad_mp);
+
+		erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next);
+
 		mp = mp->next;
 		*mpp = mp;
 		rp->msg.len--;
@@ -1411,12 +1411,7 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp,
 		    sys_memcpy((void *) tmp->m, (void *) mp->m,
 			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); 
 		    mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
-		    tmp->next = mp->next;
-		    if (rp->msg.save == &mp->next)
-			rp->msg.save = &tmp->next;
-		    if (rp->msg.last == &mp->next)
-			rp->msg.last = &tmp->next;
-		    *mpp = tmp;
+		    erts_msgq_replace_msg_ref(&rp->msg, tmp, mpp);
 		    erts_save_message_in_proc(rp, mp);
 		    mp = tmp;
 		}
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 46063ea0c2..6df969367b 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -366,6 +366,12 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz,
 ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp);
 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg);
+ERTS_GLB_INLINE void erts_msgq_update_internal_pointers(ErlMessageQueue *msgq,
+							ErtsMessage **newpp,
+							ErtsMessage **oldpp);
+ERTS_GLB_INLINE void erts_msgq_replace_msg_ref(ErlMessageQueue *msgq,
+					       ErtsMessage *newp,
+					       ErtsMessage **oldpp);
 
 #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1)
 
@@ -468,6 +474,29 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg)
 	return sz;
     }
 }
+
+ERTS_GLB_INLINE void
+erts_msgq_update_internal_pointers(ErlMessageQueue *msgq,
+				   ErtsMessage **newpp,
+				   ErtsMessage **oldpp)
+{
+    if (msgq->save == oldpp)
+	msgq->save = newpp;
+    if (msgq->last == oldpp)
+	msgq->last = newpp;
+    if (msgq->saved_last == oldpp)
+	msgq->saved_last = newpp;
+}
+
+ERTS_GLB_INLINE void
+erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage **oldpp)
+{
+    ErtsMessage *oldp = *oldpp;
+    newp->next = oldp->next;
+    erts_msgq_update_internal_pointers(msgq, &newp->next, &oldp->next);
+    *oldpp = newp;
+}
+
 #endif
 
 #endif
-- 
cgit v1.2.3


From 20383144c26512fe33e72f49dff52dc628666923 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 26 May 2016 16:23:24 +0200
Subject: Improve message allocation in enif_send()

---
 erts/emulator/beam/erl_nif.c | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 606b73c7b5..2bbb8e3c91 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -623,10 +623,28 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         }
     } else {
         Uint sz = size_object(msg);
+	ErlOffHeap *ohp;
         Eterm *hp;
-        mp = erts_alloc_message(sz, &hp);
-        msg = copy_struct(msg, sz, &hp, &mp->hfrag.off_heap);
-        ASSERT(hp == mp->hfrag.mem+mp->hfrag.used_size);
+	if (env && !env->tracee) {
+	    flush_env(env);
+	    mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+	    cache_env(env);
+	}
+	else {
+	    erts_aint_t state = erts_smp_atomic32_read_nob(&rp->state);
+	    if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) {
+		mp = erts_alloc_message(sz, &hp);
+		ohp = sz == 0 ? NULL : &mp->hfrag.off_heap;
+	    }
+	    else {
+		ErlHeapFragment *bp = new_message_buffer(sz);
+		mp = erts_alloc_message(0, NULL);
+		mp->data.heap_frag = bp;
+		hp = bp->mem;
+		ohp = &bp->off_heap;
+	    }
+	}
+        msg = copy_struct(msg, sz, &hp, ohp);
     }
 
     ERL_MESSAGE_TERM(mp) = msg;
-- 
cgit v1.2.3


From 05badf19300e2970cdeb914a62b2c2b424fe7711 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 20 May 2016 17:45:56 +0200
Subject: erts: Split large binaries into multiple iovec

On windows the max size of an iov element is long, i.e. 4GB
so in order to write larger binaries to file we split the
binary into smaller 2GB chunks so that the write is possible.
---
 erts/emulator/beam/io.c | 59 ++++++++++++++++++++++++++++++++-----------------
 1 file changed, 39 insertions(+), 20 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 0377f6cb5e..01df5476db 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -938,18 +938,32 @@ int erts_port_handle_xports(Port *prt)
 **        -2 on type error
 */
 
-#define SET_VEC(iov, bv, bin, ptr, len, vlen) do {	\
-   (iov)->iov_base = (ptr);				\
-   (iov)->iov_len = (len);				\
-   if (sizeof((iov)->iov_len) < sizeof(len)				\
-       /* Check if (len) overflowed (iov)->iov_len */                   \
-       && (iov)->iov_len != (len)) {		                        \
-       goto L_overflow;							\
-   }									\
-   *(bv)++ = (bin);					\
-   (iov)++;						\
-   (vlen)++;						\
-} while(0)
+#ifdef DEBUG
+#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1))
+#else
+#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1))
+#endif
+
+static ERTS_INLINE void
+io_list_to_vec_set_vec(SysIOVec **iov, ErlDrvBinary ***binv,
+                        ErlDrvBinary *bin, byte *ptr, Uint len,
+                        int *vlen)
+{
+    while (len > MAX_SYSIOVEC_IOVLEN) {
+        (*iov)->iov_base = ptr;
+        (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN;
+        ptr += MAX_SYSIOVEC_IOVLEN;
+        len -= MAX_SYSIOVEC_IOVLEN;
+        (*iov)++;
+        (*vlen)++;
+        *(*binv)++ = bin;
+    }
+    (*iov)->iov_base = ptr;
+    (*iov)->iov_len = len;
+    *(*binv)++ = bin;
+    (*iov)++;
+    (*vlen)++;
+}
 
 static int
 io_list_to_vec(Eterm obj,	/* io-list */
@@ -960,11 +974,11 @@ io_list_to_vec(Eterm obj,	/* io-list */
 {
     DECLARE_ESTACK(s);
     Eterm* objp;
-    char *buf  = cbin->orig_bytes;
+    byte *buf  = (byte*)cbin->orig_bytes;
     Uint len = cbin->orig_size;
     Uint csize  = 0;
     int vlen   = 0;
-    char* cptr = buf;
+    byte* cptr = buf;
 
     goto L_jump_start;  /* avoid push */
 
@@ -1032,15 +1046,17 @@ io_list_to_vec(Eterm obj,	/* io-list */
 		    len -= size;
 		} else {
 		    if (csize != 0) {
-			SET_VEC(iov, binv, cbin, cptr, csize, vlen);
+                        io_list_to_vec_set_vec(&iov, &binv, cbin,
+                                               cptr, csize, &vlen);
 			cptr = buf;
 			csize = 0;
 		    }
 		    if (pb->flags) {
 			erts_emasculate_writable_binary(pb);
 		    }
-		    SET_VEC(iov, binv, Binary2ErlDrvBinary(pb->val),
-			    pb->bytes+offset, size, vlen);
+                    io_list_to_vec_set_vec(
+                        &iov, &binv, Binary2ErlDrvBinary(pb->val),
+                        pb->bytes+offset, size, &vlen);
 		}
 	    } else {
 		ErlHeapBin* hb = (ErlHeapBin *) bptr;
@@ -1060,7 +1076,7 @@ io_list_to_vec(Eterm obj,	/* io-list */
     }
 
     if (csize != 0) {
-	SET_VEC(iov, binv, cbin, cptr, csize, vlen);
+        io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen);
     }
 
     DESTROY_ESTACK(s);
@@ -1086,10 +1102,13 @@ do {									\
     if (_bitsize != 0) goto L_type_error;				\
     if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG &&	\
 	_bitoffs == 0) {						\
-	b_size += _size;						\
+	b_size += _size;                                                \
         if (b_size < _size) goto L_overflow_error;			\
 	in_clist = 0;							\
-	v_size++;							\
+        v_size++;                                                       \
+        /* If iov_len is smaller then Uint we split the binary into*/   \
+        /* multiple smaller (2GB) elements in the iolist.*/             \
+	v_size += _size / MAX_SYSIOVEC_IOVLEN;                          \
         if (_size >= ERL_SMALL_IO_BIN_LIMIT) {				\
             p_in_clist = 0;						\
             p_v_size++;							\
-- 
cgit v1.2.3


From 728b48254bf4e3979a2e4c6cd667ace5cd020d12 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 30 May 2016 20:44:10 +0200
Subject: erts: Fix ASSERT provoked by map_SUITE:t_rare_map_overflow

The same bug was fixed for OTP 18 in cb62c989e59f0ec8556f9f1d4e9a45b
by provoking yet another GC.

But now in 19 we are ok with heap fragments
so just remove the asserts.
---
 erts/emulator/beam/erl_gc.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c7bbbd5ca0..9da82f9bf2 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1198,7 +1198,6 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     done:
 	ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
-	ASSERT(MBUF(p) == NULL);
 
         /* The heap usage during GC should be larger than what we end up
            after a GC, even if we grow it. If this assertion is not true
@@ -1591,6 +1590,9 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     HIGH_WATER(p) = HEAP_TOP(p);
 
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p);
+#endif
     remove_message_buffers(p);
 
     if (p->flags & F_ON_HEAP_MSGQ)
@@ -1603,9 +1605,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     adjusted = adjust_after_fullsweep(p, need, objv, nobj);
 
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
     ErtsGcQuickSanityCheck(p);
 
     return gc_cost(size_after, adjusted ? size_after : 0);
-- 
cgit v1.2.3


From ca3c71f3e22b93bc633562d0d1a2f6479c21b223 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 30 May 2016 20:49:04 +0200
Subject: erts: Clean up some goto spaghetti

and replace with a nice else-if chain.
---
 erts/emulator/beam/erl_gc.c | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 9da82f9bf2..d0d74bbf44 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1183,20 +1183,13 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
 		adjust_size = p->htop - p->heap;
             }
 
-	    goto done;
         }
+        else if (need_after > HEAP_SIZE(p)) {
+            grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
+            adjust_size = p->htop - p->heap;
+        }
+	/*else: The heap size turned out to be just right. We are done. */
 
-        if (HEAP_SIZE(p) >= need_after) {
-	    /*
-	     * The heap size turned out to be just right. We are done.
-	     */
-	    goto done;
-	}
-
-        grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
-	adjust_size = p->htop - p->heap;
-
-    done:
 	ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
 
         /* The heap usage during GC should be larger than what we end up
-- 
cgit v1.2.3


From a6a4ce5c63cae9e0d3a35fea3074a59dd06f22fc Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 31 May 2016 15:00:50 +0200
Subject: Do not send on_heap msgs to error logger in non-smp emulator

---
 erts/emulator/beam/utils.c | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f0418446a8..675fafa726 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2210,7 +2210,9 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
 
 #ifndef ERTS_SMP
 #ifdef USE_THREADS
-    if (erts_get_scheduler_data()) /* Must be scheduler thread */
+    if (!erts_get_scheduler_data()) /* Must be scheduler thread */
+	*p = NULL;
+    else
 #endif
     {
 	*p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
@@ -2226,18 +2228,10 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
     }
 
     /* So we have an error logger, lets build the message */
-    if (sz <= HeapWordsLeft(*p)) {
-	*ohp = &MSO(*p);
-	*hp = HEAP_TOP(*p);
-	HEAP_TOP(*p) += sz;
-    } else {
-#endif
-	*bp = new_message_buffer(sz);
-	*ohp = &(*bp)->off_heap;
-	*hp = (*bp)->mem;
-#ifndef ERTS_SMP
-    }
 #endif
+    *bp = new_message_buffer(sz);
+    *ohp = &(*bp)->off_heap;
+    *hp = (*bp)->mem;
 
     return (is_nil(gleader)
 	  ? am_noproc
-- 
cgit v1.2.3


From 60557173f8a7bd0d4deafdb2b3e066899c586f56 Mon Sep 17 00:00:00 2001
From: Steve Vinoski <vinoski@ieee.org>
Date: Wed, 23 Dec 2015 21:09:19 -0500
Subject: Add dirty_process_main function

Dirty schedulers only execute NIFs, so having them execute the full
process_main function isn't necessary. Add dirty_process_main for
dirty schedulers to execute instead.

Add erts_pre_dirty_nif(), called when preparing to execute a dirty
nif.

Add more dirty NIF tests to verify that activities requiring the
process main lock can succeed when the process is executing a dirty
NIF.
---
 erts/emulator/beam/beam_emu.c    | 307 +++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_nif.c     |  81 ++++++-----
 erts/emulator/beam/erl_process.c |   4 +-
 erts/emulator/beam/global.h      |   3 +
 4 files changed, 350 insertions(+), 45 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index f8f2e29c95..25cc41323b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3559,12 +3559,8 @@ do {						\
 		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (!c_p->scheduler_data)
-		    live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */
-		else
-#endif
-		    live_hf_end = c_p->mbuf;
+		ASSERT(c_p->scheduler_data);
+		live_hf_end = c_p->mbuf;
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
@@ -3574,10 +3570,7 @@ do {						\
 		PROCESS_MAIN_CHK_LOCKS(c_p);
 		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-		if (env.exiting) {
-		    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-		    goto do_schedule;
-		}
+		ASSERT(!env.exiting);
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	    }
 
@@ -5162,6 +5155,300 @@ do {						\
     }
 }
 
+/*
+ * dirty_process_main() is what dirty schedulers execute. Since they handle
+ * only NIF calls they do not need to be able to execute all BEAM
+ * instructions.
+ */
+void dirty_process_main(void)
+{
+    Process* c_p = NULL;
+    int reds_used;
+#ifdef DEBUG
+    ERTS_DECLARE_DUMMY(Eterm pid);
+#endif
+    BeamInstr *do_call_nif = OpCode(call_nif);
+
+    /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
+     * in all other cases x0 is used.
+     */
+    register Eterm* reg REG_xregs = NULL;
+
+    /*
+     * Top of heap (next free location); grows upwards.
+     */
+    register Eterm* HTOP REG_htop = NULL;
+
+    /* Stack pointer.  Grows downwards; points
+     * to last item pushed (normally a saved
+     * continuation pointer).
+     */
+    register Eterm* E REG_stop = NULL;
+
+    /*
+     * Pointer to next threaded instruction.
+     */
+    register BeamInstr *I REG_I = NULL;
+
+    /* Number of reductions left.  This function
+     * returns to the scheduler when FCALLS reaches zero.
+     */
+    register Sint FCALLS REG_fcalls = 0;
+
+    /*
+     * X registers and floating point registers are located in
+     * scheduler specific data.
+     */
+    register FloatDef *freg = NULL;
+
+    /*
+     * For keeping the negative old value of 'reds' when call saving is active.
+     */
+    int neg_o_reds = 0;
+
+    ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
+
+    ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
+
+    c_p = NULL;
+    reds_used = 0;
+
+    goto do_dirty_schedule1;
+
+ context_switch:
+    c_p->arity = I[-1];
+    c_p->current = I-3;		/* Pointer to Mod, Func, Arity */
+
+    {
+	Eterm* argp;
+	int i;
+
+	/*
+	 * Make sure that there is enough room for the argument registers to be saved.
+	 */
+	if (c_p->arity > c_p->max_arg_reg) {
+	    /*
+	     * Yes, this is an expensive operation, but you only pay it the first
+	     * time you call a function with more than 6 arguments which is
+	     * scheduled out.  This is better than paying for 26 words of wasted
+	     * space for most processes which never call functions with more than
+	     * 6 arguments.
+	     */
+	    Uint size = c_p->arity * sizeof(c_p->arg_reg[0]);
+	    if (c_p->arg_reg != c_p->def_arg_reg) {
+		c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG,
+						      (void *) c_p->arg_reg,
+						      size);
+	    } else {
+		c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size);
+	    }
+	    c_p->max_arg_reg = c_p->arity;
+	}
+
+	/*
+	 * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
+	 * now before saving registers.
+	 */
+
+	ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+	if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	    reds_used = REDS_IN(c_p) - FCALLS;
+	else
+	    reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+	ASSERT(reds_used >= 0);
+
+	/*
+	 * Save the argument registers and everything else.
+	 */
+
+	argp = c_p->arg_reg;
+	for (i = c_p->arity - 1; i >= 0; i--) {
+	    argp[i] = reg[i];
+	}
+	SWAPOUT;
+	c_p->i = I;
+	goto do_dirty_schedule1;
+    }
+
+ do_dirty_schedule:
+    ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	reds_used = REDS_IN(c_p) - FCALLS;
+    else
+	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+    ASSERT(reds_used >= 0);
+ do_dirty_schedule1:
+
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    c_p = schedule(c_p, reds_used);
+    ASSERT(!(c_p->flags & F_HIPE_MODE));
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+#ifdef DEBUG
+    pid = c_p->common.id; /* Save for debugging purposes */
+#endif
+    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+
+    ERTS_MSACC_UPDATE_CACHE_X();
+
+    reg = erts_proc_sched_data(c_p)->x_reg_array;
+    freg = erts_proc_sched_data(c_p)->f_reg_array;
+    ERL_BITS_RELOAD_STATEP(c_p);
+    {
+	int reds;
+	Eterm* argp;
+	int i;
+
+	argp = c_p->arg_reg;
+	for (i = c_p->arity - 1; i >= 0; i--) {
+	    reg[i] = argp[i];
+	    CHECK_TERM(reg[i]);
+	}
+
+	/*
+	 * We put the original reduction count in the process structure, to reduce
+	 * the code size (referencing a field in a struct through a pointer stored
+	 * in a register gives smaller code than referencing a global variable).
+	 */
+
+	I = c_p->i;
+
+	REDS_IN(c_p) = reds = c_p->fcalls;
+#ifdef DEBUG
+	c_p->debug_reds_in = reds;
+#endif
+
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+	    neg_o_reds = -CONTEXT_REDS;
+	    FCALLS = neg_o_reds + reds;
+	} else {
+	    neg_o_reds = 0;
+	    FCALLS = reds;
+	}
+
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
+	SWAPIN;
+
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(process_scheduled)) {
+            DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
+            DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
+            dtrace_proc_str(c_p, process_buf);
+
+            if (ERTS_PROC_IS_EXITING(c_p)) {
+                strcpy(fun_buf, "<exiting>");
+            } else {
+                BeamInstr *fptr = find_function_from_pc(c_p->i);
+                if (fptr) {
+                    dtrace_fun_decode(c_p, (Eterm)fptr[0],
+                                      (Eterm)fptr[1], (Uint)fptr[2],
+                                      NULL, fun_buf);
+                } else {
+                    erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
+                                  "<unknown/%p>", next);
+                }
+            }
+
+            DTRACE2(process_scheduled, process_buf, fun_buf);
+        }
+#endif
+    }
+
+    {
+	Eterm nif_bif_result;
+	Eterm bif_nif_arity;
+
+    OpCase(call_nif):
+	{
+	    /*
+	     * call_nif is always first instruction in function:
+	     *
+	     * I[-3]: Module
+	     * I[-2]: Function
+	     * I[-1]: Arity
+	     * I[0]: &&call_nif
+	     * I[1]: Function pointer to NIF function
+	     * I[2]: Pointer to erl_module_nif
+	     * I[3]: Function pointer to dirty NIF
+	     */
+	    BifFunction vbf;
+
+            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+                /* If we have run out of reductions, we do a context
+                   switch before calling the nif */
+                goto context_switch;
+            }
+
+	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
+
+	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    c_p->current = I-3; /* current and vbf set to please handle_error */
+	    SWAPOUT;
+	    c_p->fcalls = FCALLS - 1;
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    bif_nif_arity = I[-1];
+	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	    {
+		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
+		NifF* fp = vbf = (NifF*) I[1];
+		struct enif_environment_t env;
+		ASSERT(!c_p->scheduler_data);
+		erts_pre_dirty_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
+		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+		if (env.exception_thrown)
+		    nif_bif_result = THE_NON_VALUE;
+		erts_post_nif(&env);
+		PROCESS_MAIN_CHK_LOCKS(c_p);
+		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+		if (env.exiting) {
+		    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+		    goto do_dirty_schedule;
+		}
+		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	    }
+	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_HOLE_CHECK(c_p);
+	    if (ERTS_IS_GC_DESIRED(c_p)) {
+		nif_bif_result = erts_gc_after_bif_call(c_p,
+							nif_bif_result,
+							reg, bif_nif_arity);
+	    }
+	    SWAPIN;  /* There might have been a garbage collection. */
+	    FCALLS = c_p->fcalls;
+	    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+	    if (is_value(nif_bif_result)) {
+		r(0) = nif_bif_result;
+		CHECK_TERM(r(0));
+		I = c_p->cp;
+		c_p->cp = 0;
+		Goto(*I);
+	    } else if (c_p->freason == TRAP) {
+		I = c_p->i;
+		if (c_p->flags & F_HIBERNATE_SCHED) {
+		    c_p->flags &= ~F_HIBERNATE_SCHED;
+		    goto do_dirty_schedule;
+		}
+		DispatchMacro();
+	    }
+	    I = handle_error(c_p, c_p->cp, reg, vbf);
+	}
+    }
+    if (I == 0) {
+	goto do_dirty_schedule;
+    } else {
+	ASSERT(!is_value(r(0)));
+	SWAPIN;
+	Goto(do_call_nif);
+    }
+}
+
 static BifFunction
 translate_gc_bif(void* gcf)
 {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 2bbb8e3c91..4437adb57b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -178,8 +178,10 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need)
 void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
                   Process* tracee)
 {
+#ifdef DEBUG
 #ifdef ERTS_DIRTY_SCHEDULERS
     ErtsSchedulerData *esdp;
+#endif
 #endif
     env->mod_nif = mod_nif;
     env->proc = p;
@@ -193,12 +195,12 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 
     ASSERT(p->common.id != ERTS_INVALID_PID);
 
+#ifdef DEBUG
 #ifdef ERTS_DIRTY_SCHEDULERS
     esdp = erts_get_scheduler_data();
     ASSERT(esdp);
 
     if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-#ifdef DEBUG
 	erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
 
 	ASSERT(p->scheduler_data == esdp);
@@ -206,44 +208,57 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 			 | ERTS_PSFLG_RUNNING_SYS))
 	       && !(state & (ERTS_PSFLG_DIRTY_RUNNING
 			     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+    }
 #endif
+#endif
+}
 
-    }
-    else {
-	Process *sproc;
+void erts_pre_dirty_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
+			Process* tracee)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
 #ifdef DEBUG
-	erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+    erts_aint32_t state;
+#endif
+    Process *sproc;
+    ErtsSchedulerData *esdp;
+    esdp = erts_get_scheduler_data();
+    ASSERT(esdp);
+
+    erts_pre_nif(env, p, mod_nif, tracee);
 
-	ASSERT(!p->scheduler_data);
-	ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
-	       && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+#ifdef DEBUG
+    state = erts_smp_atomic32_read_nob(&p->state);
+
+    ASSERT(!p->scheduler_data);
+    ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
+	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
 #endif
 
-	sproc = esdp->dirty_shadow_process;
-	ASSERT(sproc);
-	ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
-	ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
-	       == (ERTS_PSFLG_ACTIVE
-		   | ERTS_PSFLG_DIRTY_RUNNING
-		   | ERTS_PSFLG_PROXY));
-
-	sproc->next = p;
-	sproc->common.id = p->common.id;
-	sproc->htop = p->htop;
-	sproc->stop = p->stop;
-	sproc->hend = p->hend;
-	sproc->heap = p->heap;
-	sproc->abandoned_heap = p->abandoned_heap;
-	sproc->heap_sz = p->heap_sz;
-	sproc->high_water = p->high_water;
-	sproc->old_hend = p->old_hend;
-	sproc->old_htop = p->old_htop;
-	sproc->old_heap = p->old_heap;
-	sproc->mbuf = NULL;
-	sproc->mbuf_sz = 0;
-	ERTS_INIT_OFF_HEAP(&sproc->off_heap);
-	env->proc = sproc;
-    }
+    sproc = esdp->dirty_shadow_process;
+    ASSERT(sproc);
+    ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+    ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
+	   == (ERTS_PSFLG_ACTIVE
+	       | ERTS_PSFLG_DIRTY_RUNNING
+	       | ERTS_PSFLG_PROXY));
+
+    sproc->next = p;
+    sproc->common.id = p->common.id;
+    sproc->htop = p->htop;
+    sproc->stop = p->stop;
+    sproc->hend = p->hend;
+    sproc->heap = p->heap;
+    sproc->abandoned_heap = p->abandoned_heap;
+    sproc->heap_sz = p->heap_sz;
+    sproc->high_water = p->high_water;
+    sproc->old_hend = p->old_hend;
+    sproc->old_htop = p->old_htop;
+    sproc->old_heap = p->old_heap;
+    sproc->mbuf = NULL;
+    sproc->mbuf_sz = 0;
+    ERTS_INIT_OFF_HEAP(&sproc->off_heap);
+    env->proc = sproc;
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f8cbe60e76..e245c9e6bb 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8197,7 +8197,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    process_main();
+    dirty_process_main();
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty CPU scheduler thread number %beu terminated\n",
@@ -8242,7 +8242,7 @@ sched_dirty_io_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    process_main();
+    dirty_process_main();
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty I/O scheduler thread number %beu terminated\n",
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b76b9cd874..15253bb53e 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -62,6 +62,8 @@ struct enif_environment_t /* ErlNifEnv */
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
+extern void erts_pre_dirty_nif(struct enif_environment_t*, Process*,
+			       struct erl_module_nif*, Process* tracee);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(int to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
@@ -1152,6 +1154,7 @@ void print_pass_through(int, byte*, int);
 int catchlevel(Process*);
 void init_emulator(void);
 void process_main(void);
+void dirty_process_main(void);
 Eterm build_stacktrace(Process* c_p, Eterm exc);
 Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
 void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth);
-- 
cgit v1.2.3


From c8bd15f0e5539ec3056565b535ff6db389f42a6b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 27 May 2016 16:19:08 +0200
Subject: Move dirty reduction count to erts_dirty_process_main()

---
 erts/emulator/beam/beam_emu.c    | 159 +++++++++++++++++----------------------
 erts/emulator/beam/erl_nif.c     |  46 +++++------
 erts/emulator/beam/erl_process.c | 113 +++++-----------------------
 erts/emulator/beam/erl_process.h |  25 +-----
 erts/emulator/beam/global.h      |   5 +-
 5 files changed, 114 insertions(+), 234 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 25cc41323b..0a59f8785c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1323,11 +1323,7 @@ void process_main(void)
 
     if (start_time != 0) {
         Sint64 diff = erts_timestamp_millis() - start_time;
-	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule
-#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
-	    && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))
-#endif
-	    ) {
+	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule) {
 	    BeamInstr *inptr = find_function_from_pc(start_time_i);
 	    BeamInstr *outptr = find_function_from_pc(c_p->i);
 	    monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff);
@@ -1337,7 +1333,7 @@ void process_main(void)
     PROCESS_MAIN_CHK_LOCKS(c_p);
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-    c_p = schedule(c_p, reds_used);
+    c_p = erts_schedule(NULL, c_p, reds_used);
     ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     start_time = 0;
@@ -3559,7 +3555,9 @@ do {						\
 		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
+#ifdef ERTS_SMP
 		ASSERT(c_p->scheduler_data);
+#endif
 		live_hf_end = c_p->mbuf;
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
@@ -5156,18 +5154,18 @@ do {						\
 }
 
 /*
- * dirty_process_main() is what dirty schedulers execute. Since they handle
+ * erts_dirty_process_main() is what dirty schedulers execute. Since they handle
  * only NIF calls they do not need to be able to execute all BEAM
  * instructions.
  */
-void dirty_process_main(void)
+void erts_dirty_process_main(ErtsSchedulerData *esdp)
 {
+#ifdef ERTS_DIRTY_SCHEDULERS
     Process* c_p = NULL;
-    int reds_used;
+    ErtsMonotonicTime start_time;
 #ifdef DEBUG
     ERTS_DECLARE_DUMMY(Eterm pid);
 #endif
-    BeamInstr *do_call_nif = OpCode(call_nif);
 
     /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
      * in all other cases x0 is used.
@@ -5190,36 +5188,30 @@ void dirty_process_main(void)
      */
     register BeamInstr *I REG_I = NULL;
 
-    /* Number of reductions left.  This function
-     * returns to the scheduler when FCALLS reaches zero.
-     */
-    register Sint FCALLS REG_fcalls = 0;
-
-    /*
-     * X registers and floating point registers are located in
-     * scheduler specific data.
-     */
-    register FloatDef *freg = NULL;
+    ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
 
     /*
-     * For keeping the negative old value of 'reds' when call saving is active.
+     * start_time always positive for dirty CPU schedulers,
+     * and negative for dirty I/O schedulers.
      */
-    int neg_o_reds = 0;
 
-    ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */
-
-    ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
-
-    c_p = NULL;
-    reds_used = 0;
+    if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) {
+	start_time = erts_get_monotonic_time(NULL);
+	ASSERT(start_time >= 0);
+    }
+    else {
+	start_time = ERTS_SINT64_MIN;
+	ASSERT(start_time < 0);
+    }
 
-    goto do_dirty_schedule1;
+    goto do_dirty_schedule;
 
  context_switch:
     c_p->arity = I[-1];
     c_p->current = I-3;		/* Pointer to Mod, Func, Arity */
 
     {
+	int reds_used;
 	Eterm* argp;
 	int i;
 
@@ -5245,18 +5237,6 @@ void dirty_process_main(void)
 	    c_p->max_arg_reg = c_p->arity;
 	}
 
-	/*
-	 * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
-	 * now before saving registers.
-	 */
-
-	ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
-	if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
-	    reds_used = REDS_IN(c_p) - FCALLS;
-	else
-	    reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
-	ASSERT(reds_used >= 0);
-
 	/*
 	 * Save the argument registers and everything else.
 	 */
@@ -5267,23 +5247,46 @@ void dirty_process_main(void)
 	}
 	SWAPOUT;
 	c_p->i = I;
-	goto do_dirty_schedule1;
-    }
 
- do_dirty_schedule:
-    ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
-    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
-	reds_used = REDS_IN(c_p) - FCALLS;
-    else
-	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
-    ASSERT(reds_used >= 0);
- do_dirty_schedule1:
+    do_dirty_schedule:
+
+	if (start_time < 0) {
+	    /*
+	     * Dirty I/O scheduler:
+	     *   One reduction consumed regardless of
+	     *   time spent in the dirty NIF.
+	     */
+	    reds_used = esdp->virtual_reds + 1;
+	}
+	else {
+	    /*
+	     * Dirty CPU scheduler:
+	     *   Currently two reductions consumed per
+	     *   micro second spent in the dirty NIF.
+	     */
+	    ErtsMonotonicTime time;
+	    time = erts_get_monotonic_time(esdp);
+	    time -= start_time;
+	    time = ERTS_MONOTONIC_TO_USEC(time);
+	    time *= (CONTEXT_REDS-1)/1000 + 1;
+	    ASSERT(time >= 0);
+	    if (time == 0)
+		time = 1; /* At least one reduction */
+	    time += esdp->virtual_reds;
+	    reds_used = time > INT_MAX ? INT_MAX : (int) time;
+	}
+
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	c_p = erts_schedule(esdp, c_p, reds_used);
+
+	if (start_time >= 0) {
+	    start_time = erts_get_monotonic_time(esdp);
+	    ASSERT(start_time >= 0);
+	}
+    }
 
-    PROCESS_MAIN_CHK_LOCKS(c_p);
-    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-    c_p = schedule(c_p, reds_used);
-    ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 #ifdef DEBUG
     pid = c_p->common.id; /* Save for debugging purposes */
@@ -5291,13 +5294,11 @@ void dirty_process_main(void)
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
+    ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_MSACC_UPDATE_CACHE_X();
 
-    reg = erts_proc_sched_data(c_p)->x_reg_array;
-    freg = erts_proc_sched_data(c_p)->f_reg_array;
-    ERL_BITS_RELOAD_STATEP(c_p);
+    reg = esdp->x_reg_array;
     {
-	int reds;
 	Eterm* argp;
 	int i;
 
@@ -5315,21 +5316,13 @@ void dirty_process_main(void)
 
 	I = c_p->i;
 
-	REDS_IN(c_p) = reds = c_p->fcalls;
-#ifdef DEBUG
-	c_p->debug_reds_in = reds;
-#endif
+	ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
 
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
-	    neg_o_reds = -CONTEXT_REDS;
-	    FCALLS = neg_o_reds + reds;
-	} else {
-	    neg_o_reds = 0;
-	    FCALLS = reds;
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
+	    && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
+	    c_p->fcalls = REDS_IN(c_p) = 0;
 	}
 
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-
 	SWAPIN;
 
 #ifdef USE_VM_PROBES
@@ -5361,7 +5354,6 @@ void dirty_process_main(void)
 	Eterm nif_bif_result;
 	Eterm bif_nif_arity;
 
-    OpCase(call_nif):
 	{
 	    /*
 	     * call_nif is always first instruction in function:
@@ -5376,18 +5368,11 @@ void dirty_process_main(void)
 	     */
 	    BifFunction vbf;
 
-            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
-                /* If we have run out of reductions, we do a context
-                   switch before calling the nif */
-                goto context_switch;
-            }
-
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
 	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    c_p->current = I-3; /* current and vbf set to please handle_error */
 	    SWAPOUT;
-	    c_p->fcalls = FCALLS - 1;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = I[-1];
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
@@ -5398,7 +5383,7 @@ void dirty_process_main(void)
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
 		ASSERT(!c_p->scheduler_data);
-		erts_pre_dirty_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
+		erts_pre_dirty_nif(esdp, &env, c_p, (struct erl_module_nif*)I[2], NULL);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
 		if (env.exception_thrown)
 		    nif_bif_result = THE_NON_VALUE;
@@ -5421,8 +5406,6 @@ void dirty_process_main(void)
 							reg, bif_nif_arity);
 	    }
 	    SWAPIN;  /* There might have been a garbage collection. */
-	    FCALLS = c_p->fcalls;
-	    ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	    if (is_value(nif_bif_result)) {
 		r(0) = nif_bif_result;
 		CHECK_TERM(r(0));
@@ -5431,11 +5414,8 @@ void dirty_process_main(void)
 		Goto(*I);
 	    } else if (c_p->freason == TRAP) {
 		I = c_p->i;
-		if (c_p->flags & F_HIBERNATE_SCHED) {
-		    c_p->flags &= ~F_HIBERNATE_SCHED;
-		    goto do_dirty_schedule;
-		}
-		DispatchMacro();
+		ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+		goto context_switch;
 	    }
 	    I = handle_error(c_p, c_p->cp, reg, vbf);
 	}
@@ -5445,8 +5425,9 @@ void dirty_process_main(void)
     } else {
 	ASSERT(!is_value(r(0)));
 	SWAPIN;
-	Goto(do_call_nif);
+	goto context_switch;
     }
+#endif /* ERTS_DIRTY_SCHEDULERS */
 }
 
 static BifFunction
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4437adb57b..f1c35cd5a5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -178,11 +178,6 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need)
 void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
                   Process* tracee)
 {
-#ifdef DEBUG
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ErtsSchedulerData *esdp;
-#endif
-#endif
     env->mod_nif = mod_nif;
     env->proc = p;
     env->hp = HEAP_TOP(p);
@@ -195,46 +190,41 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 
     ASSERT(p->common.id != ERTS_INVALID_PID);
 
-#ifdef DEBUG
-#ifdef ERTS_DIRTY_SCHEDULERS
-    esdp = erts_get_scheduler_data();
-    ASSERT(esdp);
+#if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS)
+    {
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+	ASSERT(esdp);
 
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
 
-	ASSERT(p->scheduler_data == esdp);
-	ASSERT((state & (ERTS_PSFLG_RUNNING
-			 | ERTS_PSFLG_RUNNING_SYS))
-	       && !(state & (ERTS_PSFLG_DIRTY_RUNNING
-			     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+	    ASSERT(p->scheduler_data == esdp);
+	    ASSERT((state & (ERTS_PSFLG_RUNNING
+			     | ERTS_PSFLG_RUNNING_SYS))
+		   && !(state & (ERTS_PSFLG_DIRTY_RUNNING
+				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+	}
     }
 #endif
-#endif
 }
 
-void erts_pre_dirty_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
+void erts_pre_dirty_nif(ErtsSchedulerData *esdp,
+			ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 			Process* tracee)
 {
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef DEBUG
-    erts_aint32_t state;
-#endif
     Process *sproc;
-    ErtsSchedulerData *esdp;
-    esdp = erts_get_scheduler_data();
-    ASSERT(esdp);
-
-    erts_pre_nif(env, p, mod_nif, tracee);
-
 #ifdef DEBUG
-    state = erts_smp_atomic32_read_nob(&p->state);
+    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
 
     ASSERT(!p->scheduler_data);
     ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
 	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+    ASSERT(esdp);
 #endif
 
+    erts_pre_nif(env, p, mod_nif, tracee);
+
     sproc = esdp->dirty_shadow_process;
     ASSERT(sproc);
     ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e245c9e6bb..c0b1d7246c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8197,7 +8197,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    dirty_process_main();
+    erts_dirty_process_main(esdp);
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty CPU scheduler thread number %beu terminated\n",
@@ -8242,7 +8242,7 @@ sched_dirty_io_thread_func(void *vesdp)
 #endif
     erts_thread_init_float();
 
-    dirty_process_main();
+    erts_dirty_process_main(esdp);
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
 	     "Dirty I/O scheduler thread number %beu terminated\n",
@@ -9377,77 +9377,6 @@ scheduler_gc_proc(Process *c_p, int reds_left)
     return reds;
 }
 
-static ERTS_INLINE void
-clean_dirty_start(Process *p)
-{
-#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64)
-    void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL);
-    if (ptr)
-	erts_free(ERTS_ALC_T_DIRTY_START, ptr);
-#endif
-}
-
-static ERTS_INLINE void
-save_dirty_start(ErtsSchedulerData *esdp, Process *c_p)
-{
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
-	ErtsMonotonicTime time = erts_get_monotonic_time(esdp);
-#ifdef ARCH_64
-	ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time);
-#else
-	ErtsMonotonicTime *stimep;
-
-	stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
-	if (!stimep) {
-	    stimep = erts_alloc(ERTS_ALC_T_DIRTY_START,
-				sizeof(ErtsMonotonicTime));
-	    ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep);
-	}
-	*stimep = time;
-#endif
-    }
-#endif
-}
-
-static ERTS_INLINE int
-get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p)
-{
-
-#ifndef ERTS_DIRTY_SCHEDULERS
-    return -1;
-#else
-    ErtsMonotonicTime stime, time;
-
-    if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue))
-	return 1;
-
-#ifdef ARCH_64
-    stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
-#else
-    {
-	ErtsMonotonicTime *stimep;
-	stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p);
-	ASSERT(stimep);
-	stime = *stimep;
-    }
-#endif
-
-    time = erts_get_monotonic_time(esdp);
-
-    ASSERT(stime && stime < time);
-
-    time -= stime;
-    time = ERTS_MONOTONIC_TO_USEC(time);
-    time *= 2;
-
-    if (time > INT_MAX)
-	return INT_MAX;
-    return (int) time;
-#endif
-
-}
-
 /*
  * schedule() is called from BEAM (process_main()) or HiPE
  * (hipe_mode_switch()) when the current process is to be
@@ -9466,11 +9395,10 @@ get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p)
  * so that normal processes get to run more frequently. 
  */
 
-Process *schedule(Process *p, int calls)
+Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 {
     Process *proxy_p = NULL;
     ErtsRunQueue *rq;
-    ErtsSchedulerData *esdp;
     int context_reds;
     int fcalls;
     int input_reductions;
@@ -9507,8 +9435,19 @@ Process *schedule(Process *p, int calls)
      * Clean up after the process being scheduled out.
      */
     if (!p) {	/* NULL in the very first schedule() call */
+#ifdef ERTS_DIRTY_SCHEDULERS
+	is_normal_sched = !esdp;
+	if (is_normal_sched) {
+	    esdp = erts_get_scheduler_data();
+	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+	}
+	else {
+	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
+	}
+#else
 	esdp = erts_get_scheduler_data();
-	is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp);
+	is_normal_sched = 1;
+#endif
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
 	fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
@@ -9517,12 +9456,12 @@ Process *schedule(Process *p, int calls)
     } else {
 #ifdef ERTS_SMP
 #ifdef ERTS_DIRTY_SCHEDULERS
-	esdp = p->scheduler_data;
-	is_normal_sched = esdp != NULL;
-	if (is_normal_sched)
+	is_normal_sched = !esdp;
+	if (is_normal_sched) {
+	    esdp = p->scheduler_data;
 	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+	}
 	else {
-	    esdp = erts_get_scheduler_data();
 	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
 	}
 #else
@@ -9541,10 +9480,7 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
-	if (is_normal_sched)
-	    reds = actual_reds = calls - esdp->virtual_reds;
-	else
-	    reds = actual_reds = get_dirty_reds(esdp, p);
+	reds = actual_reds = calls - esdp->virtual_reds;
 
 	ASSERT(actual_reds >= 0);
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
@@ -9994,17 +9930,10 @@ Process *schedule(Process *p, int calls)
 	    calls = 0;
 	    reds = context_reds;
 
-#ifdef ERTS_SMP
-
 	    erts_smp_runq_unlock(rq);
 
-#endif /* ERTS_SMP */
-
 	}
 
-	if (!is_normal_sched)
-	    save_dirty_start(esdp, p);
-
 #ifdef ERTS_SMP
 
 	if (flags & ERTS_RUNQ_FLG_PROTECTED)
@@ -11745,8 +11674,6 @@ delete_process(Process* p)
     if (nif_export)
 	erts_destroy_nif_export(nif_export);
 
-    clean_dirty_start(p);
-
     /* Cleanup psd */
 
     psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b44ac442aa..7569fd81bd 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -810,25 +810,13 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_DELAYED_GC_TASK_QS		4
 #define ERTS_PSD_NIF_TRAP_EXPORT		5
 #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	6
-#define ERTS_PSD_DIRTY_CPU_START		7
 
-#define ERTS_PSD_SIZE				8
+#define ERTS_PSD_SIZE				7
 
-#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS)
+#if !defined(HIPE)
 #  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
-#  undef ERTS_PSD_DIRTY_CPU_START
 #  undef ERTS_PSD_SIZE
 #  define ERTS_PSD_SIZE 6
-#elif !defined(HIPE)
-#  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
-#  undef ERTS_PSD_DIRTY_CPU_START
-#  undef ERTS_PSD_SIZE
-#  define ERTS_PSD_DIRTY_CPU_START		6
-#  define ERTS_PSD_SIZE 7
-#elif !defined(ERTS_DIRTY_SCHEDULERS)
-#  undef ERTS_PSD_DIRTY_CPU_START
-#  undef ERTS_PSD_SIZE
-#  define ERTS_PSD_SIZE 7
 #endif
 
 typedef struct {
@@ -1831,7 +1819,7 @@ Eterm erts_get_schedulers_binds(Process *c_p);
 Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
 Eterm erts_bind_schedulers(Process *c_p, Eterm how);
 ErtsRunQueue *erts_schedid2runq(Uint);
-Process *schedule(Process*, int);
+Process *erts_schedule(ErtsSchedulerData *, Process*, int);
 void erts_schedule_misc_op(void (*)(void *), void *);
 Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
 void erts_do_exit_process(Process*, Eterm);
@@ -2061,13 +2049,6 @@ erts_psd_set(Process *p, int ix, void *data)
   ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
 #endif
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-#define ERTS_PROC_GET_DIRTY_CPU_START(P) \
-  ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START))
-#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \
-  ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS)))
-#endif
-
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 15253bb53e..f3d4ac56cd 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -62,7 +62,8 @@ struct enif_environment_t /* ErlNifEnv */
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
-extern void erts_pre_dirty_nif(struct enif_environment_t*, Process*,
+extern void erts_pre_dirty_nif(ErtsSchedulerData *,
+			       struct enif_environment_t*, Process*,
 			       struct erl_module_nif*, Process* tracee);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(int to, void* to_arg);
@@ -1154,7 +1155,7 @@ void print_pass_through(int, byte*, int);
 int catchlevel(Process*);
 void init_emulator(void);
 void process_main(void);
-void dirty_process_main(void);
+void erts_dirty_process_main(ErtsSchedulerData *);
 Eterm build_stacktrace(Process* c_p, Eterm exc);
 Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
 void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth);
-- 
cgit v1.2.3


From 4d15511d6fe25ed15b6c1db00c23f1f4db1e9bb6 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 31 May 2016 18:41:58 +0200
Subject: Properly close message factories in erts_factory_trim_and_close()

---
 erts/emulator/beam/erl_message.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ac7b9d6606..ea525b50dd 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1725,7 +1725,7 @@ void erts_factory_trim_and_close(ErtsHeapFactory* factory,
     case FACTORY_MESSAGE: {
 	ErtsMessage *mp = factory->message;
 	if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) {
-	    if (!mp->hfrag.next) {
+	    if (!factory->heap_frags) {
 		Uint sz = factory->hp - factory->hp_start;
 		mp = erts_shrink_message(mp, sz, brefs, brefs_size);
 		factory->message = mp;
-- 
cgit v1.2.3


From fa0d9d191393e74a579f9c2cacb31b18f715ad6c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 31 May 2016 20:00:30 +0200
Subject: Fix bad assertion

---
 erts/emulator/beam/erl_message.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ea525b50dd..71ab92937d 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1345,7 +1345,7 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp,
 
 		mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
 
-		ASSERT((*mpp)->next == bad_mp);
+		ASSERT(*mpp == bad_mp);
 
 		erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next);
 
-- 
cgit v1.2.3


From 9e4d723e601f720e024237e56c6f302ce5125831 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 31 May 2016 21:03:47 +0200
Subject: Update process state flags in etp-commands

---
 erts/emulator/beam/erl_process.h | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 7569fd81bd..7c98b60647 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1167,6 +1167,9 @@ void erts_check_for_holes(Process* p);
  * USR_PRIO -> User prio. i.e., prio the user has set.
  * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
  *             enqueued in. 
+ *
+ * Update etp-proc-state-int in $ERL_TOP/erts/etc/unix/etp-commands.in
+ * when changing ERTS_PSFLG_*.
  */
 #define ERTS_PSFLGS_ACT_PRIO_MASK \
     (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
-- 
cgit v1.2.3


From 7c133fb1094ad1cabbb5cfc157483a43c816c6a9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 1 Jun 2016 16:13:33 +0200
Subject: erts: Change ETS hash load factor

from 700% to 200%
---
 erts/emulator/beam/erl_db_hash.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 74979f984a..4c4ff9b470 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -95,7 +95,8 @@
 /* 
  * The following symbols can be manipulated to "tune" the linear hash array 
  */
-#define CHAIN_LEN 6                 /* Medium bucket chain len      */
+#define GROW_LIMIT(NACTIVE) ((NACTIVE)*2)
+#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2)
 
 /* Number of slots per segment */
 #define SEGSZ_EXP  8
@@ -463,7 +464,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
 static ERTS_INLINE void try_shrink(DbTableHash* tb)
 {
     int nactive = NACTIVE(tb);
-    if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN)
+    if (nactive > SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive)
 	&& !IS_FIXED(tb)) {
 	shrink(tb, nactive);
     }
@@ -862,7 +863,7 @@ Lnew:
     WUNLOCK_HASH(lck);
     {
 	int nactive = NACTIVE(tb);       
-	if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) {
+	if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
 	    grow(tb, nactive);
 	}
     }
@@ -2250,12 +2251,12 @@ static int db_free_table_continue_hash(DbTable *tbl)
 
     done /= 2;
     while(tb->nslots != 0) {
-	free_seg(tb, 1);
+	done += 1 + SEGSZ/64 + free_seg(tb, 1);
 
 	/*
 	 * If we have done enough work, get out here.
 	 */
-	if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) {
+	if (done >= DELETE_RECORD_LIMIT) {
 	    return 0;	/* Not done */
 	}
     }
@@ -2871,7 +2872,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
                 int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
                 int nactive = NACTIVE(tb);
 
-                if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) {
+                if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
                     grow(tb, nactive);
                 }
             }
-- 
cgit v1.2.3


From f4bdac18cb9dd45185e911308a5ebd95ff10d7fd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 1 Jun 2016 16:19:04 +0200
Subject: erts: Remove unnecessary access of 'is_resizing'

in tables without write_concurrency
and remove it totally #ifndef ERTS_SMP
---
 erts/emulator/beam/erl_db_hash.c | 21 ++++++++++-----------
 erts/emulator/beam/erl_db_hash.h |  2 +-
 2 files changed, 11 insertions(+), 12 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 4c4ff9b470..074ac6d64e 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -671,8 +671,8 @@ int db_create_hash(Process *p, DbTable *tbl)
     tb->nsegs = NSEG_1;
     tb->nslots = SEGSZ;
 
-    erts_smp_atomic_init_nob(&tb->is_resizing, 0);
 #ifdef ERTS_SMP
+    erts_smp_atomic_init_nob(&tb->is_resizing, 0);
     if (tb->common.type & DB_FINE_LOCKED) {
 	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	int i;
@@ -2605,23 +2605,22 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
 static ERTS_INLINE int
 begin_resizing(DbTableHash* tb)
 {
+#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
-	return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1);
-    else {
-	if (erts_smp_atomic_read_nob(&tb->is_resizing))
-	    return 0;
-	erts_smp_atomic_set_nob(&tb->is_resizing, 1);
-	return 1;
-    }
+	return !erts_atomic_xchg_acqb(&tb->is_resizing, 1);
+    else
+        ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
+#endif
+    return 1;
 }
 
 static ERTS_INLINE void
 done_resizing(DbTableHash* tb)
 {
+#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
-	erts_smp_atomic_set_relb(&tb->is_resizing, 0);
-    else
-	erts_smp_atomic_set_nob(&tb->is_resizing, 0);
+	erts_atomic_set_relb(&tb->is_resizing, 0);
+#endif
 }
 
 /* Grow table with one new bucket.
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index e654363cd5..081ff8fafc 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -60,8 +60,8 @@ typedef struct db_table_hash {
     /* List of slots where elements have been deleted while table was fixed */
     erts_smp_atomic_t fixdel;  /* (FixedDeletion*) */	
     erts_smp_atomic_t nactive; /* Number of "active" slots */
-    erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
 #ifdef ERTS_SMP
+    erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
     DbTableHashFineLocks* locks;
 #endif
 #ifdef VALGRIND
-- 
cgit v1.2.3


From 58d52b1d2d3e9f14bda5b9248d9b25ae12535548 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 1 Jun 2016 16:22:09 +0200
Subject: erts: Avoid literals in process independent ErlNifEnv

that would go undetected and cause havoc if module is purged.
---
 erts/emulator/beam/erl_nif.c | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f1c35cd5a5..4fd82bad10 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -803,19 +803,13 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
 {
     Uint sz;
     Eterm* hp;
-#ifdef SHCOPY
-    erts_shcopy_t info;
-    INITIALIZE_SHCOPY(info);
-    sz = copy_shared_calculate(src_term, &info);
-    hp = alloc_heap(dst_env, sz);
-    src_term = copy_shared_perform(src_term, sz, &info, &hp, &MSO(dst_env->proc));
-    DESTROY_SHCOPY(info);
-    return src_term;
-#else
+    /*
+     * No preserved sharing allowed as long as literals are also preserved.
+     * Process independent environment can not be reached by purge.
+     */
     sz = size_object(src_term);
     hp = alloc_heap(dst_env, sz);
     return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc));
-#endif
 }
 
 
-- 
cgit v1.2.3


From e25df0378738ba17fb66ae0bd947c439ac925800 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 2 Jun 2016 12:28:12 +0200
Subject: Fix erts_dirty_process_main()

---
 erts/emulator/beam/beam_emu.c | 77 ++++++++++++++++++++-----------------------
 1 file changed, 36 insertions(+), 41 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0a59f8785c..4716460a6b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5318,10 +5318,14 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
 
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
-	    && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
-	    c_p->fcalls = REDS_IN(c_p) = 0;
-	}
+	/*
+	 * Set fcalls even though we ignore it, so we don't
+	 * confuse code accessing it...
+	 */
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	    c_p->fcalls = 0;
+	else
+	    c_p->fcalls = CONTEXT_REDS;
 
 	SWAPIN;
 
@@ -5341,7 +5345,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
                                       NULL, fun_buf);
                 } else {
                     erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
-                                  "<unknown/%p>", next);
+                                  "<unknown/%p>", *I);
                 }
             }
 
@@ -5351,8 +5355,10 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
     }
 
     {
-	Eterm nif_bif_result;
-	Eterm bif_nif_arity;
+#ifdef DEBUG
+	Eterm result;
+#endif
+	Eterm arity;
 
 	{
 	    /*
@@ -5374,7 +5380,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	    c_p->current = I-3; /* current and vbf set to please handle_error */
 	    SWAPOUT;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    bif_nif_arity = I[-1];
+	    arity = I[-1];
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -5383,50 +5389,39 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
 		ASSERT(!c_p->scheduler_data);
-		erts_pre_dirty_nif(esdp, &env, c_p, (struct erl_module_nif*)I[2], NULL);
-		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
-		if (env.exception_thrown)
-		    nif_bif_result = THE_NON_VALUE;
+
+		erts_pre_dirty_nif(esdp, &env, c_p,
+				   (struct erl_module_nif*)I[2], NULL);
+
+#ifdef DEBUG
+		result =
+#else
+		(void)
+#endif
+		    (*fp)(&env, arity, reg);
+
 		erts_post_nif(&env);
+
+		ASSERT(!is_value(result));
+		ASSERT(c_p->freason == TRAP);
+		ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+
 		PROCESS_MAIN_CHK_LOCKS(c_p);
+		ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-		if (env.exiting) {
-		    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+		if (env.exiting)
 		    goto do_dirty_schedule;
-		}
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	    }
+
 	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
-	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_HOLE_CHECK(c_p);
-	    if (ERTS_IS_GC_DESIRED(c_p)) {
-		nif_bif_result = erts_gc_after_bif_call(c_p,
-							nif_bif_result,
-							reg, bif_nif_arity);
-	    }
-	    SWAPIN;  /* There might have been a garbage collection. */
-	    if (is_value(nif_bif_result)) {
-		r(0) = nif_bif_result;
-		CHECK_TERM(r(0));
-		I = c_p->cp;
-		c_p->cp = 0;
-		Goto(*I);
-	    } else if (c_p->freason == TRAP) {
-		I = c_p->i;
-		ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
-		goto context_switch;
-	    }
-	    I = handle_error(c_p, c_p->cp, reg, vbf);
+	    SWAPIN;
+	    I = c_p->i;
+	    goto context_switch;
 	}
     }
-    if (I == 0) {
-	goto do_dirty_schedule;
-    } else {
-	ASSERT(!is_value(r(0)));
-	SWAPIN;
-	goto context_switch;
-    }
 #endif /* ERTS_DIRTY_SCHEDULERS */
 }
 
-- 
cgit v1.2.3


From 3a5c3a5c1b0eb756a2258aea87f3bacb89199064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 7 Jun 2016 14:50:50 +0200
Subject: erts: Remove tautological compare warning

---
 erts/emulator/beam/erl_bif_port.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 37f4e1de49..fefa9d8391 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1411,7 +1411,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
                     trunc_len = val;
                     goto next_option;
                 case am_line_delimiter:
-                    if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) {
+                    if (type == TCP_PB_LINE_LF && val <= 255) {
                         delimiter = (char)val;
                         goto next_option;
                     }
-- 
cgit v1.2.3


From 4c6ccf1c2f13d4b6ea34bc0eecc8a772cb38e80a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 7 Jun 2016 15:29:11 +0200
Subject: erts: Let clang have suppressable unused variables

---
 erts/emulator/beam/sys.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 9a205d50d3..dfe82cab44 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -154,8 +154,9 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #  define ERTS_WRITE_UNLIKELY(X) X
 #endif
 
+/* clang may have too low __GNUC__ versions but can handle it */
 #ifdef __GNUC__
-#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5) || defined(__clang__)
 #    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
 #  else
 #    define ERTS_DECLARE_DUMMY(X) X
-- 
cgit v1.2.3


From 1a26eefb07de7d6152543b252507d618ed844368 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 8 Jun 2016 14:38:27 +0200
Subject: erts: Change LTTng otp domain

From 'com_ericsson_otp' to 'org_erlang_otp'.
This domain name is more suitable.
---
 erts/emulator/beam/erlang_lttng.h  | 48 +++++++++++++++++++-------------------
 erts/emulator/beam/lttng-wrapper.h | 12 +++++-----
 2 files changed, 30 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index 12f68e477b..4e869671f7 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -20,7 +20,7 @@
 
 #ifdef USE_LTTNG
 #undef TRACEPOINT_PROVIDER
-#define TRACEPOINT_PROVIDER com_ericsson_otp
+#define TRACEPOINT_PROVIDER org_erlang_otp
 
 #undef TRACEPOINT_INCLUDE
 #define TRACEPOINT_INCLUDE "erlang_lttng.h"
@@ -33,7 +33,7 @@
 /* Schedulers */
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     scheduler_poll,
     TP_ARGS(
         int, id,
@@ -62,7 +62,7 @@ typedef struct {
 /* Port and Driver Scheduling */
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_start,
     TP_ARGS(
         char*, pid,
@@ -77,7 +77,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_init,
     TP_ARGS(
         char*, driver,
@@ -94,7 +94,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_outputv,
     TP_ARGS(
         char*, pid,
@@ -111,7 +111,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_output,
     TP_ARGS(
         char*, pid,
@@ -128,7 +128,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_ready_input,
     TP_ARGS(
         char*, pid,
@@ -143,7 +143,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_ready_output,
     TP_ARGS(
         char*, pid,
@@ -158,7 +158,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_event,
     TP_ARGS(
         char*, pid,
@@ -173,7 +173,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_timeout,
     TP_ARGS(
         char*, pid,
@@ -188,7 +188,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_stop_select,
     TP_ARGS(
         char*, driver
@@ -199,7 +199,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_flush,
     TP_ARGS(
         char*, pid,
@@ -214,7 +214,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_stop,
     TP_ARGS(
         char*, pid,
@@ -229,7 +229,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_process_exit,
     TP_ARGS(
         char*, pid,
@@ -244,7 +244,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_ready_async,
     TP_ARGS(
         char*, pid,
@@ -259,7 +259,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_finish,
     TP_ARGS(
         char*, driver
@@ -270,7 +270,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_call,
     TP_ARGS(
         char*, pid,
@@ -289,7 +289,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     driver_control,
     TP_ARGS(
         char*, pid,
@@ -310,7 +310,7 @@ TRACEPOINT_EVENT(
 /* Async pool */
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     aio_pool_get,
     TP_ARGS(
         char*, port,
@@ -323,7 +323,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     aio_pool_put,
     TP_ARGS(
         char*, port,
@@ -339,7 +339,7 @@ TRACEPOINT_EVENT(
 /* Memory Allocator */
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     carrier_create,
     TP_ARGS(
         const char*, type,
@@ -365,7 +365,7 @@ TRACEPOINT_EVENT(
 
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     carrier_destroy,
     TP_ARGS(
         const char*, type,
@@ -390,7 +390,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     carrier_pool_put,
     TP_ARGS(
         const char*, name,
@@ -405,7 +405,7 @@ TRACEPOINT_EVENT(
 )
 
 TRACEPOINT_EVENT(
-    com_ericsson_otp,
+    org_erlang_otp,
     carrier_pool_get,
     TP_ARGS(
         const char*, name,
diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h
index 294872c365..0bc75c1552 100644
--- a/erts/emulator/beam/lttng-wrapper.h
+++ b/erts/emulator/beam/lttng-wrapper.h
@@ -77,23 +77,23 @@
     (RQ)->scheduler->no
 
 #define LTTNG_ENABLED(Name) \
-    tracepoint_enabled(com_ericsson_otp, Name)
+    tracepoint_enabled(org_erlang_otp, Name)
 
 /* include a special LTTNG_DO for do_tracepoint ? */
 #define LTTNG1(Name, Arg1) \
-    tracepoint(com_ericsson_otp, Name, (Arg1))
+    tracepoint(org_erlang_otp, Name, (Arg1))
 
 #define LTTNG2(Name, Arg1, Arg2) \
-    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2))
+    tracepoint(org_erlang_otp, Name, (Arg1), (Arg2))
 
 #define LTTNG3(Name, Arg1, Arg2, Arg3) \
-    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3))
+    tracepoint(org_erlang_otp, Name, (Arg1), (Arg2), (Arg3))
 
 #define LTTNG4(Name, Arg1, Arg2, Arg3, Arg4) \
-    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4))
+    tracepoint(org_erlang_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4))
 
 #define LTTNG5(Name, Arg1, Arg2, Arg3, Arg4, Arg5) \
-    tracepoint(com_ericsson_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4), (Arg5))
+    tracepoint(org_erlang_otp, Name, (Arg1), (Arg2), (Arg3), (Arg4), (Arg5))
 
 #else /* USE_LTTNG */
 
-- 
cgit v1.2.3


From 049611f98d67f11f8e06fe0b0fc50eb8ca1925d8 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 8 Jun 2016 14:56:31 +0200
Subject: No GC on dirty IO schedulers

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index c0b1d7246c..5193be85b4 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10082,7 +10082,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    }
 	}
 
-	if (ERTS_IS_GC_DESIRED(p)) {
+	if (ERTS_IS_GC_DESIRED(p) && !ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) {
 	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
 		int cost = scheduler_gc_proc(p, reds);
 		calls += cost;
-- 
cgit v1.2.3


From 77192f7f32d4bbe293a6e8cb9da79b8a6dd6b181 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 8 Jun 2016 14:58:33 +0200
Subject: Replace enif_is_on_dirty_scheduler() with enif_thread_type()

---
 erts/emulator/beam/erl_nif.c           | 19 +++++++++++--------
 erts/emulator/beam/erl_nif.h           | 11 +++++++++++
 erts/emulator/beam/erl_nif_api_funcs.h |  4 ++--
 3 files changed, 24 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4fd82bad10..039f97ef43 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2658,18 +2658,21 @@ done:
 }
 
 int
-enif_is_on_dirty_scheduler(ErlNifEnv* env)
+enif_thread_type(void)
 {
-    int scheduler;
-    Process *c_p;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
-    execution_state(env, &c_p, &scheduler);
+    if (!esdp)
+	return ERL_NIF_THR_UNDEFINED;
+
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+	return ERL_NIF_THR_NORMAL_SCHEDULER;
 
-    if (!c_p || !scheduler)
-	erts_exit(ERTS_ABORT_EXIT, "enif_is_on_dirty_scheduler: "
-		  "Invalid env");
+    if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp))
+	return ERL_NIF_THR_DIRTY_CPU_SCHEDULER;
 
-    return scheduler < 0;
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(esdp));
+    return ERL_NIF_THR_DIRTY_IO_SCHEDULER;
 }
 
 /* Maps */
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index da7a754757..494971e118 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -209,6 +209,17 @@ typedef enum {
     ERL_NIF_BIN2TERM_SAFE = 0x20000000
 } ErlNifBinaryToTerm;
 
+/*
+ * Return values from enif_thread_type(). Negative values
+ * reserved for specific types of non-scheduler threads.
+ * Positive values reserved for scheduler thread types.
+ */
+
+#define ERL_NIF_THR_UNDEFINED 0
+#define ERL_NIF_THR_NORMAL_SCHEDULER 1
+#define ERL_NIF_THR_DIRTY_CPU_SCHEDULER 2
+#define ERL_NIF_THR_DIRTY_IO_SCHEDULER 3
+
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
 typedef struct {
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index b211ab4b16..9a8f216773 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -173,7 +173,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, E
 ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin));
 ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsigned char* data, size_t sz, ERL_NIF_TERM *term, unsigned int opts));
 ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg));
-ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
+ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void));
 ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...));
 
 /*
@@ -330,7 +330,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char
 #  define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary)
 #  define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term)
 #  define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command)
-#  define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler)
+#  define enif_thread_type ERL_NIF_API_FUNC_MACRO(enif_thread_type)
 #  define enif_snprintf ERL_NIF_API_FUNC_MACRO(enif_snprintf)
 
 /*
-- 
cgit v1.2.3


From 0f072d17a3ec6d793e7bec0787e8af25bca06dfb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 8 Jun 2016 15:22:01 +0200
Subject: Revert "erts: Remove unnecessary access of 'is_resizing'"

This reverts commit f4bdac18cb9dd45185e911308a5ebd95ff10d7fd.
---
 erts/emulator/beam/erl_db_hash.c | 21 +++++++++++----------
 erts/emulator/beam/erl_db_hash.h |  2 +-
 2 files changed, 12 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 074ac6d64e..4c4ff9b470 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -671,8 +671,8 @@ int db_create_hash(Process *p, DbTable *tbl)
     tb->nsegs = NSEG_1;
     tb->nslots = SEGSZ;
 
-#ifdef ERTS_SMP
     erts_smp_atomic_init_nob(&tb->is_resizing, 0);
+#ifdef ERTS_SMP
     if (tb->common.type & DB_FINE_LOCKED) {
 	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	int i;
@@ -2605,22 +2605,23 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
 static ERTS_INLINE int
 begin_resizing(DbTableHash* tb)
 {
-#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
-	return !erts_atomic_xchg_acqb(&tb->is_resizing, 1);
-    else
-        ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
-#endif
-    return 1;
+	return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1);
+    else {
+	if (erts_smp_atomic_read_nob(&tb->is_resizing))
+	    return 0;
+	erts_smp_atomic_set_nob(&tb->is_resizing, 1);
+	return 1;
+    }
 }
 
 static ERTS_INLINE void
 done_resizing(DbTableHash* tb)
 {
-#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
-	erts_atomic_set_relb(&tb->is_resizing, 0);
-#endif
+	erts_smp_atomic_set_relb(&tb->is_resizing, 0);
+    else
+	erts_smp_atomic_set_nob(&tb->is_resizing, 0);
 }
 
 /* Grow table with one new bucket.
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 081ff8fafc..e654363cd5 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -60,8 +60,8 @@ typedef struct db_table_hash {
     /* List of slots where elements have been deleted while table was fixed */
     erts_smp_atomic_t fixdel;  /* (FixedDeletion*) */	
     erts_smp_atomic_t nactive; /* Number of "active" slots */
-#ifdef ERTS_SMP
     erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
+#ifdef ERTS_SMP
     DbTableHashFineLocks* locks;
 #endif
 #ifdef VALGRIND
-- 
cgit v1.2.3


From 6bf6e2cfd4dbe48e52456d4111f9ffb52e991b6c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 8 Jun 2016 15:22:16 +0200
Subject: Revert "erts: Change ETS hash load factor"

This reverts commit 7c133fb1094ad1cabbb5cfc157483a43c816c6a9.
---
 erts/emulator/beam/erl_db_hash.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 4c4ff9b470..74979f984a 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -95,8 +95,7 @@
 /* 
  * The following symbols can be manipulated to "tune" the linear hash array 
  */
-#define GROW_LIMIT(NACTIVE) ((NACTIVE)*2)
-#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2)
+#define CHAIN_LEN 6                 /* Medium bucket chain len      */
 
 /* Number of slots per segment */
 #define SEGSZ_EXP  8
@@ -464,7 +463,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
 static ERTS_INLINE void try_shrink(DbTableHash* tb)
 {
     int nactive = NACTIVE(tb);
-    if (nactive > SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive)
+    if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN)
 	&& !IS_FIXED(tb)) {
 	shrink(tb, nactive);
     }
@@ -863,7 +862,7 @@ Lnew:
     WUNLOCK_HASH(lck);
     {
 	int nactive = NACTIVE(tb);       
-	if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
+	if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) {
 	    grow(tb, nactive);
 	}
     }
@@ -2251,12 +2250,12 @@ static int db_free_table_continue_hash(DbTable *tbl)
 
     done /= 2;
     while(tb->nslots != 0) {
-	done += 1 + SEGSZ/64 + free_seg(tb, 1);
+	free_seg(tb, 1);
 
 	/*
 	 * If we have done enough work, get out here.
 	 */
-	if (done >= DELETE_RECORD_LIMIT) {
+	if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) {
 	    return 0;	/* Not done */
 	}
     }
@@ -2872,7 +2871,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
                 int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
                 int nactive = NACTIVE(tb);
 
-                if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
+                if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) {
                     grow(tb, nactive);
                 }
             }
-- 
cgit v1.2.3


From 99655be9a918671b56846ff1731196c0d467f4cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 8 Jun 2016 18:58:21 +0200
Subject: erts: Don't crash on maps on crash dumps

- Large Maps could cause a stack overrun during crash dump generation.
- This is a simple workaround until a solution has been implemented.
- The error has no impact on a running system.
---
 erts/emulator/beam/erl_process_dump.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index eeaa9a569c..a70dfb8e73 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -560,6 +560,11 @@ dump_externally(int to, void *to_arg, Eterm term)
 	}
     }
 
+    /* Do not handle maps */
+    if (is_map(term)) {
+        term = am_undefined;
+    }
+
     s = p = sbuf;
     erts_encode_ext(term, &p);
     erts_print(to, to_arg, "E%X:", p-s);
-- 
cgit v1.2.3


From 3ee5343415d6ae0ce1ff1c2a2555051431a9315e Mon Sep 17 00:00:00 2001
From: Dmytro Lytovchenko <dmytro.lytovchenko@gmail.com>
Date: Wed, 25 May 2016 14:37:03 +0200
Subject: erts: Add port monitors

* erlang:monitor/2 with port argument is added, erlang:demonitor, using port task API and avoiding locking;
* port_info and process_info support for monitored ports (with named port monitors support);
* Exit signals contain type 'process' or 'port';
* Propagation of port exit signals;
* Self-cleaning when origin process dies with monitor on;
* 8 test cases + testcase for port driver crashing;
* Documentation for all of the above (monitor, demonitor, port_info and process_info) updated
---
 erts/emulator/beam/bif.c          | 410 +++++++++++++++++++++++++-------------
 erts/emulator/beam/erl_bif_info.c | 100 +++++++---
 erts/emulator/beam/erl_bif_port.c |   6 +
 erts/emulator/beam/erl_port.h     |  58 +++++-
 erts/emulator/beam/erl_process.c  |  21 +-
 erts/emulator/beam/io.c           | 390 +++++++++++++++++++++++++++++++-----
 erts/emulator/beam/register.c     |  51 ++---
 erts/emulator/beam/register.h     |   2 +-
 8 files changed, 783 insertions(+), 255 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index b18910e2c7..fc14061a44 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -282,20 +282,17 @@ res_no_proc: {
     }
 }
 
-#define ERTS_DEMONITOR_FALSE		2
-#define ERTS_DEMONITOR_TRUE		1
-#define ERTS_DEMONITOR_BADARG		0
-#define ERTS_DEMONITOR_YIELD_TRUE	-1
-#define ERTS_DEMONITOR_INTERNAL_ERROR	-2
-
-static int
+/* This function is allowed to return range of values handled by demonitor/1-2
+ * Namely: atoms true, false, yield, internal_error, badarg or THE_NON_VALUE
+ */
+static Eterm
 remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 {
     ErtsDSigData dsd;
     ErtsMonitor *dmon;
     ErtsMonitor *mon;
     int code;
-    int res;
+    Eterm res = am_false;
 #ifndef ERTS_SMP
     int stale_mon = 0;
 #endif
@@ -328,7 +325,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 	mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
 
-	res = ERTS_DEMONITOR_TRUE;
+        res = am_true;
 	break;
 
     case ERTS_DSIG_PREP_CONNECTED:
@@ -352,7 +349,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 	     * This is possible when smp support is enabled.
 	     * 'DOWN' message just arrived.
 	     */
-	    res = ERTS_DEMONITOR_TRUE;
+            res = am_true;
 	}
 	else {
 	    /*
@@ -367,16 +364,13 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 					     : mon->pid), 
 					    ref,
 					    0);
-	    res = (code == ERTS_DSIG_SEND_YIELD
-		   ? ERTS_DEMONITOR_YIELD_TRUE
-		   : ERTS_DEMONITOR_TRUE);
+            res = (code == ERTS_DSIG_SEND_YIELD ? am_yield : am_true);
 	    erts_destroy_monitor(dmon);
-
 	}
 	break;
     default:
 	ASSERT(! "Invalid dsig prepare result");
-	return ERTS_DEMONITOR_INTERNAL_ERROR;
+        return am_internal_error;
     }
 
 #ifndef ERTS_SMP
@@ -404,27 +398,96 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
     return res;
 }
 
-static int demonitor(Process *c_p, Eterm ref, Eterm *multip)
+static ERTS_INLINE void
+demonitor_local_process(Process *c_p, Eterm ref, Eterm to, Eterm *res)
+{
+    Process *rp = erts_pid2proc_opt(c_p,
+                           ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
+                           to,
+                           ERTS_PROC_LOCK_LINK,
+                           ERTS_P2P_FLG_ALLOW_OTHER_X);
+    ErtsMonitor *mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
+
+#ifndef ERTS_SMP
+    ASSERT(mon);
+#else
+    if (!mon)
+        *res = am_false;
+    else
+#endif
+    {
+        *res = am_true;
+        erts_destroy_monitor(mon);
+    }
+    if (rp) {
+        ErtsMonitor *rmon;
+        rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
+        if (rp != c_p)
+            erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        if (rmon != NULL)
+            erts_destroy_monitor(rmon);
+    }
+    else {
+        ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p);
+    }
+}
+
+static ERTS_INLINE BIF_RETTYPE
+demonitor_local_port(Process *origin, Eterm ref, Eterm target)
 {
-   ErtsMonitor *mon = NULL;  /* The monitor entry to delete */
-   Process  *rp;    /* Local target process */
-   Eterm     to = NIL;    /* Monitor link traget */
-   DistEntry *dep = NULL;  /* Target's distribution entry */
-   int deref_de = 0;
-   int res;
-   int unlock_link = 1;
+    BIF_RETTYPE res = am_false;
+    Port        *port = erts_port_lookup_raw(target);
+
+    if (!port) {
+        BIF_ERROR(origin, BADARG);
+    }
+    erts_smp_proc_unlock(origin, ERTS_PROC_LOCK_LINK);
+
+    if (port) {
+        Eterm trap_ref;
+        switch (erts_port_demonitor(origin, ERTS_PORT_DEMONITOR_NORMAL,
+                                    port, ref, &trap_ref)) {
+        case ERTS_PORT_OP_DROPPED:
+        case ERTS_PORT_OP_BADARG:
+            break;
+        case ERTS_PORT_OP_SCHEDULED:
+            BIF_TRAP3(await_port_send_result_trap, origin, trap_ref,
+                      am_busy_port, am_true);
+            /* the busy_port atom will never be returned, because it cannot be
+             * returned from erts_port_(de)monitor, but just in case if in future
+             * internal API changes - you may see this atom */
+        default:
+            break;
+        }
+    }
+    else {
+        ERTS_SMP_ASSERT_IS_NOT_EXITING(origin);
+    }
+    BIF_RET(res);
+}
 
+/* Can return atom true, false, yield, internal_error, badarg or
+ * THE_NON_VALUE if error occured or trap has been set up
+ */
+static
+BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
+{
+   ErtsMonitor  *mon = NULL;  /* The monitor entry to delete */
+   Eterm        to = NIL;     /* Monitor link traget */
+   DistEntry    *dep = NULL;  /* Target's distribution entry */
+   int          deref_de = 0;
+   BIF_RETTYPE  res = am_false;
+   int          unlock_link = 1;
 
    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_LINK);
 
    if (is_not_internal_ref(ref)) {
-       res = ERTS_DEMONITOR_BADARG;
+       res = am_badarg;
        goto done; /* Cannot be this monitor's ref */
    }
 
    mon = erts_lookup_monitor(ERTS_P_MONITORS(c_p), ref);
    if (!mon) {
-       res = ERTS_DEMONITOR_FALSE;
        goto done;
    }
 
@@ -432,70 +495,50 @@ static int demonitor(Process *c_p, Eterm ref, Eterm *multip)
    case MON_TIME_OFFSET:
        *multip = am_true;
        erts_demonitor_time_offset(ref);
-       res = ERTS_DEMONITOR_TRUE;
+       res = am_true;
        break;
    case MON_ORIGIN:
        to = mon->pid;
        *multip = am_false;
        if (is_atom(to)) {
-	   /* Monitoring a name at node to */
-	   ASSERT(is_node_name_atom(to));
-	   dep = erts_sysname_to_connected_dist_entry(to);
-	   ASSERT(dep != erts_this_dist_entry);
-	   if (dep)
-	       deref_de = 1;
+           /* Monitoring a name at node to */
+           ASSERT(is_node_name_atom(to));
+           dep = erts_sysname_to_connected_dist_entry(to);
+           ASSERT(dep != erts_this_dist_entry);
+           if (dep)
+               deref_de = 1;
+       } else if (is_port(to)) {
+           if (port_dist_entry(to) != erts_this_dist_entry) {
+               goto badarg;
+           }
+           res = demonitor_local_port(c_p, ref, to);
+           unlock_link = 0;
+           goto done;
        } else {
-	   ASSERT(is_pid(to));
-	   dep = pid_dist_entry(to);
+           ASSERT(is_pid(to));
+           dep = pid_dist_entry(to);
        }
        if (dep != erts_this_dist_entry) {
-	   res = remote_demonitor(c_p, dep, ref, to);
-	   /* remote_demonitor() unlocks link lock on c_p */
-	   unlock_link = 0;
+           res = remote_demonitor(c_p, dep, ref, to);
+           /* remote_demonitor() unlocks link lock on c_p */
+           unlock_link = 0;
        }
        else { /* Local monitor */
-	   if (deref_de) {
-	       deref_de = 0;
-	       erts_deref_dist_entry(dep);
-	   }
-	   dep = NULL;
-	   rp = erts_pid2proc_opt(c_p,
-				  ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
-				  to,
-				  ERTS_PROC_LOCK_LINK,
-				  ERTS_P2P_FLG_ALLOW_OTHER_X);
-	   mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
-#ifndef ERTS_SMP
-	   ASSERT(mon);
-#else
-	   if (!mon)
-	       res = ERTS_DEMONITOR_FALSE;
-	   else
-#endif
-	   {
-	       res = ERTS_DEMONITOR_TRUE;
-	       erts_destroy_monitor(mon);
-	   }
-	   if (rp) {
-	       ErtsMonitor *rmon;
-	       rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-	       if (rp != c_p)
-		   erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	       if (rmon != NULL)
-		   erts_destroy_monitor(rmon);
-	   }
-	   else {
-	       ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p);
-	   }
-
+           if (deref_de) {
+               deref_de = 0;
+               erts_deref_dist_entry(dep);
+           }
+           dep = NULL;
+           demonitor_local_process(c_p, ref, to, &res);
        }
        break;
-   default:
-       res = ERTS_DEMONITOR_BADARG;
+    default /* case */ :
+badarg:
+       res = am_badarg; /* will be converted to error by caller */
        *multip = am_false;
        break;
    }
- done:
+done:
 
    if (unlock_link)
        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
@@ -506,21 +549,20 @@ static int demonitor(Process *c_p, Eterm ref, Eterm *multip)
    }
 
    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
-   return res;
+   BIF_RET(res);
 }
 
 BIF_RETTYPE demonitor_1(BIF_ALIST_1)
 {
     Eterm multi;
     switch (demonitor(BIF_P, BIF_ARG_1, &multi)) {
-    case ERTS_DEMONITOR_FALSE:
-    case ERTS_DEMONITOR_TRUE:
-	BIF_RET(am_true);
-    case ERTS_DEMONITOR_YIELD_TRUE:
-	ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
-    case ERTS_DEMONITOR_BADARG:
-	BIF_ERROR(BIF_P, BADARG);
-    case ERTS_DEMONITOR_INTERNAL_ERROR:
+    case am_false:
+    case am_true:       BIF_RET(am_true);
+    case THE_NON_VALUE: BIF_RET(THE_NON_VALUE);
+    case am_yield:      ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+    case am_badarg:     BIF_ERROR(BIF_P, BADARG);
+
+    case am_internal_error:
     default:
 	ASSERT(! "demonitor(): internal error");
 	BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
@@ -529,11 +571,11 @@ BIF_RETTYPE demonitor_1(BIF_ALIST_1)
 
 BIF_RETTYPE demonitor_2(BIF_ALIST_2)
 {
-    Eterm res = am_true;
-    Eterm multi = am_false;
-    int info = 0;
-    int flush = 0;
-    Eterm list = BIF_ARG_2;
+    BIF_RETTYPE res = am_true;
+    Eterm       multi = am_false;
+    int         info = 0;
+    int         flush = 0;
+    Eterm       list = BIF_ARG_2;
 
     while (is_list(list)) {
 	Eterm* consp = list_val(list);
@@ -554,24 +596,27 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2)
 	goto badarg;
 
     switch (demonitor(BIF_P, BIF_ARG_1, &multi)) {
-    case ERTS_DEMONITOR_FALSE:
+    case THE_NON_VALUE:
+        /* If other error occured or trap has been set up - pass through */
+        BIF_RET(THE_NON_VALUE);
+    case am_false:
 	if (info)
 	    res = am_false;
 	if (flush) {
-	flush_messages:
+flush_messages:
 	    BIF_TRAP3(flush_monitor_messages_trap, BIF_P,
 		      BIF_ARG_1, multi, res);
 	}
-    case ERTS_DEMONITOR_TRUE:
+    case am_true:
 	if (multi == am_true && flush)
 	    goto flush_messages;
 	BIF_RET(res);
-    case ERTS_DEMONITOR_YIELD_TRUE:
+    case am_yield:
 	ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
-    case ERTS_DEMONITOR_BADARG:
-    badarg:
+    case am_badarg:
+badarg:
 	BIF_ERROR(BIF_P, BADARG);
-    case ERTS_DEMONITOR_INTERNAL_ERROR:
+    case am_internal_error:
     default:
 	ASSERT(! "demonitor(): internal error");
 	BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
@@ -615,14 +660,13 @@ erts_queue_monitor_message(Process *p,
     erts_queue_message(p, *p_locksp, msgp, tup, am_system);
 }
 
-static BIF_RETTYPE
+static Eterm
 local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean)
 {
-    BIF_RETTYPE ret;
-    Process *rp;
+    Eterm         ret = mon_ref;
+    Process      *rp;
     ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK;
 
-    ERTS_BIF_PREP_RET(ret, mon_ref);
     if (target == p->common.id) {
 	return ret;
     }
@@ -658,40 +702,112 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean)
 }
 
 static BIF_RETTYPE
-local_name_monitor(Process *p, Eterm target_name)
+local_port_monitor(Process *origin, Eterm target)
 {
-    BIF_RETTYPE ret;
-    Eterm mon_ref;
-    ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK;
-    Process *rp;
+    BIF_RETTYPE ref = erts_make_ref(origin);
+    Port *port = erts_sig_lookup_port(origin, target);
+    ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN;
 
-    mon_ref = erts_make_ref(p);
-    ERTS_BIF_PREP_RET(ret, mon_ref);
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK);
-    rp = erts_whereis_process(p, p_locks, target_name, ERTS_PROC_LOCK_LINK,
-			      ERTS_P2P_FLG_ALLOW_OTHER_X);
-    if (!rp) {
-	DeclareTmpHeap(lhp,3,p);
+    if (!port) {
+res_no_proc:
+        /* Send the DOWN message immediately. Ref is made on the fly because
+         * caller has never seen it yet. */
+        erts_queue_monitor_message(origin, &p_locks, ref,
+                                   am_port, target, am_noproc);
+    }
+    else {
+        switch (erts_port_monitor(origin, port, target, &ref)) {
+        case ERTS_PORT_OP_DROPPED:
+        case ERTS_PORT_OP_BADARG:
+            goto res_no_proc;
+        case ERTS_PORT_OP_SCHEDULED:
+            BIF_TRAP3(await_port_send_result_trap, origin, ref,
+                      am_busy_port, ref);
+            /* the busy_port atom will never be returned, because it cannot be
+             * returned from erts_port_monitor, but just in case if in future
+             * internal API changes - you may see this atom */
+        default:
+            break;
+        }
+    }
+    erts_smp_proc_unlock(origin, p_locks & ~ERTS_PROC_LOCK_MAIN);
+    BIF_RET(ref);
+}
+
+/* Type = process | port :: atom(), 1st argument passed to erlang:monitor/2
+ */
+static BIF_RETTYPE
+local_name_monitor(Process *self, Eterm type, Eterm target_name)
+{
+    BIF_RETTYPE ret = erts_make_ref(self);
+
+    ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK;
+    Process     *proc = NULL;
+    Port        *port = NULL;
+
+    erts_smp_proc_lock(self, ERTS_PROC_LOCK_LINK);
+
+    erts_whereis_name(self, p_locks, target_name,
+                      &proc, ERTS_PROC_LOCK_LINK,
+                      ERTS_P2P_FLG_ALLOW_OTHER_X,
+                      &port, 0);
+
+    /* If the name is not registered,
+     * or if we asked for proc and got a port,
+     * or if we asked for port and got a proc,
+     * we just send the 'DOWN' message.
+     */
+    if ((!proc && !port) ||
+        (type == am_process && port) ||
+        (type == am_port && proc)) {
+        DeclareTmpHeap(lhp,3,self);
 	Eterm item;
-	UseTmpHeap(3,p);
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+        UseTmpHeap(3,self);
+
+        erts_smp_proc_unlock(self, ERTS_PROC_LOCK_LINK);
 	p_locks &= ~ERTS_PROC_LOCK_LINK;
+
 	item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname);
-	erts_queue_monitor_message(p, &p_locks,
-				   mon_ref, am_process, item, am_noproc);
-	UnUseTmpHeap(3,p);
-    }
-    else if (rp != p) {
-	erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, rp->common.id,
-			 target_name);
-	erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id,
-			 target_name);
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        erts_queue_monitor_message(self, &p_locks,
+                                   ret,
+                                   type, /* = process|port :: atom() */
+                                   item, am_noproc);
+        UnUseTmpHeap(3,self);
+    }
+    else if (port) {
+        erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
+        p_locks &= ~ERTS_PROC_LOCK_MAIN;
+
+        switch (erts_port_monitor(self, port, target_name, &ret)) {
+        case ERTS_PORT_OP_DONE:
+            return ret;
+        case ERTS_PORT_OP_SCHEDULED: { /* Scheduled a signal */
+            ASSERT(is_internal_ref(ret));
+            BIF_TRAP3(await_port_send_result_trap, self,
+                      ret, am_true, ret);
+            /* bif_trap returns */
+        } break;
+        default:
+            goto badarg;
+        }
+    }
+    else if (proc != self) {
+        erts_add_monitor(&ERTS_P_MONITORS(self), MON_ORIGIN, ret,
+                         proc->common.id, target_name);
+        erts_add_monitor(&ERTS_P_MONITORS(proc), MON_TARGET, ret,
+                         self->common.id, target_name);
+        erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_LINK);
     }
 
-    erts_smp_proc_unlock(p, p_locks & ~ERTS_PROC_LOCK_MAIN);
-
-    return ret;
+    if (p_locks) {
+        erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
+    }
+    BIF_RET(ret);
+badarg:
+    if (p_locks) {
+        erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
+    }
+    BIF_ERROR(self, BADARG);
 }
 
 static BIF_RETTYPE
@@ -758,7 +874,7 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
 	break;
     }
 
-    return ret;
+    BIF_RET(ret);
 }
 	
 BIF_RETTYPE monitor_2(BIF_ALIST_2)
@@ -772,8 +888,9 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
     switch (BIF_ARG_1) {
     case am_time_offset: {
 	Eterm ref;
-	if (BIF_ARG_2 != am_clock_service)
-	    goto error;
+        if (BIF_ARG_2 != am_clock_service) {
+            goto badarg;
+        }
 	ref = erts_make_ref(BIF_P);
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
 	erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET,
@@ -783,46 +900,57 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
 	BIF_RET(ref);
     }
     case am_process:
+    case am_port:
 	break;
     default:
-	goto error;
+        goto badarg;
     }
 
-    if (is_internal_pid(target)) {
-    local_pid:
-	ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0);
-    } else if (is_external_pid(target)) {
+    if (is_internal_pid(target) && BIF_ARG_1 == am_process) {
+local_pid:
+        ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0);
+    } else if (is_external_pid(target) && BIF_ARG_1 == am_process) {
 	dep = external_pid_dist_entry(target);
 	if (dep == erts_this_dist_entry)
 	    goto local_pid;
 	ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, target, 0);
+    } else if (is_internal_port(target) && BIF_ARG_1 == am_port) {
+local_port:
+        ret = local_port_monitor(BIF_P, target);
+    } else if (is_external_port(target) && BIF_ARG_1 == am_port) {
+        dep = external_port_dist_entry(target);
+        if (dep == erts_this_dist_entry) {
+            goto local_port;
+        }
+        goto badarg; /* No want remote port */
     } else if (is_atom(target)) {
-	ret = local_name_monitor(BIF_P, target);
+        ret = local_name_monitor(BIF_P, BIF_ARG_1, target);
     } else if (is_tuple(target)) {
 	Eterm *tp = tuple_val(target);
 	Eterm remote_node;
 	Eterm name;
-	if (arityval(*tp) != 2) 
-	    goto error;
+        if (arityval(*tp) != 2) {
+            goto badarg;
+        }
 	remote_node = tp[2];
 	name = tp[1];
 	if (!is_atom(remote_node) || !is_atom(name)) {
-	    goto error;
+            goto badarg;
 	}
 	if (!erts_is_alive && remote_node != am_Noname) {
-	    goto error; /* Remote monitor from (this) undistributed node */
+            goto badarg; /* Remote monitor from (this) undistributed node */
 	}
 	dep = erts_sysname_to_connected_dist_entry(remote_node);
 	if (dep == erts_this_dist_entry) {
 	    deref_de = 1;
-	    ret = local_name_monitor(BIF_P, name);
+            ret = local_name_monitor(BIF_P, BIF_ARG_1, name);
 	} else {
 	    if (dep)
 		deref_de = 1;
 	    ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1);
 	}
     } else {
-    error:
+badarg:
 	ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
     }
     if (deref_de) {
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index b410578d37..3fb866733c 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -361,8 +361,13 @@ erts_print_system_version(int to, void *arg, Process *c_p)
 }
 
 typedef struct {
+    /* {Entity,Node} = {monitor.Name,monitor.Pid} for external by name
+     * {Entity,Node} = {monitor.Pid,NIL} for external/external by pid
+     * {Entity,Node} = {monitor.Name,erlang:node()} for internal by name */
     Eterm entity;
     Eterm node;
+    /* pid is actual target being monitored, no matter pid/port or name */
+    Eterm pid;
 } MonitorInfo;
 
 typedef struct {
@@ -420,21 +425,27 @@ static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp)
     EXTEND_MONITOR_INFOS(micp);
     if (is_atom(mon->pid)) { /* external by name */
 	micp->mi[micp->mi_i].entity = mon->name;
-	micp->mi[micp->mi_i].node = mon->pid;
-	micp->sz += 3; /* need one 2-tuple */
+        micp->mi[micp->mi_i].node = mon->pid;
+        micp->sz += 3; /* need one 2-tuple */
     } else if (is_external_pid(mon->pid)) { /* external by pid */
 	micp->mi[micp->mi_i].entity = mon->pid;
-	micp->mi[micp->mi_i].node = NIL;
-	micp->sz += NC_HEAP_SIZE(mon->pid);
+        micp->mi[micp->mi_i].node = NIL;
+        micp->sz += NC_HEAP_SIZE(mon->pid);
     } else if (!is_nil(mon->name)) { /* internal by name */
 	micp->mi[micp->mi_i].entity = mon->name;
-	micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname;
-	micp->sz += 3; /* need one 2-tuple */
+        micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname;
+        micp->sz += 3; /* need one 2-tuple */
     } else { /* internal by pid */
 	micp->mi[micp->mi_i].entity = mon->pid;
-	micp->mi[micp->mi_i].node = NIL;
+        micp->mi[micp->mi_i].node = NIL;
 	/* no additional heap space needed */
     }
+
+    /* have always pid at hand, to assist with figuring out if its a port or
+     * a process, when we monitored by name and process_info is requested.
+     * See: erl_bif_info.c:process_info_aux section for am_monitors */
+    micp->mi[micp->mi_i].pid = mon->pid;
+
     micp->mi_i++;
     micp->sz += 2 + 3; /* For a cons cell and a 2-tuple */
 }
@@ -1190,37 +1201,49 @@ process_info_aux(Process *BIF_P,
 
     case am_monitors: {
 	MonitorInfoCollection mic;
-	int i;
+        int i;
 
 	INIT_MONITOR_INFOS(mic);
-	erts_doforall_monitors(ERTS_P_MONITORS(rp),&collect_one_origin_monitor,&mic);
-	hp = HAlloc(BIF_P, 3 + mic.sz);
+        erts_doforall_monitors(ERTS_P_MONITORS(rp),
+                               &collect_one_origin_monitor, &mic);
+        hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
 	    if (is_atom(mic.mi[i].entity)) {
 		/* Monitor by name. 
-		 * Build {process, {Name, Node}} and cons it. 
+                 * Build {process|port, {Name, Node}} and cons it.
 		 */
 		Eterm t1, t2;
+                /* If pid is an atom, then it is a remote named monitor, which
+                   has to be a process */
+                Eterm m_type = is_port(mic.mi[i].pid) ? am_port : am_process;
+                ASSERT(is_pid(mic.mi[i].pid)
+                    || is_port(mic.mi[i].pid)
+                    || is_atom(mic.mi[i].pid));
 
 		t1 = TUPLE2(hp, mic.mi[i].entity, mic.mi[i].node);
 		hp += 3;
-		t2 = TUPLE2(hp, am_process, t1);
+                t2 = TUPLE2(hp, m_type, t1);
 		hp += 3;
 		res = CONS(hp, t2, res);
-		hp += 2;
+                hp += 2;
 	    }
 	    else {
-		/* Monitor by pid. Build {process, Pid} and cons it. */
+                /* Monitor by pid. Build {process|port, Pid} and cons it. */
 		Eterm t;
 		Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity);
-		t = TUPLE2(hp, am_process, pid);
+
+                Eterm m_type = is_port(mic.mi[i].pid) ? am_port : am_process;
+                ASSERT(is_pid(mic.mi[i].pid)
+                    || is_port(mic.mi[i].pid));
+
+                t = TUPLE2(hp, m_type, pid);
 		hp += 3;
 		res = CONS(hp, t, res);
-		hp += 2;
+                hp += 2;
 	    }
 	}
-	DESTROY_MONITOR_INFOS(mic);
+        DESTROY_MONITOR_INFOS(mic);
 	break;
     }
 
@@ -2880,7 +2903,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 */
 
 Eterm
-erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm item)
+erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
+                   Eterm item)
 {
     Eterm res = THE_NON_VALUE;
 
@@ -2928,8 +2952,8 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm ite
 	Eterm item;
 
 	INIT_MONITOR_INFOS(mic);
-
-	erts_doforall_monitors(ERTS_P_MONITORS(prt), &collect_one_origin_monitor, &mic);
+        erts_doforall_monitors(ERTS_P_MONITORS(prt),
+                               &collect_one_origin_monitor, &mic);
 
 	if (szp)
 	    *szp += mic.sz;
@@ -2938,14 +2962,16 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm ite
 	    res = NIL;
 	    for (i = 0; i < mic.mi_i; i++) {
 		Eterm t;
-		item = STORE_NC(hpp, ohp, mic.mi[i].entity);
-		t = TUPLE2(*hpp, am_process, item);
+                Eterm m_type;
+
+                item = STORE_NC(hpp, ohp, mic.mi[i].entity);
+                m_type = is_port(item) ? am_port : am_process;
+                t = TUPLE2(*hpp, m_type, item);
 		*hpp += 3;
 		res = CONS(*hpp, t, res);
 		*hpp += 2;
 	    }
-	}
-
+        } // hpp
 	DESTROY_MONITOR_INFOS(mic);
 
 	if (szp) {
@@ -2953,6 +2979,32 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm ite
 	    goto done;
 	}
     }
+    else if (item == am_monitored_by) {
+        MonitorInfoCollection mic;
+        int i;
+        Eterm item;
+
+        INIT_MONITOR_INFOS(mic);
+        erts_doforall_monitors(ERTS_P_MONITORS(prt),
+                               &collect_one_target_monitor, &mic);
+        if (szp)
+            *szp += mic.sz;
+
+        if (hpp) {
+            res = NIL;
+            for (i = 0; i < mic.mi_i; ++i) {
+                item = STORE_NC(hpp, ohp, mic.mi[i].entity);
+                res = CONS(*hpp, item, res);
+                *hpp += 2;
+            }
+        } // hpp
+        DESTROY_MONITOR_INFOS(mic);
+
+        if (szp) {
+            res = am_true;
+            goto done;
+        }
+    }
     else if (item == am_name) {
 	int count = sys_strlen(prt->name);
 
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 37f4e1de49..208935bed9 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -139,6 +139,12 @@ sig_lookup_port(Process *c_p, Eterm id_or_name)
     return lookup_port(c_p, id_or_name, ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
 }
 
+/* Non-inline copy of sig_lookup_port to be exported */
+Port *erts_sig_lookup_port(Process *c_p, Eterm id_or_name)
+{
+    return lookup_port(c_p, id_or_name, ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
+}
+
 static ERTS_INLINE Port *
 data_lookup_port(Process *c_p, Eterm id_or_name)
 {
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index f0075ca2b9..f90844ccc8 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -361,6 +361,8 @@ Eterm erts_request_io_bytes(Process *c_p);
 #define ERTS_PORT_REDS_CONNECT		(CONTEXT_REDS/200)
 #define ERTS_PORT_REDS_UNLINK		(CONTEXT_REDS/200)
 #define ERTS_PORT_REDS_LINK		(CONTEXT_REDS/200)
+#define ERTS_PORT_REDS_MONITOR		(CONTEXT_REDS/200)
+#define ERTS_PORT_REDS_DEMONITOR	(CONTEXT_REDS/200)
 #define ERTS_PORT_REDS_BADSIG		(CONTEXT_REDS/200)
 #define ERTS_PORT_REDS_CONTROL		(CONTEXT_REDS/100)
 #define ERTS_PORT_REDS_CALL		(CONTEXT_REDS/50)
@@ -850,16 +852,20 @@ void erts_port_resume_procs(Port *);
 
 struct binary;
 
-#define ERTS_P2P_SIG_TYPE_BAD			0
-#define ERTS_P2P_SIG_TYPE_OUTPUT		1
-#define ERTS_P2P_SIG_TYPE_OUTPUTV		2
-#define ERTS_P2P_SIG_TYPE_CONNECT		3
-#define ERTS_P2P_SIG_TYPE_EXIT			4
-#define ERTS_P2P_SIG_TYPE_CONTROL		5
-#define ERTS_P2P_SIG_TYPE_CALL			6
-#define ERTS_P2P_SIG_TYPE_INFO			7
-#define ERTS_P2P_SIG_TYPE_LINK			8
-#define ERTS_P2P_SIG_TYPE_UNLINK		9
+enum {
+    ERTS_P2P_SIG_TYPE_BAD       = 0,
+    ERTS_P2P_SIG_TYPE_OUTPUT    = 1,
+    ERTS_P2P_SIG_TYPE_OUTPUTV   = 2,
+    ERTS_P2P_SIG_TYPE_CONNECT   = 3,
+    ERTS_P2P_SIG_TYPE_EXIT      = 4,
+    ERTS_P2P_SIG_TYPE_CONTROL   = 5,
+    ERTS_P2P_SIG_TYPE_CALL      = 6,
+    ERTS_P2P_SIG_TYPE_INFO      = 7,
+    ERTS_P2P_SIG_TYPE_LINK      = 8,
+    ERTS_P2P_SIG_TYPE_UNLINK    = 9,
+    ERTS_P2P_SIG_TYPE_MONITOR   = 10,
+    ERTS_P2P_SIG_TYPE_DEMONITOR = 11
+};
 
 #define ERTS_P2P_SIG_TYPE_BITS			4
 #define ERTS_P2P_SIG_TYPE_MASK \
@@ -921,6 +927,15 @@ struct ErtsProc2PortSigData_ {
 	struct {
 	    Eterm from;
 	} unlink;
+        struct {
+            Eterm origin;   /* who receives monitor event, pid */
+            Eterm name;     /* either name for named monitor, or port id */
+        } monitor;
+        struct {
+            Eterm origin;   /* who is at the other end of the monitor, pid */
+            Eterm name;     /* port id */
+            Uint32 ref[ERTS_MAX_REF_NUMBERS]; /* box contents of a ref */
+        } demonitor;
     } u;
 } ;
 
@@ -1017,6 +1032,29 @@ ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm
 ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
 ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);
 
+/* Creates monitor between Origin and Target. Ref must be initialized to
+ * a reference (ref may be rewritten to be used to serve additionally as a
+ * signal id). Name is atom if user monitors port by name or NIL */
+ErtsPortOpResult erts_port_monitor(Process *origin, Port *target, Eterm name,
+                                   Eterm *ref);
+
+typedef enum {
+    /* Normal demonitor rules apply with locking and reductions bump */
+    ERTS_PORT_DEMONITOR_NORMAL = 1,
+    /* Relaxed demonitor rules when process is about to die, which means that
+     * pid lookup won't work, locks won't work, no reductions bump. */
+    ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED = 2,
+} ErtsDemonitorMode;
+
+/* Removes monitor between origin and target, identified by ref.
+ * origin_is_dying can be 0 (false, normal locking rules and reductions bump
+ * apply) or 1 (true, in case when we avoid origin locking) */
+ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode,
+                                    Port *target, Eterm ref,
+                                    Eterm *trap_ref);
+/* defined in erl_bif_port.c */
+Port *erts_sig_lookup_port(Process *c_p, Eterm id_or_name);
+
 int erts_port_output_async(Port *, Eterm, Eterm);
 
 /*
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index c0b1d7246c..2c3d1bb7eb 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12239,7 +12239,6 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
     ExitMonitorContext *pcontext = vpcontext;
     DistEntry *dep;
     ErtsMonitor *rmon;
-    Process *rp;
 
     switch (mon->type) {
     case MON_ORIGIN:
@@ -12268,9 +12267,10 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_deref_dist_entry(dep);
 	    }
 	} else {
-	    ASSERT(is_pid(mon->pid));
-	    if (is_internal_pid(mon->pid)) { /* local by pid or name */
-		rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK);
+            ASSERT(is_pid(mon->pid) || is_port(mon->pid));
+            /* if is local by pid or name */
+            if (is_internal_pid(mon->pid)) {
+                Process *rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK);
 		if (!rp) {
 		    goto done;
 		}
@@ -12280,7 +12280,17 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		    goto done;
 		}
 		erts_destroy_monitor(rmon);
-	    } else { /* remote by pid */
+            } else if (is_internal_port(mon->pid)) {
+                /* Is a local port */
+                Port *prt = erts_port_lookup_raw(mon->pid);
+                if (!prt) {
+                    goto done;
+                }
+                erts_port_demonitor(pcontext->p,
+                                    ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED,
+                                    prt, mon->ref, NULL);
+                return; /* let erts_port_demonitor do the deletion */
+            } else { /* remote by pid */
 		ASSERT(is_external_pid(mon->pid));
 		dep = external_pid_dist_entry(mon->pid);
 		ASSERT(dep != NULL);
@@ -12318,6 +12328,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	    erts_port_release(prt); 
 	} else if (is_internal_pid(mon->pid)) {/* local by name or pid */
 	    Eterm watched;
+            Process *rp;
 	    DeclareTmpHeapNoproc(lhp,3);
 	    ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
 				      | ERTS_PROC_LOCKS_MSG_SEND);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 01df5476db..cb8792dffa 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1260,7 +1260,7 @@ typedef struct {
 /*
  * Try doing an immediate driver callback call from a process. If
  * this fail, the operation should be scheduled in the normal case...
- *
+ * Returns: ok to do the call, or error (lock busy, does not exist, etc)
  */
 static ERTS_INLINE ErtsTryImmDrvCallResult
 try_imm_drv_call(ErtsTryImmDrvCallState *sp)
@@ -3073,6 +3073,250 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp)
 					  port_sig_link);
 }
 
+static void
+port_monitor_failure(Eterm port_id, Eterm origin, Eterm ref_DOWN)
+{
+    Process       *origin_p;
+    ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK;
+    ASSERT(is_internal_pid(origin));
+
+    origin_p = erts_pid2proc(NULL, 0, origin, p_locks);
+    if (! origin_p) { return; }
+
+    /* Send the DOWN message immediately. Ref is made on the fly because
+     * caller has never seen it yet. */
+    erts_queue_monitor_message(origin_p, &p_locks, ref_DOWN,
+                               am_port, port_id, am_noproc);
+    erts_smp_proc_unlock(origin_p, p_locks);
+}
+
+/* Origin wants to monitor port Prt. State contains possible error, which has
+ * happened just before. Name is either NIL or an atom, if user monitors
+ * a port by name. Ref is premade reference that will be returned to user */
+static void
+port_monitor(Port *prt, erts_aint32_t state, Eterm origin,
+             Eterm name, Eterm ref)
+{
+    Eterm name_or_nil = is_atom(name) ? name : NIL;
+
+    ASSERT(is_pid(origin));
+    ASSERT(is_atom(name) || is_port(name) || name == NIL);
+    ASSERT(is_internal_ref(ref));
+
+    if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
+        ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK;
+
+        Process *origin_p = erts_pid2proc(NULL, 0, origin, p_locks);
+        if (! origin_p) {
+            goto failure;
+        }
+        erts_add_monitor(&ERTS_P_MONITORS(origin_p), MON_ORIGIN, ref,
+                         prt->common.id, name_or_nil);
+        erts_add_monitor(&ERTS_P_MONITORS(prt), MON_TARGET, ref,
+                         origin, name_or_nil);
+
+        erts_smp_proc_unlock(origin_p, p_locks);
+    } else {
+failure:
+        port_monitor_failure(prt->common.id, origin, ref);
+    }
+}
+
+static int
+port_sig_monitor(Port *prt, erts_aint32_t state, int op,
+                 ErtsProc2PortSigData *sigdp)
+{
+    Eterm hp[REF_THING_SIZE];
+    Eterm ref = make_internal_ref(&hp);
+    write_ref_thing(hp, sigdp->ref[0], sigdp->ref[1], sigdp->ref[2]);
+
+    if (op == ERTS_PROC2PORT_SIG_EXEC) {
+        /* erts_add_monitor call inside port_monitor will copy ref from hp */
+        port_monitor(prt, state,
+                     sigdp->u.monitor.origin,
+                     sigdp->u.monitor.name,
+                     ref);
+    } else {
+        port_monitor_failure(sigdp->u.monitor.name,
+                             sigdp->u.monitor.origin,
+                             ref);
+    }
+    if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) {
+        port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
+    }
+    return ERTS_PORT_REDS_MONITOR;
+}
+
+/* Creates monitor between Origin and Target. Ref must be initialized to
+ * a reference (ref may be rewritten to be used to serve additionally as a
+ * signal id). Name is atom if user monitors port by name or NIL */
+ErtsPortOpResult
+erts_port_monitor(Process *origin, Port *port, Eterm name, Eterm *refp)
+{
+    ErtsProc2PortSigData *sigdp;
+    ErtsTryImmDrvCallState try_call_state
+        = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+                origin, port, ERTS_PORT_SFLGS_INVALID_LOOKUP,
+                0,
+                0, /* trap_ref is always set so !trap_ref always is false */
+                am_monitor);
+
+    ASSERT(origin);
+    ASSERT(port);
+    ASSERT(is_atom(name) || is_port(name));
+    ASSERT(refp);
+
+    switch (try_imm_drv_call(&try_call_state)) {
+    case ERTS_TRY_IMM_DRV_CALL_OK:
+        port_monitor(port, try_call_state.state, origin->common.id, name, *refp);
+        finalize_imm_drv_call(&try_call_state);
+        BUMP_REDS(origin, ERTS_PORT_REDS_MONITOR);
+        return ERTS_PORT_OP_DONE;
+    case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+        return ERTS_PORT_OP_BADARG;
+    default:
+        break; /* Schedule call instead... */
+    }
+
+    sigdp = erts_port_task_alloc_p2p_sig_data();
+    sigdp->flags = ERTS_P2P_SIG_TYPE_MONITOR;
+    sigdp->u.monitor.origin = origin->common.id;
+    sigdp->u.monitor.name   = name; /* either named monitor, or port id */
+
+    /* Ref contents will be initialized here */
+    return erts_schedule_proc2port_signal(origin, port, origin->common.id,
+                                          refp, sigdp, 0, NULL,
+                                          port_sig_monitor);
+}
+
+static void
+port_demonitor_failure(Eterm port_id, Eterm origin, Eterm ref)
+{
+    Process       *origin_p;
+    ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
+    ErtsMonitor   *mon1;
+    ASSERT(is_internal_pid(origin));
+
+    origin_p = erts_pid2proc(NULL, 0, origin, rp_locks);
+    if (! origin_p) { return; }
+
+    /* do not send any DOWN messages, drop monitors on process */
+    mon1 = erts_remove_monitor(&ERTS_P_MONITORS(origin_p), ref);
+    if (mon1 != NULL) {
+        erts_destroy_monitor(mon1);
+    }
+
+    erts_smp_proc_unlock(origin_p, rp_locks);
+}
+
+/* Origin wants to demonitor port Prt. State contains possible error, which has
+ * happened just before. Ref is reference to monitor */
+static void
+port_demonitor(Port *port, erts_aint32_t state, Eterm origin, Eterm ref)
+{
+    ASSERT(port);
+    ASSERT(is_pid(origin));
+    ASSERT(is_internal_ref(ref));
+
+    if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
+        ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK;
+        Process *origin_p = erts_pid2proc(NULL, 0, origin, p_locks);
+        if (origin_p) {
+            ErtsMonitor *mon1 = erts_remove_monitor(&ERTS_P_MONITORS(origin_p),
+                                                    ref);
+            if (mon1 != NULL) {
+                erts_destroy_monitor(mon1);
+            }
+        }
+        if (1) {
+            ErtsMonitor *mon2 = erts_remove_monitor(&ERTS_P_MONITORS(port),
+                                                    ref);
+            if (mon2 != NULL) {
+                erts_destroy_monitor(mon2);
+            }
+        }
+        if (origin_p) { /* when origin is dying, it won't be found */
+            erts_smp_proc_unlock(origin_p, p_locks);
+        }
+    } else {
+        port_demonitor_failure(port->common.id, origin, ref);
+    }
+}
+
+static int
+port_sig_demonitor(Port *prt, erts_aint32_t state, int op,
+                   ErtsProc2PortSigData *sigdp)
+{
+    Eterm hp[REF_THING_SIZE];
+    Eterm ref = make_internal_ref(&hp);
+    write_ref_thing(hp, sigdp->u.demonitor.ref[0],
+                    sigdp->u.demonitor.ref[1],
+                    sigdp->u.demonitor.ref[2]);
+    if (op == ERTS_PROC2PORT_SIG_EXEC) {
+        port_demonitor(prt, state, sigdp->u.demonitor.origin, ref);
+    } else {
+        port_demonitor_failure(sigdp->u.demonitor.name,
+                               sigdp->u.demonitor.origin,
+                               ref);
+    }
+    if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) {
+        port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
+    }
+    return ERTS_PORT_REDS_DEMONITOR;
+}
+
+/* Removes monitor between origin and target, identified by ref.
+ * Mode defines normal or relaxed demonitor rules (process is at death) */
+ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode,
+                                     Port *target, Eterm ref,
+                                     Eterm *trap_ref)
+{
+    Process *c_p = mode == ERTS_PORT_DEMONITOR_NORMAL ? origin : NULL;
+    ErtsProc2PortSigData *sigdp;
+    ErtsTryImmDrvCallState try_call_state
+        = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+                c_p,
+                target, ERTS_PORT_SFLGS_INVALID_LOOKUP,
+                0,
+                !trap_ref,
+                am_demonitor);
+
+    ASSERT(origin);
+    ASSERT(target);
+    ASSERT(is_internal_ref(ref));
+
+    switch (try_imm_drv_call(&try_call_state)) {
+    case ERTS_TRY_IMM_DRV_CALL_OK:
+        port_demonitor(target, try_call_state.state, origin->common.id, ref);
+        finalize_imm_drv_call(&try_call_state);
+        if (mode == ERTS_PORT_DEMONITOR_NORMAL) {
+            BUMP_REDS(origin, ERTS_PORT_REDS_DEMONITOR);
+        }
+        return ERTS_PORT_OP_DONE;
+    case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+        return ERTS_PORT_OP_BADARG;
+    default:
+        break; /* Schedule call instead... */
+    }
+
+    sigdp = erts_port_task_alloc_p2p_sig_data();
+    sigdp->flags = ERTS_P2P_SIG_TYPE_DEMONITOR;
+    sigdp->u.demonitor.origin = origin->common.id;
+    sigdp->u.demonitor.name = target->common.id;
+    {
+        RefThing *reft = ref_thing_ptr(ref);
+        /* Start from 1 skip ref arity */
+        sys_memcpy(sigdp->u.demonitor.ref,
+                   internal_thing_ref_numbers(reft),
+                   sizeof(sigdp->u.demonitor.ref));
+    }
+
+    /* Ref contents will be initialized here */
+    return erts_schedule_proc2port_signal(c_p, target, origin->common.id,
+                                          trap_ref, sigdp, 0, NULL,
+                                          port_sig_demonitor);
+}
+
 static void
 init_ack_send_reply(Port *port, Eterm resp)
 {
@@ -3942,23 +4186,30 @@ erts_terminate_port(Port *pp)
     terminate_port(pp);
 }
 
+static void port_fire_one_monitor(ErtsMonitor *mon, void *ctx0);
 static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc)
 {
-    ErtsMonitor *rmon;
-    Process *rp;
+    switch (mon->type) {
+    case MON_ORIGIN: {
+        ErtsMonitor *rmon;
+        Process *rp;
 
-    ASSERT(mon->type == MON_ORIGIN);
-    ASSERT(is_internal_pid(mon->pid));
-    rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK);
-    if (!rp) {
-	goto done;
-    }
-    rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-    if (rmon == NULL) {
-	goto done;
+        ASSERT(is_internal_pid(mon->pid));
+        rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK);
+        if (!rp) {
+            goto done;
+        }
+        rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
+        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        if (rmon == NULL) {
+            goto done;
+        }
+        erts_destroy_monitor(rmon);
+    } break;
+    case MON_TARGET: {
+        port_fire_one_monitor(mon, vpsc); /* forward call */
+    } break;
     }
-    erts_destroy_monitor(rmon);
  done:
     erts_destroy_monitor(mon);
 }
@@ -4039,6 +4290,43 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
     erts_destroy_link(lnk);
 }
 
+static void
+port_fire_one_monitor(ErtsMonitor *mon, void *ctx0)
+{
+    Process      *origin;
+    ErtsProcLocks origin_locks;
+
+    if (mon->type != MON_TARGET || ! is_pid(mon->pid)) {
+        return;
+    }
+    /*
+     * Proceed here if someone monitors us, we (port) are the target and
+     * origin is some process
+     */
+    origin_locks = ERTS_PROC_LOCKS_MSG_SEND | ERTS_PROC_LOCK_LINK;
+
+    origin = erts_pid2proc(NULL, 0, mon->pid, origin_locks);
+    if (origin) {
+        DeclareTmpHeapNoproc(lhp,3);
+        SweepContext    *ctx = (SweepContext *)ctx0;
+        ErtsMonitor     *rmon;
+        Eterm           watched = (is_atom(mon->name)
+                       ? TUPLE2(lhp, mon->name, erts_this_dist_entry->sysname)
+                       : ctx->port->common.id);
+
+        erts_queue_monitor_message(origin, &origin_locks, mon->ref, am_port,
+                                   watched, ctx->reason);
+        UnUseTmpHeapNoproc(3);
+
+        rmon = erts_remove_monitor(&ERTS_P_MONITORS(origin), mon->ref);
+        erts_smp_proc_unlock(origin, origin_locks);
+
+        if (rmon) {
+            erts_destroy_monitor(rmon);
+        }
+    }
+}
+
 /* 'from' is sending 'this_port' an exit signal, (this_port must be internal).
  * If reason is normal we don't do anything, *unless* from is our connected
  * process in which case we close the port. Any other reason kills the port.
@@ -4050,39 +4338,40 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
  */
 
 int
-erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed,
+erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed,
                        int drop_normal)
 {
    ErtsLink *lnk;
-   Eterm rreason;
+   Eterm modified_reason;
    erts_aint32_t state, set_state_flags;
 
    ERTS_SMP_CHK_NO_PROC_LOCKS;
-   ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
+   ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-   rreason = (reason == am_kill) ? am_killed : reason;
+   modified_reason = (reason == am_kill) ? am_killed : reason;
 
 #ifdef USE_VM_PROBES
    if (DTRACE_ENABLED(port_exit)) {
        DTRACE_CHARBUF(from_str, DTRACE_TERM_BUF_SIZE);
        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
-       DTRACE_CHARBUF(rreason_str, 64);
+       DTRACE_CHARBUF(reason_str, 64);
 
        erts_snprintf(from_str, sizeof(DTRACE_CHARBUF_NAME(from_str)), "%T", from);
-       dtrace_port_str(p, port_str);
-       erts_snprintf(rreason_str, sizeof(DTRACE_CHARBUF_NAME(rreason_str)), "%T", rreason);
-       DTRACE4(port_exit, from_str, port_str, p->name, rreason_str);
+       dtrace_port_str(prt, port_str);
+       erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T",
+                     modified_reason);
+       DTRACE4(port_exit, from_str, port_str, prt->name, reason_str);
    }
 #endif
 
-   state = erts_atomic32_read_nob(&p->state);
+   state = erts_atomic32_read_nob(&prt->state);
    if (state & (ERTS_PORT_SFLGS_DEAD
 		| ERTS_PORT_SFLG_EXITING
 		| ERTS_PORT_SFLG_CLOSING))
        return 0;
 
-   if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(p)
-       && from != p->common.id && drop_normal) {
+   if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(prt)
+       && from != prt->common.id && drop_normal) {
        return 0;
    }
 
@@ -4090,53 +4379,54 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed,
    if (send_closed)
        set_state_flags |= ERTS_PORT_SFLG_SEND_CLOSED;
 
-   erts_port_task_sched_enter_exiting_state(&p->sched);
+   erts_port_task_sched_enter_exiting_state(&prt->sched);
    
-   state = erts_atomic32_read_bor_mb(&p->state, set_state_flags);
+   state = erts_atomic32_read_bor_mb(&prt->state, set_state_flags);
    state |= set_state_flags;
 
-   if (IS_TRACED_FL(p, F_TRACE_PORTS))
-   	trace_port(p, am_closed, reason);
+   if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+        trace_port(prt, am_closed, reason);
 
-   erts_trace_check_exiting(p->common.id);
+   erts_trace_check_exiting(prt->common.id);
 
-   set_busy_port(ERTS_Port2ErlDrvPort(p), 0);
+   set_busy_port(ERTS_Port2ErlDrvPort(prt), 0);
 
-   if (p->common.u.alive.reg != NULL)
-       (void) erts_unregister_name(NULL, 0, p, p->common.u.alive.reg->name);
+   if (prt->common.u.alive.reg != NULL)
+       (void) erts_unregister_name(NULL, 0, prt, prt->common.u.alive.reg->name);
 
    {
-       SweepContext sc = {p, rreason};
-       lnk = ERTS_P_LINKS(p);
-       ERTS_P_LINKS(p) = NULL;
+       SweepContext sc = {prt, modified_reason};
+       lnk = ERTS_P_LINKS(prt);
+       ERTS_P_LINKS(prt) = NULL;
        erts_sweep_links(lnk, &sweep_one_link, &sc);
    }
-   DRV_MONITOR_LOCK_PDL(p);
+   DRV_MONITOR_LOCK_PDL(prt);
    {
-       ErtsMonitor *moni = ERTS_P_MONITORS(p);
-       ERTS_P_MONITORS(p) = NULL;
-       erts_sweep_monitors(moni, &sweep_one_monitor, NULL);
+       SweepContext ctx = {prt, modified_reason};
+       ErtsMonitor *moni = ERTS_P_MONITORS(prt);
+       ERTS_P_MONITORS(prt) = NULL;
+       erts_sweep_monitors(moni, &sweep_one_monitor, &ctx);
    } 
-   DRV_MONITOR_UNLOCK_PDL(p);
+   DRV_MONITOR_UNLOCK_PDL(prt);
 
-   if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) {
-       erts_do_net_exits(p->dist_entry, rreason);
-       erts_deref_dist_entry(p->dist_entry); 
-       p->dist_entry = NULL;
-       erts_atomic32_read_band_relb(&p->state,
+   if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && prt->dist_entry) {
+       erts_do_net_exits(prt->dist_entry, modified_reason);
+       erts_deref_dist_entry(prt->dist_entry);
+       prt->dist_entry = NULL;
+       erts_atomic32_read_band_relb(&prt->state,
 				    ~ERTS_PORT_SFLG_DISTRIBUTION);
    }
        
-   if ((reason != am_kill) && !is_port_ioq_empty(p)) {
+   if ((reason != am_kill) && !is_port_ioq_empty(prt)) {
        /* must turn exiting flag off */
-       erts_atomic32_read_bset_relb(&p->state,
+       erts_atomic32_read_bset_relb(&prt->state,
 				    (ERTS_PORT_SFLG_EXITING
 				     | ERTS_PORT_SFLG_CLOSING),
 				    ERTS_PORT_SFLG_CLOSING);
-      flush_port(p);
+      flush_port(prt);
    }
    else {
-       terminate_port(p);
+       terminate_port(prt);
    }
 
    return 1;
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 77f79fcea4..ac7096745e 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -323,7 +323,8 @@ erts_whereis_name(Process *c_p,
 		  Process** proc,
 		  ErtsProcLocks need_locks,
 		  int flags,
-		  Port** port)
+		  Port** port,
+                  int lock_port)
 {
     RegProc* rp = NULL;
     HashValue hval;
@@ -406,31 +407,33 @@ erts_whereis_name(Process *c_p,
 	    *port = NULL;
 	else {
 #ifdef ERTS_SMP
-	    if (pending_port == rp->pt)
-		pending_port = NULL;
-	    else {
-		if (pending_port) {
-		    /* Ahh! Registered port changed while reg lock
-		       was unlocked... */
-		    erts_port_release(pending_port);
-		    pending_port = NULL;
-		}
+            if (lock_port) {
+                if (pending_port == rp->pt)
+                    pending_port = NULL;
+                else {
+                    if (pending_port) {
+                        /* Ahh! Registered port changed while reg lock
+                           was unlocked... */
+                        erts_port_release(pending_port);
+                        pending_port = NULL;
+                    }
 		    
-		if (erts_smp_port_trylock(rp->pt) == EBUSY) {
-		    Eterm id = rp->pt->common.id; /* id read only... */
-		    /* Unlock all locks, acquire port lock, and restart... */
-		    if (current_c_p_locks) {
-			erts_smp_proc_unlock(c_p, current_c_p_locks);
-			current_c_p_locks = 0;
-		    }
-		    reg_read_unlock();
-		    pending_port = erts_id2port(id);
-		    goto restart;
-		}
-	    }
+                    if (erts_smp_port_trylock(rp->pt) == EBUSY) {
+                        Eterm id = rp->pt->common.id; /* id read only... */
+                        /* Unlock all locks, acquire port lock, and restart... */
+                        if (current_c_p_locks) {
+                            erts_smp_proc_unlock(c_p, current_c_p_locks);
+                            current_c_p_locks = 0;
+                        }
+                        reg_read_unlock();
+                        pending_port = erts_id2port(id);
+                        goto restart;
+                    }
+                }
+                ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(rp->pt));
+            }
 #endif
 	    *port = rp->pt;
-	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(*port));
 	}
     }
 
@@ -452,7 +455,7 @@ erts_whereis_process(Process *c_p,
 		     int flags)
 {
     Process *proc;
-    erts_whereis_name(c_p, c_p_locks, name, &proc, need_locks, flags, NULL);
+    erts_whereis_name(c_p, c_p_locks, name, &proc, need_locks, flags, NULL, 0);
     return proc;
 }
 
diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h
index 88ab7b7bf1..d839f55d6b 100644
--- a/erts/emulator/beam/register.h
+++ b/erts/emulator/beam/register.h
@@ -49,7 +49,7 @@ int erts_register_name(Process *, Eterm, Eterm);
 Eterm erts_whereis_name_to_id(Process *, Eterm);
 void erts_whereis_name(Process *, ErtsProcLocks,
 		       Eterm, Process**, ErtsProcLocks, int,
-		       Port**);
+		       Port**, int);
 Process *erts_whereis_process(Process *,
 			      ErtsProcLocks,
 			      Eterm,
-- 
cgit v1.2.3


From 222b0d728869feae00fad9e7da7633e7cb349c6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 10 Jun 2016 19:01:43 +0200
Subject: erts: Fix undefined shift to msb in erl_thr_progress

---
 erts/emulator/beam/erl_thr_progress.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 542541165b..21938e7684 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -95,9 +95,9 @@
 
 #define ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL 100
 
-#define ERTS_THR_PRGR_LFLG_BLOCK	(((erts_aint32_t) 1) << 31)
-#define ERTS_THR_PRGR_LFLG_NO_LEADER	(((erts_aint32_t) 1) << 30)
-#define ERTS_THR_PRGR_LFLG_WAITING_UM	(((erts_aint32_t) 1) << 29)
+#define ERTS_THR_PRGR_LFLG_BLOCK	((erts_aint32_t) (1U << 31))
+#define ERTS_THR_PRGR_LFLG_NO_LEADER	((erts_aint32_t) (1U << 30))
+#define ERTS_THR_PRGR_LFLG_WAITING_UM	((erts_aint32_t) (1U << 29))
 #define ERTS_THR_PRGR_LFLG_ACTIVE_MASK	(~(ERTS_THR_PRGR_LFLG_NO_LEADER	\
 					   | ERTS_THR_PRGR_LFLG_BLOCK	\
 					   | ERTS_THR_PRGR_LFLG_WAITING_UM))
@@ -142,8 +142,8 @@ init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 #warning "Thread progress state debug is on"
 #endif
 
-#define ERTS_THR_PROGRESS_STATE_DEBUG_LEADER	(((erts_aint32_t) 1) << 0)
-#define ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE	(((erts_aint32_t) 1) << 1)
+#define ERTS_THR_PROGRESS_STATE_DEBUG_LEADER	((erts_aint32_t) (1U << 0))
+#define ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE	((erts_aint32_t) (1U << 1))
 
 #define ERTS_THR_PROGRESS_STATE_DEBUG_INIT(ID)						\
     erts_atomic32_init_nob(&intrnl->thr[(ID)].data.state_debug,				\
@@ -179,10 +179,10 @@ do {											\
 
 #endif /* ERTS_THR_PROGRESS_STATE_DEBUG */
 
-#define ERTS_THR_PRGR_BLCKR_INVALID (~((erts_aint32_t) 0))
-#define ERTS_THR_PRGR_BLCKR_UNMANAGED (((erts_aint32_t) 1) << 31)
+#define ERTS_THR_PRGR_BLCKR_INVALID        ((erts_aint32_t) (~0U))
+#define ERTS_THR_PRGR_BLCKR_UNMANAGED      ((erts_aint32_t) (1U << 31))
 
-#define ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING (((erts_aint32_t) 1) << 31)
+#define ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING  ((erts_aint32_t) (1U << 31))
 
 #define ERTS_THR_PRGR_BM_BITS 32
 #define ERTS_THR_PRGR_BM_SHIFT 5
@@ -1186,7 +1186,7 @@ wakeup_unmanaged_threads(ErtsThrPrgrUnmanagedWakeupData *umwd)
 	    int hbase = hix << ERTS_THR_PRGR_BM_SHIFT;
 	    int hbit;
 	    for (hbit = 0; hbit < ERTS_THR_PRGR_BM_BITS; hbit++) {
-		if (hmask & (1 << hbit)) {
+		if (hmask & (1U << hbit)) {
 		    erts_aint_t lmask;
 		    int lix = hbase + hbit;
 		    ASSERT(0 <= lix && lix < umwd->low_sz);
@@ -1195,7 +1195,7 @@ wakeup_unmanaged_threads(ErtsThrPrgrUnmanagedWakeupData *umwd)
 			int lbase = lix << ERTS_THR_PRGR_BM_SHIFT;
 			int lbit;
 			for (lbit = 0; lbit < ERTS_THR_PRGR_BM_BITS; lbit++) {
-			    if (lmask & (1 << lbit)) {
+			    if (lmask & (1U << lbit)) {
 				int id = lbase + lbit;
 				wakeup_unmanaged(id);
 			    }
-- 
cgit v1.2.3


From 45c595ab6150792b2b7be9a407c3d30e634e3924 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 10 Jun 2016 19:47:47 +0200
Subject: erts: Fix undefined shift to msb in erl_process

---
 erts/emulator/beam/erl_process.c | 2 +-
 erts/emulator/beam/erl_ptab.h    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 48f89d2bd7..66f22979ad 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -3509,7 +3509,7 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one)
 #endif
 
 #define ERTS_NO_USED_RUNQS_SHIFT 16
-#define ERTS_NO_RUNQS_MASK 0xffff
+#define ERTS_NO_RUNQS_MASK 0xffffU
 
 #if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK
 #  error "Too large amount of schedulers allowed"
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index a5931ffc25..fecfd96ab0 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -168,7 +168,7 @@ typedef struct {
 
 #define ERTS_PTAB_INVALID_ID(TAG)					\
     ((Eterm)								\
-     ((((1 << ERTS_PTAB_ID_DATA_SIZE) - 1) << ERTS_PTAB_ID_DATA_SHIFT)	\
+     ((((1U << ERTS_PTAB_ID_DATA_SIZE) - 1) << ERTS_PTAB_ID_DATA_SHIFT)	\
       | (TAG)))
 
 #define erts_ptab_is_valid_id(ID)					\
-- 
cgit v1.2.3


From 4e87bebe004039465548ed432d363f2d7e42fee6 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas.larsson@erlang-solutions.com>
Date: Mon, 13 Jun 2016 10:08:47 +0200
Subject: erts: Allow enif_port_command in non-sched thread

It has to be possible to send trace messages to a port
from non-schedulers threads (specifically from the
sys_msg_dispatcher).
---
 erts/emulator/beam/erl_nif.c | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 039f97ef43..23931f0e54 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -769,32 +769,25 @@ enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port,
 
     if (scheduler > 0)
 	prt = erts_port_lookup(to_port->port_id, iflags);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    else if (scheduler < 0) {
+    else {
+#ifdef ERTS_SMP
 	if (ERTS_PROC_IS_EXITING(c_p))
 	    return 0;
 	prt = erts_thr_port_lookup(to_port->port_id, iflags);
-    }
+#else
+        erts_exit(ERTS_ABORT_EXIT,
+		  "enif_port_command: called from non-scheduler "
+                  "thread on non-SMP VM");
 #endif
-    else {
-	erts_exit(ERTS_ABORT_EXIT, "enif_port_command: "
-		  "called from non-scheduler thread");
     }
 
     if (!prt)
 	res = 0;
-    else {
-
-	if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
-	    trace_port_receive(prt, c_p->common.id, am_command, msg);
-
-	res = erts_port_output_async(prt, c_p->common.id, msg);
-    }
+    else
+        res = erts_port_output_async(prt, c_p->common.id, msg);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (scheduler < 0)
+    if (scheduler <= 0)
 	erts_port_dec_refc(prt);
-#endif
 
     return res;
 }
-- 
cgit v1.2.3


From 8c19f3d39ff848817e8032f081577afbd5e75493 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 13 Jun 2016 16:55:55 +0200
Subject: erts: Fix profile runnable processes race

---
 erts/emulator/beam/erl_trace.c | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 3dca58d60b..4cf38bf894 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2115,32 +2115,36 @@ profile_runnable_proc(Process *p, Eterm status){
     Eterm *hp, msg;
     Eterm where = am_undefined;
     ErlHeapFragment *bp = NULL;
-    int use_current = 1;
+    BeamInstr *current = NULL;
 
 #ifndef ERTS_SMP
 #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
-
     DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
     hp = local_heap;
 #else
+    ErtsThrPrgrDelayHandle dhndl;
     Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1;
 #endif
-	
-    if (ERTS_PROC_IS_EXITING(p)) {
-        use_current = 0;
-        /* could probably set 'where' to 'exiting' here,
-         * though it's not documented as such */
-    } else {
-        if (!p->current) {
-            p->current = find_function_from_pc(p->i);
+    /* Assumptions:
+     * We possibly don't have the MAIN_LOCK for the process p here.
+     * We assume that we can read from p->current and p->i atomically
+     */
+#ifdef ERTS_SMP
+    dhndl = erts_thr_progress_unmanaged_delay(); /* suspend purge operations */
+#endif
+
+    if (!ERTS_PROC_IS_EXITING(p)) {
+        if (p->current) {
+            current = p->current;
+        } else {
+            current = find_function_from_pc(p->i);
         }
-        use_current = p->current != NULL;
     }
 
 #ifdef ERTS_SMP
-    if (!use_current) {
+    if (!current) {
 	hsz -= 4;
     }
 
@@ -2148,11 +2152,15 @@ profile_runnable_proc(Process *p, Eterm status){
     hp = bp->mem;
 #endif
 
-    if (use_current) {
-	where = TUPLE3(hp, p->current[0], p->current[1], make_small(p->current[2])); hp += 4;
+    if (current) {
+	where = TUPLE3(hp, current[0], current[1], make_small(current[2])); hp += 4;
     } else {
 	where = make_small(0);
     }
+
+#ifdef ERTS_SMP
+    erts_thr_progress_unmanaged_continue(dhndl);
+#endif
 	
     erts_smp_mtx_lock(&smq_mtx);
 
-- 
cgit v1.2.3


From e3a07b08709bd7eacae0bcb0353c347475134ad7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Jun 2016 16:32:16 +0200
Subject: erts: Change local sysname for ETS compressed

Yes this is an ugly workaround. One approach for a better
solution could be to introduce an internal secret atom
tagged as an atom with a unique index, but impossible
to find by string hash lookup/insert.
---
 erts/emulator/beam/external.c | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 3c002d43a7..beed847578 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2159,12 +2159,23 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
     return ep;
 }
 
+/*
+ * We use this atom as sysname in local pid/port/refs
+ * for the ETS compressed format (DFLAG_INTERNAL_TAGS).
+ *
+ * We used atom '' earlier but that turned out to cause problems
+ * for buggy erl_interface/ic usage of c-nodes with empty node names.
+ * A long atom reduces risk of nodes actually called this and the length
+ * does not matter anyway as it's encoded with atom index (ATOM_INTERNAL_REF2).
+ */
+#define INTERNAL_LOCAL_SYSNAME am_await_microstate_accounting_modifications
+
 static byte*
 enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
 {
     Uint on, os;
     Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS))
-		      ? am_Empty : pid_node_name(pid));
+		      ? INTERNAL_LOCAL_SYSNAME : pid_node_name(pid));
     Uint32 creation = pid_creation(pid);
     byte* tagp = ep++;
 
@@ -2268,7 +2279,7 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp)
 
 static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint32 creation)
 {
-    if (sysname == am_Empty)  /* && DFLAG_INTERNAL_TAGS */
+    if (sysname == INTERNAL_LOCAL_SYSNAME)  /* && DFLAG_INTERNAL_TAGS */
 	return erts_this_node;
 
     if (sysname == erts_this_node->sysname
@@ -2555,7 +2566,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	case EXTERNAL_REF_DEF: {
 	    Uint32 *ref_num;
 	    Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj))
-			     ? am_Empty : ref_node_name(obj));
+			     ? INTERNAL_LOCAL_SYSNAME : ref_node_name(obj));
             Uint32 creation = ref_creation(obj);
             byte* tagp = ep++;
 
@@ -2584,7 +2595,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	case PORT_DEF:
 	case EXTERNAL_PORT_DEF: {
 	    Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj))
-			     ? am_Empty : port_node_name(obj));
+			     ? INTERNAL_LOCAL_SYSNAME : port_node_name(obj));
             Uint32 creation = port_creation(obj);
             byte* tagp = ep++;
 
-- 
cgit v1.2.3